ClickHouse/dbms/DataStreams/narrowBlockInputStreams.cpp
Ivan 97f2a2213e
Move all folders inside /dbms one level up (#9974)
* Move some code outside dbms/src folder
* Fix paths
2020-04-02 02:51:21 +03:00

73 lines
1.7 KiB
C++

#include <random>
#include <Common/thread_local_rng.h>
#include <DataStreams/ConcatBlockInputStream.h>
#include <Processors/ConcatProcessor.h>
#include <Processors/Pipe.h>
#include "narrowBlockInputStreams.h"
namespace DB
{
namespace
{
using Distribution = std::vector<size_t>;
Distribution getDistribution(size_t from, size_t to)
{
Distribution distribution(from);
for (size_t i = 0; i < from; ++i)
distribution[i] = i % to;
std::shuffle(distribution.begin(), distribution.end(), thread_local_rng);
return distribution;
}
}
BlockInputStreams narrowBlockInputStreams(BlockInputStreams & inputs, size_t width)
{
size_t size = inputs.size();
if (size <= width)
return inputs;
std::vector<BlockInputStreams> partitions(width);
auto distribution = getDistribution(size, width);
for (size_t i = 0; i < size; ++i)
partitions[distribution[i]].push_back(inputs[i]);
BlockInputStreams res(width);
for (size_t i = 0; i < width; ++i)
res[i] = std::make_shared<ConcatBlockInputStream>(partitions[i]);
return res;
}
Pipes narrowPipes(Pipes pipes, size_t width)
{
size_t size = pipes.size();
if (size <= width)
return pipes;
std::vector<Pipes> partitions(width);
auto distribution = getDistribution(size, width);
for (size_t i = 0; i < size; ++i)
partitions[distribution[i]].emplace_back(std::move(pipes[i]));
Pipes res;
res.reserve(width);
for (size_t i = 0; i < width; ++i)
{
auto processor = std::make_shared<ConcatProcessor>(partitions[i].at(0).getHeader(), partitions[i].size());
res.emplace_back(std::move(partitions[i]), std::move(processor));
}
return res;
}
}