ClickHouse/src/Interpreters/SquashingTransform.cpp

130 lines
3.5 KiB
C++
Raw Normal View History

#include <Interpreters/SquashingTransform.h>
namespace DB
{
2020-02-25 18:10:48 +00:00
namespace ErrorCodes
{
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
}
SquashingTransform::SquashingTransform(size_t min_block_size_rows_, size_t min_block_size_bytes_)
: min_block_size_rows(min_block_size_rows_)
, min_block_size_bytes(min_block_size_bytes_)
{
}
2020-04-21 14:59:57 +00:00
Block SquashingTransform::add(Block && input_block)
{
return addImpl<Block &&>(std::move(input_block));
}
Block SquashingTransform::add(const Block & input_block)
{
return addImpl<const Block &>(input_block);
}
2020-04-21 14:59:57 +00:00
/*
* To minimize copying, accept two types of argument: const reference for output
2020-10-26 19:12:40 +00:00
* stream, and rvalue reference for input stream, and decide whether to copy
2020-04-21 14:59:57 +00:00
* inside this function. This allows us not to copy Block unless we absolutely
* have to.
*/
template <typename ReferenceType>
Block SquashingTransform::addImpl(ReferenceType input_block)
{
2018-09-09 02:23:24 +00:00
/// End of input stream.
2020-04-21 14:59:57 +00:00
if (!input_block)
2020-04-17 20:05:35 +00:00
{
2020-04-21 14:59:57 +00:00
Block to_return;
std::swap(to_return, accumulated_block);
2020-04-17 20:05:35 +00:00
return to_return;
}
/// Just read block is already enough.
2020-04-21 14:59:57 +00:00
if (isEnoughSize(input_block))
{
/// If no accumulated data, return just read block.
2020-04-21 14:59:57 +00:00
if (!accumulated_block)
2020-04-17 20:05:35 +00:00
{
2020-04-21 14:59:57 +00:00
return std::move(input_block);
2020-04-17 20:05:35 +00:00
}
2018-09-09 02:23:24 +00:00
/// Return accumulated data (maybe it has small size) and place new block to accumulated data.
2020-04-21 14:59:57 +00:00
Block to_return = std::move(input_block);
std::swap(to_return, accumulated_block);
return to_return;
}
/// Accumulated block is already enough.
2020-04-21 14:59:57 +00:00
if (isEnoughSize(accumulated_block))
{
/// Return accumulated data and place new block to accumulated data.
2020-04-21 14:59:57 +00:00
Block to_return = std::move(input_block);
std::swap(to_return, accumulated_block);
return to_return;
}
2020-04-21 14:59:57 +00:00
append<ReferenceType>(std::move(input_block));
if (isEnoughSize(accumulated_block))
{
2020-04-21 14:59:57 +00:00
Block to_return;
std::swap(to_return, accumulated_block);
2020-04-17 20:05:35 +00:00
return to_return;
}
/// Squashed block is not ready.
2020-04-21 14:59:57 +00:00
return {};
}
2020-04-21 14:59:57 +00:00
template <typename ReferenceType>
void SquashingTransform::append(ReferenceType input_block)
{
2020-04-21 14:59:57 +00:00
if (!accumulated_block)
{
2020-04-21 14:59:57 +00:00
accumulated_block = std::move(input_block);
return;
}
2020-04-21 14:59:57 +00:00
assert(blocksHaveEqualStructure(input_block, accumulated_block));
2020-04-17 20:05:35 +00:00
2020-04-21 14:59:57 +00:00
for (size_t i = 0, size = accumulated_block.columns(); i < size; ++i)
{
2020-04-21 14:59:57 +00:00
const auto source_column = input_block.getByPosition(i).column;
2020-05-14 08:30:18 +00:00
auto mutable_column = IColumn::mutate(std::move(accumulated_block.getByPosition(i).column));
2020-04-21 14:59:57 +00:00
mutable_column->insertRangeFrom(*source_column, 0, source_column->size());
accumulated_block.getByPosition(i).column = std::move(mutable_column);
}
2018-09-08 19:23:48 +00:00
}
2020-04-21 14:59:57 +00:00
bool SquashingTransform::isEnoughSize(const Block & block)
2018-09-08 19:23:48 +00:00
{
size_t rows = 0;
size_t bytes = 0;
2020-04-21 14:59:57 +00:00
for (const auto & [column, type, name] : block)
{
2018-09-08 19:23:48 +00:00
if (!rows)
rows = column->size();
else if (rows != column->size())
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Sizes of columns doesn't match");
2018-09-08 19:23:48 +00:00
bytes += column->byteSize();
}
2018-09-08 19:23:48 +00:00
return isEnoughSize(rows, bytes);
}
bool SquashingTransform::isEnoughSize(size_t rows, size_t bytes) const
{
return (!min_block_size_rows && !min_block_size_bytes)
|| (min_block_size_rows && rows >= min_block_size_rows)
|| (min_block_size_bytes && bytes >= min_block_size_bytes);
}
}