2021-10-18 07:23:46 +00:00
|
|
|
#include <Interpreters/SquashingTransform.h>
|
2016-07-06 21:48:11 +00:00
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
2020-02-25 18:10:48 +00:00
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
|
|
|
|
}
|
2016-07-06 21:48:11 +00:00
|
|
|
|
2022-02-28 09:55:31 +00:00
|
|
|
SquashingTransform::SquashingTransform(size_t min_block_size_rows_, size_t min_block_size_bytes_)
|
2019-09-26 18:51:17 +00:00
|
|
|
: min_block_size_rows(min_block_size_rows_)
|
|
|
|
, min_block_size_bytes(min_block_size_bytes_)
|
2016-07-06 21:48:11 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2020-04-21 14:59:57 +00:00
|
|
|
Block SquashingTransform::add(Block && input_block)
|
|
|
|
{
|
|
|
|
return addImpl<Block &&>(std::move(input_block));
|
|
|
|
}
|
|
|
|
|
|
|
|
Block SquashingTransform::add(const Block & input_block)
|
|
|
|
{
|
|
|
|
return addImpl<const Block &>(input_block);
|
|
|
|
}
|
2016-07-06 21:48:11 +00:00
|
|
|
|
2020-04-21 14:59:57 +00:00
|
|
|
/*
|
|
|
|
* To minimize copying, accept two types of argument: const reference for output
|
2020-10-26 19:12:40 +00:00
|
|
|
* stream, and rvalue reference for input stream, and decide whether to copy
|
2020-04-21 14:59:57 +00:00
|
|
|
* inside this function. This allows us not to copy Block unless we absolutely
|
|
|
|
* have to.
|
|
|
|
*/
|
|
|
|
template <typename ReferenceType>
|
|
|
|
Block SquashingTransform::addImpl(ReferenceType input_block)
|
2016-07-06 21:48:11 +00:00
|
|
|
{
|
2018-09-09 02:23:24 +00:00
|
|
|
/// End of input stream.
|
2020-04-21 14:59:57 +00:00
|
|
|
if (!input_block)
|
2020-04-17 20:05:35 +00:00
|
|
|
{
|
2020-04-21 14:59:57 +00:00
|
|
|
Block to_return;
|
|
|
|
std::swap(to_return, accumulated_block);
|
2020-04-17 20:05:35 +00:00
|
|
|
return to_return;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-09-20 12:12:32 +00:00
|
|
|
/// Just read block is already enough.
|
2020-04-21 14:59:57 +00:00
|
|
|
if (isEnoughSize(input_block))
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
/// If no accumulated data, return just read block.
|
2020-04-21 14:59:57 +00:00
|
|
|
if (!accumulated_block)
|
2020-04-17 20:05:35 +00:00
|
|
|
{
|
2020-04-21 14:59:57 +00:00
|
|
|
return std::move(input_block);
|
2020-04-17 20:05:35 +00:00
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-09-09 02:23:24 +00:00
|
|
|
/// Return accumulated data (maybe it has small size) and place new block to accumulated data.
|
2020-04-21 14:59:57 +00:00
|
|
|
Block to_return = std::move(input_block);
|
|
|
|
std::swap(to_return, accumulated_block);
|
|
|
|
return to_return;
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Accumulated block is already enough.
|
2020-04-21 14:59:57 +00:00
|
|
|
if (isEnoughSize(accumulated_block))
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
/// Return accumulated data and place new block to accumulated data.
|
2020-04-21 14:59:57 +00:00
|
|
|
Block to_return = std::move(input_block);
|
|
|
|
std::swap(to_return, accumulated_block);
|
|
|
|
return to_return;
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2020-04-21 14:59:57 +00:00
|
|
|
append<ReferenceType>(std::move(input_block));
|
|
|
|
if (isEnoughSize(accumulated_block))
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2020-04-21 14:59:57 +00:00
|
|
|
Block to_return;
|
|
|
|
std::swap(to_return, accumulated_block);
|
2020-04-17 20:05:35 +00:00
|
|
|
return to_return;
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Squashed block is not ready.
|
2020-04-21 14:59:57 +00:00
|
|
|
return {};
|
2016-07-06 21:48:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-04-21 14:59:57 +00:00
|
|
|
template <typename ReferenceType>
|
|
|
|
void SquashingTransform::append(ReferenceType input_block)
|
2016-07-06 21:48:11 +00:00
|
|
|
{
|
2020-04-21 14:59:57 +00:00
|
|
|
if (!accumulated_block)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2020-04-21 14:59:57 +00:00
|
|
|
accumulated_block = std::move(input_block);
|
2017-04-01 07:20:54 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2020-04-21 14:59:57 +00:00
|
|
|
assert(blocksHaveEqualStructure(input_block, accumulated_block));
|
2020-04-17 20:05:35 +00:00
|
|
|
|
2020-04-21 14:59:57 +00:00
|
|
|
for (size_t i = 0, size = accumulated_block.columns(); i < size; ++i)
|
2019-09-26 18:51:17 +00:00
|
|
|
{
|
2020-04-21 14:59:57 +00:00
|
|
|
const auto source_column = input_block.getByPosition(i).column;
|
|
|
|
|
2020-05-14 08:30:18 +00:00
|
|
|
auto mutable_column = IColumn::mutate(std::move(accumulated_block.getByPosition(i).column));
|
2020-04-21 14:59:57 +00:00
|
|
|
mutable_column->insertRangeFrom(*source_column, 0, source_column->size());
|
|
|
|
accumulated_block.getByPosition(i).column = std::move(mutable_column);
|
2019-09-26 18:51:17 +00:00
|
|
|
}
|
2018-09-08 19:23:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-04-21 14:59:57 +00:00
|
|
|
bool SquashingTransform::isEnoughSize(const Block & block)
|
2018-09-08 19:23:48 +00:00
|
|
|
{
|
|
|
|
size_t rows = 0;
|
|
|
|
size_t bytes = 0;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2020-04-21 14:59:57 +00:00
|
|
|
for (const auto & [column, type, name] : block)
|
2017-12-15 01:34:30 +00:00
|
|
|
{
|
2018-09-08 19:23:48 +00:00
|
|
|
if (!rows)
|
|
|
|
rows = column->size();
|
|
|
|
else if (rows != column->size())
|
2023-01-23 21:13:58 +00:00
|
|
|
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Sizes of columns doesn't match");
|
2018-09-08 19:23:48 +00:00
|
|
|
|
|
|
|
bytes += column->byteSize();
|
2017-12-15 01:34:30 +00:00
|
|
|
}
|
2018-09-08 19:23:48 +00:00
|
|
|
|
|
|
|
return isEnoughSize(rows, bytes);
|
2016-07-06 21:48:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
bool SquashingTransform::isEnoughSize(size_t rows, size_t bytes) const
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
return (!min_block_size_rows && !min_block_size_bytes)
|
|
|
|
|| (min_block_size_rows && rows >= min_block_size_rows)
|
|
|
|
|| (min_block_size_bytes && bytes >= min_block_size_bytes);
|
2016-07-06 21:48:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|