2019-10-16 18:27:53 +00:00
|
|
|
#include <Storages/MergeTree/MergeTreeDataPartWriterCompact.h>
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2019-10-21 17:23:06 +00:00
|
|
|
size_t MergeTreeDataPartWriterCompact::write(
|
|
|
|
const Block & block, const IColumn::Permutation * permutation,
|
|
|
|
size_t from_mark, size_t index_offset,
|
2019-10-21 00:28:29 +00:00
|
|
|
const MergeTreeIndexGranularity & index_granularity,
|
2019-10-16 18:27:53 +00:00
|
|
|
const Block & primary_key_block, const Block & skip_indexes_block)
|
|
|
|
{
|
|
|
|
size_t total_rows = block.rows();
|
|
|
|
size_t current_mark = from_mark;
|
|
|
|
size_t current_row = 0;
|
|
|
|
|
|
|
|
ColumnsWithTypeAndName columns_to_write(columns_list.size());
|
|
|
|
auto it = columns_list.begin();
|
|
|
|
for (size_t i = 0; i < columns_list.size(); ++i, ++it)
|
|
|
|
{
|
|
|
|
if (permutation)
|
|
|
|
{
|
|
|
|
if (primary_key_block.has(it->name))
|
|
|
|
columns_to_write[i] = primary_key_block.getByName(it->name);
|
|
|
|
else if (skip_indexes_block.has(it->name))
|
|
|
|
columns_to_write[i] = skip_indexes_block.getByName(it->name);
|
|
|
|
else
|
|
|
|
{
|
|
|
|
columns_to_write[i] = block.getByName(it->name);
|
|
|
|
columns_to_write[i].column = columns_to_write[i].column->permute(*permutation, 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
columns_to_write[i] = block.getByName(it->name);
|
|
|
|
}
|
|
|
|
|
|
|
|
while (current_row < total_rows)
|
|
|
|
{
|
|
|
|
bool write_marks = true;
|
|
|
|
size_t rows_to_write;
|
2019-10-21 00:28:29 +00:00
|
|
|
if (current_row == 0 && index_offset != 0)
|
2019-10-16 18:27:53 +00:00
|
|
|
{
|
2019-10-21 00:28:29 +00:00
|
|
|
rows_to_write = index_offset;
|
2019-10-16 18:27:53 +00:00
|
|
|
write_marks = false;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2019-10-21 00:28:29 +00:00
|
|
|
rows_to_write = index_granularity.getMarkRows(current_mark);
|
2019-10-16 18:27:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (write_marks)
|
|
|
|
{
|
2019-10-21 00:28:29 +00:00
|
|
|
writeIntBinary(rows_to_write, stream->marks);
|
|
|
|
for (size_t i = 0; i < columns_to_write.size(); ++i)
|
|
|
|
{
|
|
|
|
writeIntBinary(stream->plain_hashing.count(), stream->marks);
|
|
|
|
writeIntBinary(stream->compressed.offset(), stream->marks);
|
|
|
|
current_row = writeColumnSingleGranule(columns_to_write[i], current_row, rows_to_write);
|
|
|
|
}
|
2019-10-16 18:27:53 +00:00
|
|
|
++current_mark;
|
|
|
|
}
|
2019-10-21 00:28:29 +00:00
|
|
|
else
|
|
|
|
{
|
|
|
|
for (size_t i = 0; i < columns_to_write.size(); ++i)
|
|
|
|
current_row = writeColumnSingleGranule(columns_to_write[i], current_row, rows_to_write);
|
|
|
|
}
|
2019-10-16 18:27:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/// We always write end granule for block in Compact parts.
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2019-10-21 00:28:29 +00:00
|
|
|
size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWithTypeAndName & column, size_t from_row, size_t number_of_rows)
|
2019-10-16 18:27:53 +00:00
|
|
|
{
|
2019-10-21 00:28:29 +00:00
|
|
|
IDataType::SerializeBinaryBulkStatePtr state;
|
|
|
|
IDataType::SerializeBinaryBulkSettings serialize_settings;
|
2019-10-16 18:27:53 +00:00
|
|
|
|
2019-10-21 15:33:59 +00:00
|
|
|
serialize_settings.getter = [this](IDataType::SubstreamPath) -> WriteBuffer * { return &stream->compressed; };
|
2019-10-21 00:28:29 +00:00
|
|
|
serialize_settings.position_independent_encoding = false;
|
|
|
|
serialize_settings.low_cardinality_max_dictionary_size = 0;
|
2019-10-16 18:27:53 +00:00
|
|
|
|
2019-10-21 00:28:29 +00:00
|
|
|
column.type->serializeBinaryBulkStatePrefix(serialize_settings, state);
|
|
|
|
column.type->serializeBinaryBulkWithMultipleStreams(*column.column, from_row, number_of_rows, serialize_settings, state);
|
|
|
|
column.type->serializeBinaryBulkStateSuffix(serialize_settings, state);
|
2019-10-16 18:27:53 +00:00
|
|
|
|
2019-10-21 00:28:29 +00:00
|
|
|
return from_row + number_of_rows;
|
2019-10-16 18:27:53 +00:00
|
|
|
}
|
|
|
|
|
2019-10-21 17:23:06 +00:00
|
|
|
void MergeTreeDataPartWriterCompact::finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark)
|
|
|
|
{
|
|
|
|
if (write_final_mark)
|
|
|
|
{
|
|
|
|
writeIntBinary(0, stream->marks);
|
|
|
|
for (size_t i = 0; i < columns_list.size(); ++i)
|
|
|
|
{
|
|
|
|
writeIntBinary(stream->plain_hashing.count(), stream->marks);
|
|
|
|
writeIntBinary(stream->compressed.offset(), stream->marks);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
stream->finalize();
|
|
|
|
stream->addToChecksums(checksums);
|
|
|
|
stream.reset();
|
|
|
|
}
|
|
|
|
|
2019-10-21 15:33:59 +00:00
|
|
|
}
|