thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-04 21:42:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

ColumnSparse: use varint for offsets

Browse Source
This commit is contained in:
Anton Popov 2021-05-12 04:36:47 +03:00
parent a842a599c4
commit e1e1435b00
2 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/DataTypes/Serializations/SerializationSparse.cpp
View File

@ -6,6 +6,7 @@
#include <Common/assert_cast.h> #include <Common/assert_cast.h>
#include <IO/ReadHelpers.h> #include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h> #include <IO/WriteHelpers.h>
#include <IO/VarInt.h>
namespace DB namespace DB
{ {
@ -13,7 +14,7 @@ namespace DB
namespace namespace
{ {
static constexpr auto END_OF_GRANULE_FLAG = 1ULL << 63; static constexpr auto END_OF_GRANULE_FLAG = 1ULL << 62;
struct DeserializeStateSparse : public ISerialization::DeserializeBinaryBulkState struct DeserializeStateSparse : public ISerialization::DeserializeBinaryBulkState
{ {
@ -34,13 +35,13 @@ void serializeOffsets(const IColumn::Offsets & offsets, WriteBuffer & ostr, size
for (size_t i = 0; i < size; ++i) for (size_t i = 0; i < size; ++i)
{ {
size_t group_size = offsets[i] - start; size_t group_size = offsets[i] - start;
writeIntBinary(group_size, ostr); writeVarUInt(group_size, ostr);
start += group_size + 1; start += group_size + 1;
} }
size_t group_size = start < end ? end - start : 0; size_t group_size = start < end ? end - start : 0;
group_size |= END_OF_GRANULE_FLAG; group_size |= END_OF_GRANULE_FLAG;
writeIntBinary(group_size, ostr); writeVarUInt(group_size, ostr);
} }
size_t deserializeOffsets(IColumn::Offsets & offsets, size_t deserializeOffsets(IColumn::Offsets & offsets,
@ -70,7 +71,7 @@ size_t deserializeOffsets(IColumn::Offsets & offsets,
size_t group_size; size_t group_size;
while (!istr.eof()) while (!istr.eof())
{ {
readIntBinary(group_size, istr); readVarUInt(group_size, istr);
bool end_of_granule = group_size & END_OF_GRANULE_FLAG; bool end_of_granule = group_size & END_OF_GRANULE_FLAG;
group_size &= ~END_OF_GRANULE_FLAG; group_size &= ~END_OF_GRANULE_FLAG;

2
src/Storages/MergeTree/MergeTreeSettings.h
View File

@ -33,7 +33,7 @@ struct Settings;
M(UInt64, min_rows_for_compact_part, 0, "Experimental. Minimal number of rows to create part in compact format instead of saving it in RAM", 0) \ M(UInt64, min_rows_for_compact_part, 0, "Experimental. Minimal number of rows to create part in compact format instead of saving it in RAM", 0) \
M(Bool, in_memory_parts_enable_wal, true, "Whether to write blocks in Native format to write-ahead-log before creation in-memory part", 0) \ M(Bool, in_memory_parts_enable_wal, true, "Whether to write blocks in Native format to write-ahead-log before creation in-memory part", 0) \
M(UInt64, write_ahead_log_max_bytes, 1024 * 1024 * 1024, "Rotate WAL, if it exceeds that amount of bytes", 0) \ M(UInt64, write_ahead_log_max_bytes, 1024 * 1024 * 1024, "Rotate WAL, if it exceeds that amount of bytes", 0) \
M(Float, ratio_for_sparse_serialization, 0, "", 0) \ M(Float, ratio_for_sparse_serialization, 1.1, "", 0) \
\ \
/** Merge settings. */ \ /** Merge settings. */ \
M(UInt64, merge_max_block_size, DEFAULT_MERGE_BLOCK_SIZE, "How many rows in blocks should be formed for merge operations.", 0) \ M(UInt64, merge_max_block_size, DEFAULT_MERGE_BLOCK_SIZE, "How many rows in blocks should be formed for merge operations.", 0) \