From d018f3ad60a3ff6c794e09d8f5c4ac2777802b21 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 21 May 2018 17:15:55 +0300 Subject: [PATCH] Per granula reading for ColumnWithDictionary in MergeTreeRangeReader. --- dbms/src/Storages/MergeTree/MergeTreeReader.cpp | 14 ++++++++++++-- dbms/src/Storages/MergeTree/MergeTreeReader.h | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index 7539a40a1a0..ebc32c902cd 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -42,7 +42,7 @@ MergeTreeReader::MergeTreeReader(const String & path, clockid_t clock_type) : avg_value_size_hints(avg_value_size_hints), path(path), data_part(data_part), columns(columns) , uncompressed_cache(uncompressed_cache), mark_cache(mark_cache), save_marks_in_cache(save_marks_in_cache), storage(storage) - , all_mark_ranges(all_mark_ranges), aio_threshold(aio_threshold), max_read_buffer_size(max_read_buffer_size) + , all_mark_ranges(all_mark_ranges), aio_threshold(aio_threshold), max_read_buffer_size(max_read_buffer_size), index_granularity(storage.index_granularity) { try { @@ -383,7 +383,17 @@ void MergeTreeReader::readData( }; double & avg_value_size_hint = avg_value_size_hints[name]; - type.deserializeBinaryBulkWithMultipleStreams(column, stream_getter, max_rows_to_read, avg_value_size_hint, true, {}); + if (column.withDictionary()) + { + for (size_t read_rows = 0; read_rows < max_rows_to_read; read_rows += index_granularity) + { + size_t rows_to_read = std::min(index_granularity, max_rows_to_read - read_rows); + type.deserializeBinaryBulkWithMultipleStreams(column, stream_getter, rows_to_read, avg_value_size_hint, true, {}); + continue_reading = true; + } + } + else + type.deserializeBinaryBulkWithMultipleStreams(column, stream_getter, max_rows_to_read, avg_value_size_hint, true, {}); IDataType::updateAvgValueSizeHint(column, avg_value_size_hint); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.h b/dbms/src/Storages/MergeTree/MergeTreeReader.h index a4a6b4dc0f7..05ef06b7419 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.h @@ -107,6 +107,7 @@ private: MarkRanges all_mark_ranges; size_t aio_threshold; size_t max_read_buffer_size; + size_t index_granularity; void addStreams(const String & name, const IDataType & type, const MarkRanges & all_mark_ranges, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type);