From cbfd862fa3df628aa30804107924e083cab8c566 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Mar 2018 22:43:00 +0300 Subject: [PATCH] fix reading with evaluated defaults [#CLICKHOUSE-3663] --- .../Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp | 8 ++++---- dbms/src/Storages/MergeTree/MergeTreeReader.cpp | 7 +++---- dbms/src/Storages/MergeTree/MergeTreeReader.h | 6 +++++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp index b5095e79118..0cd903038ed 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp @@ -216,7 +216,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() } progressImpl({ res.rows(), res.bytes() }); - pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder); + pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, res.rows()); /// Compute the expression in PREWHERE. prewhere_actions->execute(res); @@ -396,7 +396,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() /// as a result of reading components of Nested data structures with no data in filesystem. /// We must fill these arrays to filter them correctly. - reader->fillMissingColumns(res, task->ordered_names, task->should_reorder); + reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, post_filter.size(), true); /// Filter the columns related to PREWHERE using pre_filter, /// other columns - using post_filter. @@ -422,7 +422,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() if (task->size_predictor) task->size_predictor->update(res); - reader->fillMissingColumns(res, task->ordered_names, true); + reader->fillMissingColumns(res, task->ordered_names, true, res.rows()); res.checkNumberOfRows(); } @@ -464,7 +464,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() return res; progressImpl({ res.rows(), res.bytes() }); - reader->fillMissingColumns(res, task->ordered_names, task->should_reorder); + reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, res.rows()); } return res; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index e172760b17f..37d41749420 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -413,7 +413,8 @@ static bool arrayHasNoElementsRead(const IColumn & column) } -void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_names, bool always_reorder) +void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_names, bool always_reorder, + size_t rows, bool never_evaluate_defaults) { if (!res) throw Exception("Empty block passed to fillMissingColumns", ErrorCodes::LOGICAL_ERROR); @@ -444,8 +445,6 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name bool should_evaluate_defaults = false; bool should_sort = always_reorder; - size_t rows = res.rows(); - /// insert default values only for columns without default expressions for (const auto & requested_column : columns) { @@ -497,7 +496,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name } /// evaluate defaulted columns if necessary - if (should_evaluate_defaults) + if (!never_evaluate_defaults && should_evaluate_defaults) evaluateMissingDefaults(res, columns, storage.getColumns().defaults, storage.context); /// sort columns to ensure consistent order among all blocks diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.h b/dbms/src/Storages/MergeTree/MergeTreeReader.h index c2cd086d343..b76cf19ff93 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.h @@ -44,7 +44,11 @@ public: /// Add columns from ordered_names that are not present in the block. /// Missing columns are added in the order specified by ordered_names. /// If at least one column was added, reorders all columns in the block according to ordered_names. - void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder); + /// Rows is the number of rows will be used to create default columns. It's res.rows() when all columns in block has the same size. + /// This function may be used in order to create default columns in nested structures. In that case block may has + /// columns with different size, and default columns can't be evaluated, so never_evaluate_defaults = false is used. + void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder, + size_t rows, bool never_evaluate_defaults = false); private: class Stream