From 6b3cd34cff9c78cdd482bcf6909c636d09e8e4c0 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 17 Sep 2021 23:12:44 +0800 Subject: [PATCH] Disable projections when ARRAY JOIN is used. --- src/Storages/MergeTree/MergeTreeData.cpp | 13 +++- src/Storages/ProjectionsDescription.cpp | 73 +++++++------------ src/Storages/ProjectionsDescription.h | 6 -- .../01710_projection_array_join.reference | 1 + .../01710_projection_array_join.sql | 11 +++ 5 files changed, 50 insertions(+), 54 deletions(-) create mode 100644 tests/queries/0_stateless/01710_projection_array_join.reference create mode 100644 tests/queries/0_stateless/01710_projection_array_join.sql diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index ee1387af49b..b7846c331f6 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -4312,9 +4312,16 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( const auto & query_ptr = query_info.query; - // Currently projections don't support final yet. - if (auto * select = query_ptr->as(); select && select->final()) - return false; + if (auto * select = query_ptr->as(); select) + { + // Currently projections don't support final yet. + if (select->final()) + return false; + + // Currently projections don't support ARRAY JOIN yet. + if (select->arrayJoinExpressionList().first) + return false; + } // Currently projections don't support sampling yet. if (settings.parallel_replicas_count > 1) diff --git a/src/Storages/ProjectionsDescription.cpp b/src/Storages/ProjectionsDescription.cpp index d1b72d71b83..8d64a590a3d 100644 --- a/src/Storages/ProjectionsDescription.cpp +++ b/src/Storages/ProjectionsDescription.cpp @@ -54,8 +54,6 @@ ProjectionDescription ProjectionDescription::clone() const other.name = name; other.type = type; other.required_columns = required_columns; - other.column_names = column_names; - other.data_types = data_types; other.sample_block = sample_block; other.sample_block_for_keys = sample_block_for_keys; other.metadata = metadata; @@ -111,31 +109,16 @@ ProjectionDescription::getProjectionFromAST(const ASTPtr & definition_ast, const result.required_columns = select.getRequiredColumns(); result.sample_block = select.getSampleBlock(); - const auto & analysis_result = select.getAnalysisResult(); - if (analysis_result.need_aggregate) - { - for (const auto & key : select.getQueryAnalyzer()->aggregationKeys()) - result.sample_block_for_keys.insert({nullptr, key.type, key.name}); - } - - for (size_t i = 0; i < result.sample_block.columns(); ++i) - { - const auto & column_with_type_name = result.sample_block.getByPosition(i); - - if (column_with_type_name.column && isColumnConst(*column_with_type_name.column)) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Projections cannot contain constant columns: {}", column_with_type_name.name); - - result.column_names.emplace_back(column_with_type_name.name); - result.data_types.emplace_back(column_with_type_name.type); - } - StorageInMemoryMetadata metadata; - metadata.setColumns(ColumnsDescription(result.sample_block.getNamesAndTypesList())); - metadata.partition_key = KeyDescription::getSortingKeyFromAST({}, metadata.columns, query_context, {}); + metadata.partition_key = KeyDescription::getSortingKeyFromAST({}, {}, query_context, {}); const auto & query_select = result.query_ast->as(); if (select.hasAggregation()) { + if (query.orderBy()) + throw Exception( + "When aggregation is used in projection, ORDER BY cannot be specified", ErrorCodes::ILLEGAL_PROJECTION); + result.type = ProjectionDescription::Type::Aggregate; if (const auto & group_expression_list = query_select.groupBy()) { @@ -156,25 +139,36 @@ ProjectionDescription::getProjectionFromAST(const ASTPtr & definition_ast, const function_node->children.push_back(function_node->arguments); order_expression = function_node; } - metadata.sorting_key = KeyDescription::getSortingKeyFromAST(order_expression, metadata.columns, query_context, {}); - metadata.primary_key = KeyDescription::getKeyFromAST(order_expression, metadata.columns, query_context); + auto columns_with_state = ColumnsDescription(result.sample_block.getNamesAndTypesList()); + metadata.sorting_key = KeyDescription::getSortingKeyFromAST(order_expression, columns_with_state, query_context, {}); + metadata.primary_key = KeyDescription::getKeyFromAST(order_expression, columns_with_state, query_context); } else { - metadata.sorting_key = KeyDescription::getSortingKeyFromAST({}, metadata.columns, query_context, {}); - metadata.primary_key = KeyDescription::getKeyFromAST({}, metadata.columns, query_context); + metadata.sorting_key = KeyDescription::getSortingKeyFromAST({}, {}, query_context, {}); + metadata.primary_key = KeyDescription::getKeyFromAST({}, {}, query_context); } - if (query.orderBy()) - throw Exception( - "When aggregation is used in projection, ORDER BY cannot be specified", ErrorCodes::ILLEGAL_PROJECTION); + for (const auto & key : select.getQueryAnalyzer()->aggregationKeys()) + result.sample_block_for_keys.insert({nullptr, key.type, key.name}); } else { result.type = ProjectionDescription::Type::Normal; - metadata.sorting_key = KeyDescription::getSortingKeyFromAST(query.orderBy(), metadata.columns, query_context, {}); - metadata.primary_key = KeyDescription::getKeyFromAST(query.orderBy(), metadata.columns, query_context); + metadata.sorting_key = KeyDescription::getSortingKeyFromAST(query.orderBy(), columns, query_context, {}); + metadata.primary_key = KeyDescription::getKeyFromAST(query.orderBy(), columns, query_context); } metadata.primary_key.definition_ast = nullptr; + + auto block = result.sample_block; + for (const auto & [name, type] : metadata.sorting_key.expression->getRequiredColumnsWithTypes()) + block.insertUnique({nullptr, type, name}); + for (const auto & column_with_type_name : block) + { + if (column_with_type_name.column && isColumnConst(*column_with_type_name.column)) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Projections cannot contain constant columns: {}", column_with_type_name.name); + } + + metadata.setColumns(ColumnsDescription(block.getNamesAndTypesList())); result.metadata = std::make_shared(metadata); return result; } @@ -203,23 +197,12 @@ ProjectionDescription::getMinMaxCountProjection(const ColumnsDescription & colum result.query_ast, query_context, storage, {}, SelectQueryOptions{QueryProcessingStage::WithMergeableState}.modify().ignoreAlias()); result.required_columns = select.getRequiredColumns(); result.sample_block = select.getSampleBlock(); - - for (size_t i = 0; i < result.sample_block.columns(); ++i) - { - const auto & column_with_type_name = result.sample_block.getByPosition(i); - - if (column_with_type_name.column && isColumnConst(*column_with_type_name.column)) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Projections cannot contain constant columns: {}", column_with_type_name.name); - - result.column_names.emplace_back(column_with_type_name.name); - result.data_types.emplace_back(column_with_type_name.type); - } result.type = ProjectionDescription::Type::Aggregate; StorageInMemoryMetadata metadata; metadata.setColumns(ColumnsDescription(result.sample_block.getNamesAndTypesList())); - metadata.partition_key = KeyDescription::getSortingKeyFromAST({}, metadata.columns, query_context, {}); - metadata.sorting_key = KeyDescription::getSortingKeyFromAST({}, metadata.columns, query_context, {}); - metadata.primary_key = KeyDescription::getKeyFromAST({}, metadata.columns, query_context); + metadata.partition_key = KeyDescription::getSortingKeyFromAST({}, {}, query_context, {}); + metadata.sorting_key = KeyDescription::getSortingKeyFromAST({}, {}, query_context, {}); + metadata.primary_key = KeyDescription::getKeyFromAST({}, {}, query_context); metadata.primary_key.definition_ast = nullptr; result.metadata = std::make_shared(metadata); result.is_minmax_count_projection = true; diff --git a/src/Storages/ProjectionsDescription.h b/src/Storages/ProjectionsDescription.h index 5c4db25feaa..b9c11cb0771 100644 --- a/src/Storages/ProjectionsDescription.h +++ b/src/Storages/ProjectionsDescription.h @@ -47,12 +47,6 @@ struct ProjectionDescription Names getRequiredColumns() const { return required_columns; } - /// Names of projection columns (not to be confused with required columns) - Names column_names; - - /// Data types of projection columns - DataTypes data_types; - /// Sample block with projection columns. (NOTE: columns in block are empty, but not nullptr) Block sample_block; diff --git a/tests/queries/0_stateless/01710_projection_array_join.reference b/tests/queries/0_stateless/01710_projection_array_join.reference new file mode 100644 index 00000000000..0cfbf08886f --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_array_join.reference @@ -0,0 +1 @@ +2 diff --git a/tests/queries/0_stateless/01710_projection_array_join.sql b/tests/queries/0_stateless/01710_projection_array_join.sql new file mode 100644 index 00000000000..cd18d9282b9 --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_array_join.sql @@ -0,0 +1,11 @@ +set allow_experimental_projection_optimization = 1; + +drop table if exists x; + +create table x (pk int, arr Array(int), projection p (select arr order by pk)) engine MergeTree order by tuple(); + +insert into x values (1, [2]); + +select a from x array join arr as a; + +drop table x;