From 2c56b0b2b991e4a564fc9928ba27ea97cb0ee2d7 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 13 Jan 2023 17:53:53 +0100 Subject: [PATCH 1/4] Planner small fixes --- src/Planner/Planner.cpp | 1 + src/Planner/PlannerAggregation.cpp | 87 ++++++++++++++++++- src/Planner/PlannerAggregation.h | 5 +- src/Planner/PlannerExpressionAnalysis.cpp | 14 +-- src/Planner/PlannerExpressionAnalysis.h | 2 +- src/Planner/PlannerJoinTree.cpp | 18 ++-- src/Storages/StorageDistributed.cpp | 4 +- ...analyzer_aggregation_with_rollup.reference | 10 +++ ...02532_analyzer_aggregation_with_rollup.sql | 20 +++++ 9 files changed, 134 insertions(+), 27 deletions(-) create mode 100644 tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.reference create mode 100644 tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.sql diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 2a9d06bc17b..94f54981b00 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -1112,6 +1112,7 @@ void Planner::buildPlanForQueryNode() checkStoragesSupportTransactions(planner_context); collectTableExpressionData(query_tree, *planner_context); collectSets(query_tree, *planner_context); + resolveGroupingFunctions(query_tree, *planner_context); QueryProcessingStage::Enum from_stage = QueryProcessingStage::FetchColumns; diff --git a/src/Planner/PlannerAggregation.cpp b/src/Planner/PlannerAggregation.cpp index 05e7b5418e3..951db7cd189 100644 --- a/src/Planner/PlannerAggregation.cpp +++ b/src/Planner/PlannerAggregation.cpp @@ -91,6 +91,7 @@ public: QueryTreeNodeWeakPtr column_source; auto grouping_set_argument_column = std::make_shared(NameAndTypePair{"__grouping_set", std::make_shared()}, column_source); + auto previous_arguments = function_node->getArguments().getNodes(); function_node->getArguments().getNodes().clear(); bool force_grouping_standard_compatibility = planner_context.getQueryContext()->getSettingsRef().force_grouping_standard_compatibility; @@ -129,6 +130,9 @@ public: break; } } + + auto & function_node_arguments = function_node->getArguments().getNodes(); + function_node_arguments.insert(function_node_arguments.end(), previous_arguments.begin(), previous_arguments.end()); } static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) @@ -163,8 +167,6 @@ void resolveGroupingFunctions(QueryTreeNodePtr & node, visitor.visit(query_node_typed.getProjectionNode()); } -} - void resolveGroupingFunctions(QueryTreeNodePtr & query_node, const Names & aggregation_keys, const GroupingSetsParamsList & grouping_sets_parameters_list, @@ -183,6 +185,87 @@ void resolveGroupingFunctions(QueryTreeNodePtr & query_node, resolveGroupingFunctions(query_node, group_by_kind, aggregation_keys, grouping_sets_parameters_list, planner_context); } +} + +void resolveGroupingFunctions(QueryTreeNodePtr & query_node, const PlannerContext & planner_context) +{ + auto & query_node_typed = query_node->as(); + + std::unordered_set used_aggregation_keys; + Names aggregation_keys; + GroupingSetsParamsList grouping_sets_parameters_list; + + QueryTreeNodeToName group_by_node_to_name; + + /// Add expressions from GROUP BY + + if (query_node_typed.hasGroupBy()) + { + if (query_node_typed.isGroupByWithGroupingSets()) + { + for (const auto & grouping_set_keys_list_node : query_node_typed.getGroupBy().getNodes()) + { + auto & grouping_set_keys_list_node_typed = grouping_set_keys_list_node->as(); + grouping_sets_parameters_list.emplace_back(); + auto & grouping_sets_parameters = grouping_sets_parameters_list.back(); + + for (auto & grouping_set_key_node : grouping_set_keys_list_node_typed.getNodes()) + { + auto grouping_set_key_name = calculateActionNodeName(grouping_set_key_node, planner_context, group_by_node_to_name); + grouping_sets_parameters.used_keys.push_back(grouping_set_key_name); + + if (used_aggregation_keys.contains(grouping_set_key_name)) + continue; + + aggregation_keys.push_back(grouping_set_key_name); + used_aggregation_keys.insert(aggregation_keys.back()); + } + } + + for (auto & grouping_sets_parameter : grouping_sets_parameters_list) + { + NameSet grouping_sets_used_keys; + Names grouping_sets_keys; + + for (auto & key : grouping_sets_parameter.used_keys) + { + auto [_, inserted] = grouping_sets_used_keys.insert(key); + if (inserted) + grouping_sets_keys.push_back(key); + } + + for (auto & key : aggregation_keys) + { + if (grouping_sets_used_keys.contains(key)) + continue; + + grouping_sets_parameter.missing_keys.push_back(key); + } + + grouping_sets_parameter.used_keys = std::move(grouping_sets_keys); + } + + /// It is expected by execution layer that if there are only 1 grouping sets it will be removed + if (grouping_sets_parameters_list.size() == 1) + grouping_sets_parameters_list.clear(); + } + else + { + for (auto & group_by_key_node : query_node_typed.getGroupBy().getNodes()) + { + auto group_by_key_name = calculateActionNodeName(group_by_key_node, planner_context, group_by_node_to_name); + if (used_aggregation_keys.contains(group_by_key_name)) + continue; + + aggregation_keys.push_back(group_by_key_name); + used_aggregation_keys.insert(aggregation_keys.back()); + } + } + } + + resolveGroupingFunctions(query_node, aggregation_keys, grouping_sets_parameters_list, planner_context); +} + AggregateDescriptions extractAggregateDescriptions(const QueryTreeNodes & aggregate_function_nodes, const PlannerContext & planner_context) { QueryTreeNodeToName node_to_name; diff --git a/src/Planner/PlannerAggregation.h b/src/Planner/PlannerAggregation.h index 6dfd7faca22..eb91cb11264 100644 --- a/src/Planner/PlannerAggregation.h +++ b/src/Planner/PlannerAggregation.h @@ -13,11 +13,10 @@ namespace DB /** Resolve GROUPING functions in query node. * GROUPING function is replaced with specialized GROUPING function based on GROUP BY modifiers. - * For ROLLUP, CUBE, GROUPING SETS specialized GROUPING function take special __grouping_set column as argument. + * For ROLLUP, CUBE, GROUPING SETS specialized GROUPING function take special __grouping_set column as argument + * and previous GROUPING function arguments. */ void resolveGroupingFunctions(QueryTreeNodePtr & query_node, - const Names & aggregation_keys, - const GroupingSetsParamsList & grouping_sets_parameters_list, const PlannerContext & planner_context); /// Extract aggregate descriptions from aggregate function nodes diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index 91a04b090fc..b14915c9527 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -52,7 +52,7 @@ FilterAnalysisResult analyzeFilter(const QueryTreeNodePtr & filter_expression_no /** Construct aggregation analysis result if query tree has GROUP BY or aggregates. * Actions before aggregation are added into actions chain, if result is not null optional. */ -std::optional analyzeAggregation(QueryTreeNodePtr & query_tree, +std::optional analyzeAggregation(const QueryTreeNodePtr & query_tree, const ColumnsWithTypeAndName & join_tree_input_columns, const PlannerContextPtr & planner_context, ActionsChain & actions_chain) @@ -193,8 +193,6 @@ std::optional analyzeAggregation(QueryTreeNodePtr & q if (query_node.isGroupByWithRollup() || query_node.isGroupByWithCube() || (query_node.isGroupByWithGroupingSets() && !disable_grouping_sets)) aggregates_columns.emplace_back(nullptr, std::make_shared(), "__grouping_set"); - resolveGroupingFunctions(query_tree, aggregation_keys, grouping_sets_parameters_list, *planner_context); - /// Only aggregation keys and aggregates are available for next steps after GROUP BY step auto aggregate_step = std::make_unique(before_aggregation_actions, ActionsChainStep::AvailableOutputColumnsStrategy::OUTPUT_NODES, aggregates_columns); actions_chain.addStep(std::move(aggregate_step)); @@ -212,7 +210,7 @@ std::optional analyzeAggregation(QueryTreeNodePtr & q /** Construct window analysis result if query tree has window functions. * Actions before window functions are added into actions chain, if result is not null optional. */ -std::optional analyzeWindow(QueryTreeNodePtr & query_tree, +std::optional analyzeWindow(const QueryTreeNodePtr & query_tree, const ColumnsWithTypeAndName & join_tree_input_columns, const PlannerContextPtr & planner_context, ActionsChain & actions_chain) @@ -417,7 +415,7 @@ LimitByAnalysisResult analyzeLimitBy(const QueryNode & query_node, } -PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(QueryTreeNodePtr query_tree, +PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNodePtr & query_tree, const ColumnsWithTypeAndName & join_tree_input_columns, const PlannerContextPtr & planner_context) { @@ -463,14 +461,8 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(QueryTreeNodePtr project_names_actions->project(projection_analysis_result.projection_column_names_with_display_aliases); actions_chain.addStep(std::make_unique(project_names_actions)); - // std::cout << "Chain dump before finalize" << std::endl; - // std::cout << actions_chain.dump() << std::endl; - actions_chain.finalize(); - // std::cout << "Chain dump after finalize" << std::endl; - // std::cout << actions_chain.dump() << std::endl; - projection_analysis_result.project_names_actions = std::move(project_names_actions); PlannerExpressionsAnalysisResult expressions_analysis_result(std::move(projection_analysis_result)); diff --git a/src/Planner/PlannerExpressionAnalysis.h b/src/Planner/PlannerExpressionAnalysis.h index aefb3c369d0..d697641964c 100644 --- a/src/Planner/PlannerExpressionAnalysis.h +++ b/src/Planner/PlannerExpressionAnalysis.h @@ -168,7 +168,7 @@ private: }; /// Build expression analysis result for query tree, join tree input columns and planner context -PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(QueryTreeNodePtr query_tree, +PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNodePtr & query_tree, const ColumnsWithTypeAndName & join_tree_input_columns, const PlannerContextPtr & planner_context); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 6a48f322ba5..2484bd093ec 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -248,7 +248,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(const QueryTreeNodePtr & tabl if (max_streams == 0) max_streams = 1; - /// If necessary, we request more sources than the number of threads - to distribute the work evenly over the threads. + /// If necessary, we request more sources than the number of threads - to distribute the work evenly over the threads if (max_streams > 1 && !is_remote) max_streams = static_cast(max_streams * settings.max_streams_to_max_threads_ratio); @@ -841,7 +841,7 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, std::vector table_expressions_outer_scope_columns(table_expressions_stack_size); ColumnIdentifierSet current_outer_scope_columns = outer_scope_columns; - for (Int64 i = table_expressions_stack_size - 1; i >= 0; --i) + for (Int64 i = static_cast(table_expressions_stack_size) - 1; i >= 0; --i) { table_expressions_outer_scope_columns[i] = current_outer_scope_columns; @@ -859,7 +859,8 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, { if (query_plans_stack.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, - "Expected at least 1 query plan on stack before ARRAY JOIN processing"); + "Expected at least 1 query plan on stack before ARRAY JOIN processing. Actual {}", + query_plans_stack.size()); auto query_plan = std::move(query_plans_stack.back()); query_plans_stack.back() = buildQueryPlanForArrayJoinNode(table_expression, @@ -868,11 +869,10 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, } else if (auto * join_node = table_expression->as()) { - size_t table_expressions_column_nodes_with_names_stack_size = query_plans_stack.size(); - if (table_expressions_column_nodes_with_names_stack_size < 2) + if (query_plans_stack.size() < 2) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected at least 2 query plans on stack before JOIN processing. Actual {}", - table_expressions_column_nodes_with_names_stack_size); + query_plans_stack.size()); auto right_query_plan = std::move(query_plans_stack.back()); query_plans_stack.pop_back(); @@ -901,8 +901,10 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, } } - if (query_plans_stack.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected at least 1 query plan for JOIN TREE"); + if (query_plans_stack.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Expected 1 query plan for JOIN TREE. Actual {}", + query_plans_stack.size()); return std::move(query_plans_stack.back()); } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 5111bd38e58..c0818302fb2 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -578,9 +578,9 @@ std::optional StorageDistributed::getOptimizedQueryP bool has_aggregates = query_info.has_aggregates; if (query_info.syntax_analyzer_result) - has_aggregates = query_info.syntax_analyzer_result->aggregates.empty(); + has_aggregates = !query_info.syntax_analyzer_result->aggregates.empty(); - if (!has_aggregates || group_by) + if (has_aggregates || group_by) { if (!optimize_sharding_key_aggregation || !group_by || !expr_contains_sharding_key(group_by->children)) return {}; diff --git a/tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.reference b/tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.reference new file mode 100644 index 00000000000..2ce5e955866 --- /dev/null +++ b/tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.reference @@ -0,0 +1,10 @@ +45 0 0 0 1 +45 0 1 0 1 +45 0 2 0 1 +45 0 3 0 1 +45 0 4 0 1 +45 0 5 0 1 +45 0 6 0 1 +45 0 7 0 1 +45 0 8 0 1 +45 0 9 0 1 diff --git a/tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.sql b/tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.sql new file mode 100644 index 00000000000..09097eb029f --- /dev/null +++ b/tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.sql @@ -0,0 +1,20 @@ +SET allow_experimental_analyzer = 1; + +SELECT + sum(a.number) AS total, + c.number AS cn, + b.number AS bn, + grouping(c.number) + grouping(b.number) AS l, + rank() OVER (PARTITION BY grouping(c.number) + grouping(b.number), multiIf(grouping(c.number) = 0, b.number, NULL) ORDER BY sum(a.number) DESC) AS r +FROM numbers(10) AS a, numbers(10) AS b, numbers(10) AS c +GROUP BY + cn, + bn + WITH ROLLUP +ORDER BY + total ASC, + cn ASC, + bn ASC, + l ASC, + r ASC +LIMIT 10; From 506f91b841460834b95319dbf4a7baab4321d533 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 17 Jan 2023 12:35:14 +0100 Subject: [PATCH 2/4] Fixed tests --- src/Planner/PlannerAggregation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Planner/PlannerAggregation.cpp b/src/Planner/PlannerAggregation.cpp index 951db7cd189..7f52f86dc38 100644 --- a/src/Planner/PlannerAggregation.cpp +++ b/src/Planner/PlannerAggregation.cpp @@ -191,7 +191,7 @@ void resolveGroupingFunctions(QueryTreeNodePtr & query_node, const PlannerContex { auto & query_node_typed = query_node->as(); - std::unordered_set used_aggregation_keys; + std::unordered_set used_aggregation_keys; Names aggregation_keys; GroupingSetsParamsList grouping_sets_parameters_list; From 3363f7c7189cc6e3b46b83a6caf4115af8fca859 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 19 Jan 2023 19:05:31 +0100 Subject: [PATCH 3/4] Added GroupingFunctionsResolvePass --- .../Passes/GroupingFunctionsResolvePass.cpp | 253 +++++++++++++++++ .../Passes/GroupingFunctionsResolvePass.h | 31 +++ src/Analyzer/QueryTreePassManager.cpp | 5 +- src/Planner/CollectTableExpressionData.cpp | 3 + src/Planner/Planner.cpp | 1 - src/Planner/PlannerAggregation.cpp | 258 ------------------ src/Planner/PlannerAggregation.h | 8 - src/Planner/PlannerExpressionAnalysis.cpp | 10 +- ...02534_analyzer_grouping_function.reference | 141 ++++++++++ .../02534_analyzer_grouping_function.sql | 41 +++ 10 files changed, 474 insertions(+), 277 deletions(-) create mode 100644 src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp create mode 100644 src/Analyzer/Passes/GroupingFunctionsResolvePass.h create mode 100644 tests/queries/0_stateless/02534_analyzer_grouping_function.reference create mode 100644 tests/queries/0_stateless/02534_analyzer_grouping_function.sql diff --git a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp new file mode 100644 index 00000000000..55152fccee9 --- /dev/null +++ b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp @@ -0,0 +1,253 @@ +#include + +#include + +#include + +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; +} + +namespace +{ + +enum class GroupByKind +{ + ORDINARY, + ROLLUP, + CUBE, + GROUPING_SETS +}; + +class GroupingFunctionResolveVisitor : public InDepthQueryTreeVisitor +{ +public: + GroupingFunctionResolveVisitor(GroupByKind group_by_kind_, + QueryTreeNodePtrWithHashMap aggregation_key_to_index_, + ColumnNumbersList grouping_sets_keys_indices_, + ContextPtr context_) + : group_by_kind(group_by_kind_) + , aggregation_key_to_index(std::move(aggregation_key_to_index_)) + , grouping_sets_keys_indexes(std::move(grouping_sets_keys_indices_)) + , context(std::move(context_)) + { + } + + void visitImpl(const QueryTreeNodePtr & node) + { + auto * function_node = node->as(); + if (!function_node || function_node->getFunctionName() != "grouping") + return; + + auto & function_arguments = function_node->getArguments().getNodes(); + + ColumnNumbers arguments_indexes; + arguments_indexes.reserve(function_arguments.size()); + + for (const auto & argument : function_arguments) + { + auto it = aggregation_key_to_index.find(argument); + if (it == aggregation_key_to_index.end()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Argument {} of GROUPING function is not a part of GROUP BY clause", + argument->formatASTForErrorMessage()); + + arguments_indexes.push_back(it->second); + } + + FunctionOverloadResolverPtr grouping_function_resolver; + bool add_grouping_set_column = false; + + bool force_grouping_standard_compatibility = context->getSettingsRef().force_grouping_standard_compatibility; + size_t aggregation_keys_size = aggregation_key_to_index.size(); + + switch (group_by_kind) + { + case GroupByKind::ORDINARY: + { + auto grouping_ordinary_function = std::make_shared(arguments_indexes, + force_grouping_standard_compatibility); + grouping_function_resolver = std::make_shared(std::move(grouping_ordinary_function)); + break; + } + case GroupByKind::ROLLUP: + { + auto grouping_rollup_function = std::make_shared(arguments_indexes, + aggregation_keys_size, + force_grouping_standard_compatibility); + grouping_function_resolver = std::make_shared(std::move(grouping_rollup_function)); + add_grouping_set_column = true; + break; + } + case GroupByKind::CUBE: + { + auto grouping_cube_function = std::make_shared(arguments_indexes, + aggregation_keys_size, + force_grouping_standard_compatibility); + grouping_function_resolver = std::make_shared(std::move(grouping_cube_function)); + add_grouping_set_column = true; + break; + } + case GroupByKind::GROUPING_SETS: + { + auto grouping_grouping_sets_function = std::make_shared(arguments_indexes, + grouping_sets_keys_indexes, + force_grouping_standard_compatibility); + grouping_function_resolver = std::make_shared(std::move(grouping_grouping_sets_function)); + add_grouping_set_column = true; + break; + } + } + + if (add_grouping_set_column) + { + QueryTreeNodeWeakPtr column_source; + auto grouping_set_column = NameAndTypePair{"__grouping_set", std::make_shared()}; + auto grouping_set_argument_column = std::make_shared(std::move(grouping_set_column), std::move(column_source)); + function_arguments.insert(function_arguments.begin(), std::move(grouping_set_argument_column)); + } + + function_node->resolveAsFunction(grouping_function_resolver->build(function_node->getArgumentColumns())); + } + + static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) + { + return !(child_node->getNodeType() == QueryTreeNodeType::QUERY || child_node->getNodeType() == QueryTreeNodeType::UNION); + } + +private: + GroupByKind group_by_kind; + QueryTreeNodePtrWithHashMap aggregation_key_to_index; + ColumnNumbersList grouping_sets_keys_indexes; + ContextPtr context; +}; + +void resolveGroupingFunctions(QueryTreeNodePtr & query_node, ContextPtr context) +{ + auto & query_node_typed = query_node->as(); + + size_t aggregation_node_index = 0; + QueryTreeNodePtrWithHashMap aggregation_key_to_index; + + std::vector grouping_sets_used_aggregation_keys_list; + + if (query_node_typed.hasGroupBy()) + { + /// It is expected by execution layer that if there are only 1 grouping set it will be removed + if (query_node_typed.isGroupByWithGroupingSets() && query_node_typed.getGroupBy().getNodes().size() == 1) + { + auto & grouping_set_list_node = query_node_typed.getGroupBy().getNodes().front()->as(); + query_node_typed.getGroupBy().getNodes() = std::move(grouping_set_list_node.getNodes()); + query_node_typed.setIsGroupByWithGroupingSets(false); + } + + if (query_node_typed.isGroupByWithGroupingSets()) + { + for (const auto & grouping_set_keys_list_node : query_node_typed.getGroupBy().getNodes()) + { + auto & grouping_set_keys_list_node_typed = grouping_set_keys_list_node->as(); + + grouping_sets_used_aggregation_keys_list.emplace_back(); + auto & grouping_sets_used_aggregation_keys = grouping_sets_used_aggregation_keys_list.back(); + + for (auto & grouping_set_key_node : grouping_set_keys_list_node_typed.getNodes()) + { + if (aggregation_key_to_index.contains(grouping_set_key_node)) + continue; + + grouping_sets_used_aggregation_keys.push_back(grouping_set_key_node); + aggregation_key_to_index.emplace(grouping_set_key_node, aggregation_node_index); + ++aggregation_node_index; + } + } + } + else + { + for (auto & group_by_key_node : query_node_typed.getGroupBy().getNodes()) + { + if (aggregation_key_to_index.contains(group_by_key_node)) + continue; + + aggregation_key_to_index.emplace(group_by_key_node, aggregation_node_index); + ++aggregation_node_index; + } + } + } + + /// Indexes of aggregation keys used in each grouping set (only for GROUP BY GROUPING SETS) + ColumnNumbersList grouping_sets_keys_indexes; + + for (const auto & grouping_set_used_aggregation_keys : grouping_sets_used_aggregation_keys_list) + { + grouping_sets_keys_indexes.emplace_back(); + auto & grouping_set_keys_indexes = grouping_sets_keys_indexes.back(); + + for (const auto & used_aggregation_key : grouping_set_used_aggregation_keys) + { + auto aggregation_node_index_it = aggregation_key_to_index.find(used_aggregation_key); + if (aggregation_node_index_it == aggregation_key_to_index.end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Aggregation key {} in GROUPING SETS is not found in GROUP BY keys", + used_aggregation_key->formatASTForErrorMessage()); + + grouping_set_keys_indexes.push_back(aggregation_node_index_it->second); + } + } + + GroupByKind group_by_kind = GroupByKind::ORDINARY; + if (query_node_typed.isGroupByWithRollup()) + group_by_kind = GroupByKind::ROLLUP; + else if (query_node_typed.isGroupByWithCube()) + group_by_kind = GroupByKind::CUBE; + else if (query_node_typed.isGroupByWithGroupingSets()) + group_by_kind = GroupByKind::GROUPING_SETS; + + GroupingFunctionResolveVisitor visitor(group_by_kind, + std::move(aggregation_key_to_index), + std::move(grouping_sets_keys_indexes), + std::move(context)); + visitor.visit(query_node); +} + +class GroupingFunctionsResolveVisitor : public InDepthQueryTreeVisitor +{ +public: + explicit GroupingFunctionsResolveVisitor(ContextPtr context_) + : context(std::move(context_)) + {} + + void visitImpl(QueryTreeNodePtr & node) + { + if (node->getNodeType() != QueryTreeNodeType::QUERY) + return; + + resolveGroupingFunctions(node, context); + } + +private: + ContextPtr context; +}; + +} + +void GroupingFunctionsResolvePass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +{ + GroupingFunctionsResolveVisitor visitor(std::move(context)); + visitor.visit(query_tree_node); +} + +} + diff --git a/src/Analyzer/Passes/GroupingFunctionsResolvePass.h b/src/Analyzer/Passes/GroupingFunctionsResolvePass.h new file mode 100644 index 00000000000..070c8dd9389 --- /dev/null +++ b/src/Analyzer/Passes/GroupingFunctionsResolvePass.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +namespace DB +{ + +/** Resolve GROUPING functions in query node. + * GROUPING function is replaced with specialized GROUPING function based on GROUP BY modifiers. + * For ROLLUP, CUBE, GROUPING SETS specialized GROUPING function take special __grouping_set column as argument + * and previous GROUPING function arguments. + * + * Example: SELECT grouping(id) FROM test_table GROUP BY id; + * Result: SELECT groupingOrdinary(id) FROM test_table GROUP BY id; + * + * Example: SELECT grouping(id), grouping(value) FROM test_table GROUP BY GROUPING SETS ((id), (value)); + * Result: SELECT groupingForGroupingSets(__grouping_set, id), groupingForGroupingSets(__grouping_set, value) + * FROM test_table GROUP BY GROUPING SETS ((id), (value)); + */ +class GroupingFunctionsResolvePass final : public IQueryTreePass +{ +public: + String getName() override { return "GroupingFunctionsResolvePass"; } + + String getDescription() override { return "Resolve GROUPING functions based on GROUP BY modifiers"; } + + void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + +}; + +} diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 66b060de6c4..33fc5f1cb6b 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -32,6 +32,7 @@ #include #include #include +#include namespace DB { @@ -67,7 +68,7 @@ public: private: void visitColumn(ColumnNode * column) const { - if (column->getColumnSourceOrNull() == nullptr) + if (column->getColumnSourceOrNull() == nullptr && column->getColumnName() != "__grouping_set") throw Exception(ErrorCodes::LOGICAL_ERROR, "Column {} {} query tree node does not have valid source node after running {} pass", column->getColumnName(), column->getColumnType(), pass_name); @@ -258,6 +259,8 @@ void addQueryTreePasses(QueryTreePassManager & manager) manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); + + manager.addPass(std::make_unique()); } } diff --git a/src/Planner/CollectTableExpressionData.cpp b/src/Planner/CollectTableExpressionData.cpp index 897959fa456..0b820e849f3 100644 --- a/src/Planner/CollectTableExpressionData.cpp +++ b/src/Planner/CollectTableExpressionData.cpp @@ -35,6 +35,9 @@ public: if (!column_node) return; + if (column_node->getColumnName() == "__grouping_set") + return; + auto column_source_node = column_node->getColumnSource(); auto column_source_node_type = column_source_node->getNodeType(); diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 94f54981b00..2a9d06bc17b 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -1112,7 +1112,6 @@ void Planner::buildPlanForQueryNode() checkStoragesSupportTransactions(planner_context); collectTableExpressionData(query_tree, *planner_context); collectSets(query_tree, *planner_context); - resolveGroupingFunctions(query_tree, *planner_context); QueryProcessingStage::Enum from_stage = QueryProcessingStage::FetchColumns; diff --git a/src/Planner/PlannerAggregation.cpp b/src/Planner/PlannerAggregation.cpp index 7f52f86dc38..67ce474b37c 100644 --- a/src/Planner/PlannerAggregation.cpp +++ b/src/Planner/PlannerAggregation.cpp @@ -1,271 +1,13 @@ #include -#include - -#include #include -#include #include -#include -#include - -#include - -#include #include namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; - extern const int BAD_ARGUMENTS; -} - -namespace -{ - -enum class GroupByKind -{ - ORDINARY, - ROLLUP, - CUBE, - GROUPING_SETS -}; - -class GroupingFunctionResolveVisitor : public InDepthQueryTreeVisitor -{ -public: - GroupingFunctionResolveVisitor(GroupByKind group_by_kind_, - const Names & aggregation_keys_, - const GroupingSetsParamsList & grouping_sets_parameters_list_, - const PlannerContext & planner_context_) - : group_by_kind(group_by_kind_) - , planner_context(planner_context_) - { - size_t aggregation_keys_size = aggregation_keys_.size(); - for (size_t i = 0; i < aggregation_keys_size; ++i) - aggegation_key_to_index.emplace(aggregation_keys_[i], i); - - for (const auto & grouping_sets_parameter : grouping_sets_parameters_list_) - { - grouping_sets_keys_indices.emplace_back(); - auto & grouping_set_keys_indices = grouping_sets_keys_indices.back(); - - for (const auto & used_key : grouping_sets_parameter.used_keys) - { - auto aggregation_key_index_it = aggegation_key_to_index.find(used_key); - if (aggregation_key_index_it == aggegation_key_to_index.end()) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Aggregation key {} in GROUPING SETS is not found in GROUP BY keys"); - - grouping_set_keys_indices.push_back(aggregation_key_index_it->second); - } - } - } - - void visitImpl(const QueryTreeNodePtr & node) - { - auto * function_node = node->as(); - if (!function_node || function_node->getFunctionName() != "grouping") - return; - - size_t aggregation_keys_size = aggegation_key_to_index.size(); - - ColumnNumbers arguments_indexes; - - for (const auto & argument : function_node->getArguments().getNodes()) - { - String action_node_name = calculateActionNodeName(argument, planner_context); - - auto it = aggegation_key_to_index.find(action_node_name); - if (it == aggegation_key_to_index.end()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Argument of GROUPING function {} is not a part of GROUP BY clause", - argument->formatASTForErrorMessage()); - - arguments_indexes.push_back(it->second); - } - - QueryTreeNodeWeakPtr column_source; - auto grouping_set_argument_column = std::make_shared(NameAndTypePair{"__grouping_set", std::make_shared()}, column_source); - auto previous_arguments = function_node->getArguments().getNodes(); - function_node->getArguments().getNodes().clear(); - - bool force_grouping_standard_compatibility = planner_context.getQueryContext()->getSettingsRef().force_grouping_standard_compatibility; - - switch (group_by_kind) - { - case GroupByKind::ORDINARY: - { - auto grouping_ordinary_function = std::make_shared(arguments_indexes, force_grouping_standard_compatibility); - auto grouping_ordinary_function_adaptor = std::make_shared(std::move(grouping_ordinary_function)); - function_node->resolveAsFunction(grouping_ordinary_function_adaptor->build({})); - break; - } - case GroupByKind::ROLLUP: - { - auto grouping_rollup_function = std::make_shared(arguments_indexes, aggregation_keys_size, force_grouping_standard_compatibility); - auto grouping_rollup_function_adaptor = std::make_shared(std::move(grouping_rollup_function)); - function_node->resolveAsFunction(grouping_rollup_function_adaptor->build({})); - function_node->getArguments().getNodes().push_back(std::move(grouping_set_argument_column)); - break; - } - case GroupByKind::CUBE: - { - auto grouping_cube_function = std::make_shared(arguments_indexes, aggregation_keys_size, force_grouping_standard_compatibility); - auto grouping_cube_function_adaptor = std::make_shared(std::move(grouping_cube_function)); - function_node->resolveAsFunction(grouping_cube_function_adaptor->build({})); - function_node->getArguments().getNodes().push_back(std::move(grouping_set_argument_column)); - break; - } - case GroupByKind::GROUPING_SETS: - { - auto grouping_grouping_sets_function = std::make_shared(arguments_indexes, grouping_sets_keys_indices, force_grouping_standard_compatibility); - auto grouping_grouping_sets_function_adaptor = std::make_shared(std::move(grouping_grouping_sets_function)); - function_node->resolveAsFunction(grouping_grouping_sets_function_adaptor->build({})); - function_node->getArguments().getNodes().push_back(std::move(grouping_set_argument_column)); - break; - } - } - - auto & function_node_arguments = function_node->getArguments().getNodes(); - function_node_arguments.insert(function_node_arguments.end(), previous_arguments.begin(), previous_arguments.end()); - } - - static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) - { - return !(child_node->getNodeType() == QueryTreeNodeType::QUERY || child_node->getNodeType() == QueryTreeNodeType::UNION); - } - -private: - GroupByKind group_by_kind; - std::unordered_map aggegation_key_to_index; - // Indexes of aggregation keys used in each grouping set (only for GROUP BY GROUPING SETS) - ColumnNumbersList grouping_sets_keys_indices; - const PlannerContext & planner_context; -}; - -void resolveGroupingFunctions(QueryTreeNodePtr & node, - GroupByKind group_by_kind, - const Names & aggregation_keys, - const GroupingSetsParamsList & grouping_sets_parameters_list, - const PlannerContext & planner_context) -{ - auto & query_node_typed = node->as(); - - GroupingFunctionResolveVisitor visitor(group_by_kind, aggregation_keys, grouping_sets_parameters_list, planner_context); - - if (query_node_typed.hasHaving()) - visitor.visit(query_node_typed.getHaving()); - - if (query_node_typed.hasOrderBy()) - visitor.visit(query_node_typed.getOrderByNode()); - - visitor.visit(query_node_typed.getProjectionNode()); -} - -void resolveGroupingFunctions(QueryTreeNodePtr & query_node, - const Names & aggregation_keys, - const GroupingSetsParamsList & grouping_sets_parameters_list, - const PlannerContext & planner_context) -{ - auto & query_node_typed = query_node->as(); - - GroupByKind group_by_kind = GroupByKind::ORDINARY; - if (query_node_typed.isGroupByWithRollup()) - group_by_kind = GroupByKind::ROLLUP; - else if (query_node_typed.isGroupByWithCube()) - group_by_kind = GroupByKind::CUBE; - else if (query_node_typed.isGroupByWithGroupingSets()) - group_by_kind = GroupByKind::GROUPING_SETS; - - resolveGroupingFunctions(query_node, group_by_kind, aggregation_keys, grouping_sets_parameters_list, planner_context); -} - -} - -void resolveGroupingFunctions(QueryTreeNodePtr & query_node, const PlannerContext & planner_context) -{ - auto & query_node_typed = query_node->as(); - - std::unordered_set used_aggregation_keys; - Names aggregation_keys; - GroupingSetsParamsList grouping_sets_parameters_list; - - QueryTreeNodeToName group_by_node_to_name; - - /// Add expressions from GROUP BY - - if (query_node_typed.hasGroupBy()) - { - if (query_node_typed.isGroupByWithGroupingSets()) - { - for (const auto & grouping_set_keys_list_node : query_node_typed.getGroupBy().getNodes()) - { - auto & grouping_set_keys_list_node_typed = grouping_set_keys_list_node->as(); - grouping_sets_parameters_list.emplace_back(); - auto & grouping_sets_parameters = grouping_sets_parameters_list.back(); - - for (auto & grouping_set_key_node : grouping_set_keys_list_node_typed.getNodes()) - { - auto grouping_set_key_name = calculateActionNodeName(grouping_set_key_node, planner_context, group_by_node_to_name); - grouping_sets_parameters.used_keys.push_back(grouping_set_key_name); - - if (used_aggregation_keys.contains(grouping_set_key_name)) - continue; - - aggregation_keys.push_back(grouping_set_key_name); - used_aggregation_keys.insert(aggregation_keys.back()); - } - } - - for (auto & grouping_sets_parameter : grouping_sets_parameters_list) - { - NameSet grouping_sets_used_keys; - Names grouping_sets_keys; - - for (auto & key : grouping_sets_parameter.used_keys) - { - auto [_, inserted] = grouping_sets_used_keys.insert(key); - if (inserted) - grouping_sets_keys.push_back(key); - } - - for (auto & key : aggregation_keys) - { - if (grouping_sets_used_keys.contains(key)) - continue; - - grouping_sets_parameter.missing_keys.push_back(key); - } - - grouping_sets_parameter.used_keys = std::move(grouping_sets_keys); - } - - /// It is expected by execution layer that if there are only 1 grouping sets it will be removed - if (grouping_sets_parameters_list.size() == 1) - grouping_sets_parameters_list.clear(); - } - else - { - for (auto & group_by_key_node : query_node_typed.getGroupBy().getNodes()) - { - auto group_by_key_name = calculateActionNodeName(group_by_key_node, planner_context, group_by_node_to_name); - if (used_aggregation_keys.contains(group_by_key_name)) - continue; - - aggregation_keys.push_back(group_by_key_name); - used_aggregation_keys.insert(aggregation_keys.back()); - } - } - } - - resolveGroupingFunctions(query_node, aggregation_keys, grouping_sets_parameters_list, planner_context); -} - AggregateDescriptions extractAggregateDescriptions(const QueryTreeNodes & aggregate_function_nodes, const PlannerContext & planner_context) { QueryTreeNodeToName node_to_name; diff --git a/src/Planner/PlannerAggregation.h b/src/Planner/PlannerAggregation.h index eb91cb11264..2c12361e954 100644 --- a/src/Planner/PlannerAggregation.h +++ b/src/Planner/PlannerAggregation.h @@ -11,14 +11,6 @@ namespace DB { -/** Resolve GROUPING functions in query node. - * GROUPING function is replaced with specialized GROUPING function based on GROUP BY modifiers. - * For ROLLUP, CUBE, GROUPING SETS specialized GROUPING function take special __grouping_set column as argument - * and previous GROUPING function arguments. - */ -void resolveGroupingFunctions(QueryTreeNodePtr & query_node, - const PlannerContext & planner_context); - /// Extract aggregate descriptions from aggregate function nodes AggregateDescriptions extractAggregateDescriptions(const QueryTreeNodes & aggregate_function_nodes, const PlannerContext & planner_context); diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index b14915c9527..b1017c99c3e 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -79,7 +79,6 @@ std::optional analyzeAggregation(const QueryTreeNodeP GroupingSetsParamsList grouping_sets_parameters_list; bool group_by_with_constant_keys = false; - bool disable_grouping_sets = false; PlannerActionsVisitor actions_visitor(planner_context); @@ -137,13 +136,6 @@ std::optional analyzeAggregation(const QueryTreeNodeP grouping_sets_parameter.used_keys = std::move(grouping_sets_keys); } - - /// It is expected by execution layer that if there are only 1 grouping sets it will be removed - if (grouping_sets_parameters_list.size() == 1) - { - disable_grouping_sets = true; - grouping_sets_parameters_list.clear(); - } } else { @@ -190,7 +182,7 @@ std::optional analyzeAggregation(const QueryTreeNodeP /** For non ordinary GROUP BY we add virtual __grouping_set column * With set number, which is used as an additional key at the stage of merging aggregating data. */ - if (query_node.isGroupByWithRollup() || query_node.isGroupByWithCube() || (query_node.isGroupByWithGroupingSets() && !disable_grouping_sets)) + if (query_node.isGroupByWithRollup() || query_node.isGroupByWithCube() || query_node.isGroupByWithGroupingSets()) aggregates_columns.emplace_back(nullptr, std::make_shared(), "__grouping_set"); /// Only aggregation keys and aggregates are available for next steps after GROUP BY step diff --git a/tests/queries/0_stateless/02534_analyzer_grouping_function.reference b/tests/queries/0_stateless/02534_analyzer_grouping_function.reference new file mode 100644 index 00000000000..fcbf625ef22 --- /dev/null +++ b/tests/queries/0_stateless/02534_analyzer_grouping_function.reference @@ -0,0 +1,141 @@ +-- { echoOn } + +EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY id, value; +QUERY id: 0 + PROJECTION COLUMNS + grouping(id) UInt64 + grouping(value) UInt64 + PROJECTION + LIST id: 1, nodes: 2 + FUNCTION id: 2, function_name: groupingOrdinary, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 3, nodes: 1 + COLUMN id: 4, column_name: id, result_type: UInt64, source_id: 5 + FUNCTION id: 6, function_name: groupingOrdinary, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 7, nodes: 1 + COLUMN id: 8, column_name: value, result_type: String, source_id: 5 + JOIN TREE + TABLE id: 5, table_name: default.test_table + GROUP BY + LIST id: 9, nodes: 2 + COLUMN id: 4, column_name: id, result_type: UInt64, source_id: 5 + COLUMN id: 8, column_name: value, result_type: String, source_id: 5 +SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table +GROUP BY id, value ORDER BY grouping_id, grouping_value; +0 0 0 Value +EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY ROLLUP (id, value); +QUERY id: 0, group_by_type: rollup + PROJECTION COLUMNS + grouping(id) UInt64 + grouping(value) UInt64 + PROJECTION + LIST id: 1, nodes: 2 + FUNCTION id: 2, function_name: groupingForRollup, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 3, nodes: 2 + COLUMN id: 4, column_name: __grouping_set, result_type: UInt64 + COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6 + FUNCTION id: 7, function_name: groupingForRollup, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: __grouping_set, result_type: UInt64 + COLUMN id: 10, column_name: value, result_type: String, source_id: 6 + JOIN TREE + TABLE id: 6, table_name: default.test_table + GROUP BY + LIST id: 11, nodes: 2 + COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6 + COLUMN id: 10, column_name: value, result_type: String, source_id: 6 +SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table +GROUP BY ROLLUP (id, value) ORDER BY grouping_id, grouping_value; +0 0 0 Value +0 1 0 +1 1 0 +EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY CUBE (id, value); +QUERY id: 0, group_by_type: cube + PROJECTION COLUMNS + grouping(id) UInt64 + grouping(value) UInt64 + PROJECTION + LIST id: 1, nodes: 2 + FUNCTION id: 2, function_name: groupingForCube, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 3, nodes: 2 + COLUMN id: 4, column_name: __grouping_set, result_type: UInt64 + COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6 + FUNCTION id: 7, function_name: groupingForCube, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: __grouping_set, result_type: UInt64 + COLUMN id: 10, column_name: value, result_type: String, source_id: 6 + JOIN TREE + TABLE id: 6, table_name: default.test_table + GROUP BY + LIST id: 11, nodes: 2 + COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6 + COLUMN id: 10, column_name: value, result_type: String, source_id: 6 +SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table +GROUP BY CUBE (id, value) ORDER BY grouping_id, grouping_value; +0 0 0 Value +0 1 0 +1 0 0 Value +1 1 0 +EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY GROUPING SETS (id, value); +QUERY id: 0, group_by_type: grouping_sets + PROJECTION COLUMNS + grouping(id) UInt64 + grouping(value) UInt64 + PROJECTION + LIST id: 1, nodes: 2 + FUNCTION id: 2, function_name: groupingForGroupingSets, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 3, nodes: 2 + COLUMN id: 4, column_name: __grouping_set, result_type: UInt64 + COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6 + FUNCTION id: 7, function_name: groupingForGroupingSets, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: __grouping_set, result_type: UInt64 + COLUMN id: 10, column_name: value, result_type: String, source_id: 6 + JOIN TREE + TABLE id: 6, table_name: default.test_table + GROUP BY + LIST id: 11, nodes: 2 + LIST id: 12, nodes: 1 + COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6 + LIST id: 13, nodes: 1 + COLUMN id: 10, column_name: value, result_type: String, source_id: 6 +SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table +GROUP BY GROUPING SETS (id, value) ORDER BY grouping_id, grouping_value; +0 1 0 +1 0 0 Value +EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY GROUPING SETS ((id), (value)); +QUERY id: 0, group_by_type: grouping_sets + PROJECTION COLUMNS + grouping(id) UInt64 + grouping(value) UInt64 + PROJECTION + LIST id: 1, nodes: 2 + FUNCTION id: 2, function_name: groupingForGroupingSets, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 3, nodes: 2 + COLUMN id: 4, column_name: __grouping_set, result_type: UInt64 + COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6 + FUNCTION id: 7, function_name: groupingForGroupingSets, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: __grouping_set, result_type: UInt64 + COLUMN id: 10, column_name: value, result_type: String, source_id: 6 + JOIN TREE + TABLE id: 6, table_name: default.test_table + GROUP BY + LIST id: 11, nodes: 2 + LIST id: 12, nodes: 1 + COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6 + LIST id: 13, nodes: 1 + COLUMN id: 10, column_name: value, result_type: String, source_id: 6 +SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table +GROUP BY GROUPING SETS ((id), (value)) ORDER BY grouping_id, grouping_value; +0 1 0 +1 0 0 Value diff --git a/tests/queries/0_stateless/02534_analyzer_grouping_function.sql b/tests/queries/0_stateless/02534_analyzer_grouping_function.sql new file mode 100644 index 00000000000..3163e03d579 --- /dev/null +++ b/tests/queries/0_stateless/02534_analyzer_grouping_function.sql @@ -0,0 +1,41 @@ +SET allow_experimental_analyzer = 1; + +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table +( + id UInt64, + value String +) ENGINE=MergeTree ORDER BY id; + +INSERT INTO test_table VALUES (0, 'Value'); + +-- { echoOn } + +EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY id, value; + +SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table +GROUP BY id, value ORDER BY grouping_id, grouping_value; + +EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY ROLLUP (id, value); + +SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table +GROUP BY ROLLUP (id, value) ORDER BY grouping_id, grouping_value; + +EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY CUBE (id, value); + +SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table +GROUP BY CUBE (id, value) ORDER BY grouping_id, grouping_value; + +EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY GROUPING SETS (id, value); + +SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table +GROUP BY GROUPING SETS (id, value) ORDER BY grouping_id, grouping_value; + +EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY GROUPING SETS ((id), (value)); + +SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table +GROUP BY GROUPING SETS ((id), (value)) ORDER BY grouping_id, grouping_value; + +-- { echoOff } + +DROP TABLE test_table; From e6ee5554d11ae2c7c42379718645ccef393e2187 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 20 Jan 2023 11:15:13 +0100 Subject: [PATCH 4/4] Fixed tests --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 695a542fc42..b5efd090e4c 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -4352,7 +4352,8 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi bool force_grouping_standard_compatibility = scope.context->getSettingsRef().force_grouping_standard_compatibility; auto grouping_function = std::make_shared(force_grouping_standard_compatibility); auto grouping_function_adaptor = std::make_shared(std::move(grouping_function)); - function_node.resolveAsFunction(grouping_function_adaptor->build({})); + function_node.resolveAsFunction(grouping_function_adaptor->build(argument_columns)); + return result_projection_names; } }