thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 09:32:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #45254 from kitaisreal/planner-small-fixes

Browse Source 
Planner small fixes
This commit is contained in:
Maksim Kita 2023-01-21 19:54:17 +03:00 committed by GitHub
commit 19f1bae5ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 522 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

253
src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp Normal file
View File

@ -0,0 +1,253 @@
#include <Analyzer/Passes/GroupingFunctionsResolvePass.h>
#include <Core/ColumnNumbers.h>
#include <Functions/grouping.h>
#include <Interpreters/Context.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/QueryNode.h>
#include <Analyzer/HashUtils.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/ColumnNode.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int LOGICAL_ERROR;
}
namespace
{
enum class GroupByKind
{
ORDINARY,
ROLLUP,
CUBE,
GROUPING_SETS
};
class GroupingFunctionResolveVisitor : public InDepthQueryTreeVisitor<GroupingFunctionResolveVisitor>
{
public:
GroupingFunctionResolveVisitor(GroupByKind group_by_kind_,
QueryTreeNodePtrWithHashMap<size_t> aggregation_key_to_index_,
ColumnNumbersList grouping_sets_keys_indices_,
ContextPtr context_)
: group_by_kind(group_by_kind_)
, aggregation_key_to_index(std::move(aggregation_key_to_index_))
, grouping_sets_keys_indexes(std::move(grouping_sets_keys_indices_))
, context(std::move(context_))
{
}
void visitImpl(const QueryTreeNodePtr & node)
{
auto * function_node = node->as<FunctionNode>();
if (!function_node || function_node->getFunctionName() != "grouping")
return;
auto & function_arguments = function_node->getArguments().getNodes();
ColumnNumbers arguments_indexes;
arguments_indexes.reserve(function_arguments.size());
for (const auto & argument : function_arguments)
{
auto it = aggregation_key_to_index.find(argument);
if (it == aggregation_key_to_index.end())
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Argument {} of GROUPING function is not a part of GROUP BY clause",
argument->formatASTForErrorMessage());
arguments_indexes.push_back(it->second);
}
FunctionOverloadResolverPtr grouping_function_resolver;
bool add_grouping_set_column = false;
bool force_grouping_standard_compatibility = context->getSettingsRef().force_grouping_standard_compatibility;
size_t aggregation_keys_size = aggregation_key_to_index.size();
switch (group_by_kind)
{
case GroupByKind::ORDINARY:
{
auto grouping_ordinary_function = std::make_shared<FunctionGroupingOrdinary>(arguments_indexes,
force_grouping_standard_compatibility);
grouping_function_resolver = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_ordinary_function));
break;
}
case GroupByKind::ROLLUP:
{
auto grouping_rollup_function = std::make_shared<FunctionGroupingForRollup>(arguments_indexes,
aggregation_keys_size,
force_grouping_standard_compatibility);
grouping_function_resolver = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_rollup_function));
add_grouping_set_column = true;
break;
}
case GroupByKind::CUBE:
{
auto grouping_cube_function = std::make_shared<FunctionGroupingForCube>(arguments_indexes,
aggregation_keys_size,
force_grouping_standard_compatibility);
grouping_function_resolver = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_cube_function));
add_grouping_set_column = true;
break;
}
case GroupByKind::GROUPING_SETS:
{
auto grouping_grouping_sets_function = std::make_shared<FunctionGroupingForGroupingSets>(arguments_indexes,
grouping_sets_keys_indexes,
force_grouping_standard_compatibility);
grouping_function_resolver = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_grouping_sets_function));
add_grouping_set_column = true;
break;
}
}
if (add_grouping_set_column)
{
QueryTreeNodeWeakPtr column_source;
auto grouping_set_column = NameAndTypePair{"__grouping_set", std::make_shared<DataTypeUInt64>()};
auto grouping_set_argument_column = std::make_shared<ColumnNode>(std::move(grouping_set_column), std::move(column_source));
function_arguments.insert(function_arguments.begin(), std::move(grouping_set_argument_column));
}
function_node->resolveAsFunction(grouping_function_resolver->build(function_node->getArgumentColumns()));
}
static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node)
{
return !(child_node->getNodeType() == QueryTreeNodeType::QUERY || child_node->getNodeType() == QueryTreeNodeType::UNION);
}
private:
GroupByKind group_by_kind;
QueryTreeNodePtrWithHashMap<size_t> aggregation_key_to_index;
ColumnNumbersList grouping_sets_keys_indexes;
ContextPtr context;
};
void resolveGroupingFunctions(QueryTreeNodePtr & query_node, ContextPtr context)
{
auto & query_node_typed = query_node->as<QueryNode &>();
size_t aggregation_node_index = 0;
QueryTreeNodePtrWithHashMap<size_t> aggregation_key_to_index;
std::vector<QueryTreeNodes> grouping_sets_used_aggregation_keys_list;
if (query_node_typed.hasGroupBy())
{
/// It is expected by execution layer that if there are only 1 grouping set it will be removed
if (query_node_typed.isGroupByWithGroupingSets() && query_node_typed.getGroupBy().getNodes().size() == 1)
{
auto & grouping_set_list_node = query_node_typed.getGroupBy().getNodes().front()->as<ListNode &>();
query_node_typed.getGroupBy().getNodes() = std::move(grouping_set_list_node.getNodes());
query_node_typed.setIsGroupByWithGroupingSets(false);
}
if (query_node_typed.isGroupByWithGroupingSets())
{
for (const auto & grouping_set_keys_list_node : query_node_typed.getGroupBy().getNodes())
{
auto & grouping_set_keys_list_node_typed = grouping_set_keys_list_node->as<ListNode &>();
grouping_sets_used_aggregation_keys_list.emplace_back();
auto & grouping_sets_used_aggregation_keys = grouping_sets_used_aggregation_keys_list.back();
for (auto & grouping_set_key_node : grouping_set_keys_list_node_typed.getNodes())
{
if (aggregation_key_to_index.contains(grouping_set_key_node))
continue;
grouping_sets_used_aggregation_keys.push_back(grouping_set_key_node);
aggregation_key_to_index.emplace(grouping_set_key_node, aggregation_node_index);
++aggregation_node_index;
}
}
}
else
{
for (auto & group_by_key_node : query_node_typed.getGroupBy().getNodes())
{
if (aggregation_key_to_index.contains(group_by_key_node))
continue;
aggregation_key_to_index.emplace(group_by_key_node, aggregation_node_index);
++aggregation_node_index;
}
}
}
/// Indexes of aggregation keys used in each grouping set (only for GROUP BY GROUPING SETS)
ColumnNumbersList grouping_sets_keys_indexes;
for (const auto & grouping_set_used_aggregation_keys : grouping_sets_used_aggregation_keys_list)
{
grouping_sets_keys_indexes.emplace_back();
auto & grouping_set_keys_indexes = grouping_sets_keys_indexes.back();
for (const auto & used_aggregation_key : grouping_set_used_aggregation_keys)
{
auto aggregation_node_index_it = aggregation_key_to_index.find(used_aggregation_key);
if (aggregation_node_index_it == aggregation_key_to_index.end())
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Aggregation key {} in GROUPING SETS is not found in GROUP BY keys",
used_aggregation_key->formatASTForErrorMessage());
grouping_set_keys_indexes.push_back(aggregation_node_index_it->second);
}
}
GroupByKind group_by_kind = GroupByKind::ORDINARY;
if (query_node_typed.isGroupByWithRollup())
group_by_kind = GroupByKind::ROLLUP;
else if (query_node_typed.isGroupByWithCube())
group_by_kind = GroupByKind::CUBE;
else if (query_node_typed.isGroupByWithGroupingSets())
group_by_kind = GroupByKind::GROUPING_SETS;
GroupingFunctionResolveVisitor visitor(group_by_kind,
std::move(aggregation_key_to_index),
std::move(grouping_sets_keys_indexes),
std::move(context));
visitor.visit(query_node);
}
class GroupingFunctionsResolveVisitor : public InDepthQueryTreeVisitor<GroupingFunctionsResolveVisitor>
{
public:
explicit GroupingFunctionsResolveVisitor(ContextPtr context_)
: context(std::move(context_))
{}
void visitImpl(QueryTreeNodePtr & node)
{
if (node->getNodeType() != QueryTreeNodeType::QUERY)
return;
resolveGroupingFunctions(node, context);
}
private:
ContextPtr context;
};
}
void GroupingFunctionsResolvePass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
{
GroupingFunctionsResolveVisitor visitor(std::move(context));
visitor.visit(query_tree_node);
}
}

31
src/Analyzer/Passes/GroupingFunctionsResolvePass.h Normal file
View File

@ -0,0 +1,31 @@
#pragma once
#include <Analyzer/IQueryTreePass.h>
namespace DB
{
/** Resolve GROUPING functions in query node.
* GROUPING function is replaced with specialized GROUPING function based on GROUP BY modifiers.
* For ROLLUP, CUBE, GROUPING SETS specialized GROUPING function take special __grouping_set column as argument
* and previous GROUPING function arguments.
*
* Example: SELECT grouping(id) FROM test_table GROUP BY id;
* Result: SELECT groupingOrdinary(id) FROM test_table GROUP BY id;
*
* Example: SELECT grouping(id), grouping(value) FROM test_table GROUP BY GROUPING SETS ((id), (value));
* Result: SELECT groupingForGroupingSets(__grouping_set, id), groupingForGroupingSets(__grouping_set, value)
* FROM test_table GROUP BY GROUPING SETS ((id), (value));
*/
class GroupingFunctionsResolvePass final : public IQueryTreePass
{
public:
String getName() override { return "GroupingFunctionsResolvePass"; }
String getDescription() override { return "Resolve GROUPING functions based on GROUP BY modifiers"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
};
}

3
src/Analyzer/Passes/QueryAnalysisPass.cpp
View File

@ -4352,7 +4352,8 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
bool force_grouping_standard_compatibility = scope.context->getSettingsRef().force_grouping_standard_compatibility;
auto grouping_function = std::make_shared<FunctionGrouping>(force_grouping_standard_compatibility);
auto grouping_function_adaptor = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_function));
function_node.resolveAsFunction(grouping_function_adaptor->build({}));
function_node.resolveAsFunction(grouping_function_adaptor->build(argument_columns));
return result_projection_names;
}
}

5
src/Analyzer/QueryTreePassManager.cpp
View File

@ -32,6 +32,7 @@
#include <Analyzer/Passes/IfTransformStringsToEnumPass.h>
#include <Analyzer/Passes/ConvertOrLikeChainPass.h>
#include <Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h>
#include <Analyzer/Passes/GroupingFunctionsResolvePass.h>
namespace DB
{
@ -67,7 +68,7 @@ public:
private:
void visitColumn(ColumnNode * column) const
{
if (column->getColumnSourceOrNull() == nullptr)
if (column->getColumnSourceOrNull() == nullptr && column->getColumnName() != "__grouping_set")
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Column {} {} query tree node does not have valid source node after running {} pass",
column->getColumnName(), column->getColumnType(), pass_name);
@ -258,6 +259,8 @@ void addQueryTreePasses(QueryTreePassManager & manager)
manager.addPass(std::make_unique<IfTransformStringsToEnumPass>());
manager.addPass(std::make_unique<ConvertOrLikeChainPass>());
manager.addPass(std::make_unique<GroupingFunctionsResolvePass>());
}
}

3
src/Planner/CollectTableExpressionData.cpp
View File

@ -35,6 +35,9 @@ public:
if (!column_node)
return;
if (column_node->getColumnName() == "__grouping_set")
return;
auto column_source_node = column_node->getColumnSource();
auto column_source_node_type = column_source_node->getNodeType();

175
src/Planner/PlannerAggregation.cpp
View File

@ -1,188 +1,13 @@
#include <Planner/PlannerAggregation.h>
#include <Functions/grouping.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/ConstantNode.h>
#include <Analyzer/ColumnNode.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/QueryNode.h>
#include <Analyzer/AggregationUtils.h>
#include <Interpreters/Context.h>
#include <Processors/QueryPlan/AggregatingStep.h>
#include <Planner/PlannerActionsVisitor.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int BAD_ARGUMENTS;
}
namespace
{
enum class GroupByKind
{
ORDINARY,
ROLLUP,
CUBE,
GROUPING_SETS
};
class GroupingFunctionResolveVisitor : public InDepthQueryTreeVisitor<GroupingFunctionResolveVisitor>
{
public:
GroupingFunctionResolveVisitor(GroupByKind group_by_kind_,
const Names & aggregation_keys_,
const GroupingSetsParamsList & grouping_sets_parameters_list_,
const PlannerContext & planner_context_)
: group_by_kind(group_by_kind_)
, planner_context(planner_context_)
{
size_t aggregation_keys_size = aggregation_keys_.size();
for (size_t i = 0; i < aggregation_keys_size; ++i)
aggegation_key_to_index.emplace(aggregation_keys_[i], i);
for (const auto & grouping_sets_parameter : grouping_sets_parameters_list_)
{
grouping_sets_keys_indices.emplace_back();
auto & grouping_set_keys_indices = grouping_sets_keys_indices.back();
for (const auto & used_key : grouping_sets_parameter.used_keys)
{
auto aggregation_key_index_it = aggegation_key_to_index.find(used_key);
if (aggregation_key_index_it == aggegation_key_to_index.end())
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Aggregation key {} in GROUPING SETS is not found in GROUP BY keys");
grouping_set_keys_indices.push_back(aggregation_key_index_it->second);
}
}
}
void visitImpl(const QueryTreeNodePtr & node)
{
auto * function_node = node->as<FunctionNode>();
if (!function_node || function_node->getFunctionName() != "grouping")
return;
size_t aggregation_keys_size = aggegation_key_to_index.size();
ColumnNumbers arguments_indexes;
for (const auto & argument : function_node->getArguments().getNodes())
{
String action_node_name = calculateActionNodeName(argument, planner_context);
auto it = aggegation_key_to_index.find(action_node_name);
if (it == aggegation_key_to_index.end())
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Argument of GROUPING function {} is not a part of GROUP BY clause",
argument->formatASTForErrorMessage());
arguments_indexes.push_back(it->second);
}
QueryTreeNodeWeakPtr column_source;
auto grouping_set_argument_column = std::make_shared<ColumnNode>(NameAndTypePair{"__grouping_set", std::make_shared<DataTypeUInt64>()}, column_source);
function_node->getArguments().getNodes().clear();
bool force_grouping_standard_compatibility = planner_context.getQueryContext()->getSettingsRef().force_grouping_standard_compatibility;
switch (group_by_kind)
{
case GroupByKind::ORDINARY:
{
auto grouping_ordinary_function = std::make_shared<FunctionGroupingOrdinary>(arguments_indexes, force_grouping_standard_compatibility);
auto grouping_ordinary_function_adaptor = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_ordinary_function));
function_node->resolveAsFunction(grouping_ordinary_function_adaptor->build({}));
break;
}
case GroupByKind::ROLLUP:
{
auto grouping_rollup_function = std::make_shared<FunctionGroupingForRollup>(arguments_indexes, aggregation_keys_size, force_grouping_standard_compatibility);
auto grouping_rollup_function_adaptor = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_rollup_function));
function_node->resolveAsFunction(grouping_rollup_function_adaptor->build({}));
function_node->getArguments().getNodes().push_back(std::move(grouping_set_argument_column));
break;
}
case GroupByKind::CUBE:
{
auto grouping_cube_function = std::make_shared<FunctionGroupingForCube>(arguments_indexes, aggregation_keys_size, force_grouping_standard_compatibility);
auto grouping_cube_function_adaptor = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_cube_function));
function_node->resolveAsFunction(grouping_cube_function_adaptor->build({}));
function_node->getArguments().getNodes().push_back(std::move(grouping_set_argument_column));
break;
}
case GroupByKind::GROUPING_SETS:
{
auto grouping_grouping_sets_function = std::make_shared<FunctionGroupingForGroupingSets>(arguments_indexes, grouping_sets_keys_indices, force_grouping_standard_compatibility);
auto grouping_grouping_sets_function_adaptor = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_grouping_sets_function));
function_node->resolveAsFunction(grouping_grouping_sets_function_adaptor->build({}));
function_node->getArguments().getNodes().push_back(std::move(grouping_set_argument_column));
break;
}
}
}
static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node)
{
return !(child_node->getNodeType() == QueryTreeNodeType::QUERY || child_node->getNodeType() == QueryTreeNodeType::UNION);
}
private:
GroupByKind group_by_kind;
std::unordered_map<std::string, size_t> aggegation_key_to_index;
// Indexes of aggregation keys used in each grouping set (only for GROUP BY GROUPING SETS)
ColumnNumbersList grouping_sets_keys_indices;
const PlannerContext & planner_context;
};
void resolveGroupingFunctions(QueryTreeNodePtr & node,
GroupByKind group_by_kind,
const Names & aggregation_keys,
const GroupingSetsParamsList & grouping_sets_parameters_list,
const PlannerContext & planner_context)
{
auto & query_node_typed = node->as<QueryNode &>();
GroupingFunctionResolveVisitor visitor(group_by_kind, aggregation_keys, grouping_sets_parameters_list, planner_context);
if (query_node_typed.hasHaving())
visitor.visit(query_node_typed.getHaving());
if (query_node_typed.hasOrderBy())
visitor.visit(query_node_typed.getOrderByNode());
visitor.visit(query_node_typed.getProjectionNode());
}
}
void resolveGroupingFunctions(QueryTreeNodePtr & query_node,
const Names & aggregation_keys,
const GroupingSetsParamsList & grouping_sets_parameters_list,
const PlannerContext & planner_context)
{
auto & query_node_typed = query_node->as<QueryNode &>();
GroupByKind group_by_kind = GroupByKind::ORDINARY;
if (query_node_typed.isGroupByWithRollup())
group_by_kind = GroupByKind::ROLLUP;
else if (query_node_typed.isGroupByWithCube())
group_by_kind = GroupByKind::CUBE;
else if (query_node_typed.isGroupByWithGroupingSets())
group_by_kind = GroupByKind::GROUPING_SETS;
resolveGroupingFunctions(query_node, group_by_kind, aggregation_keys, grouping_sets_parameters_list, planner_context);
}
AggregateDescriptions extractAggregateDescriptions(const QueryTreeNodes & aggregate_function_nodes, const PlannerContext & planner_context)
{
QueryTreeNodeToName node_to_name;

9
src/Planner/PlannerAggregation.h
View File

@ -11,15 +11,6 @@
namespace DB
{
/** Resolve GROUPING functions in query node.
* GROUPING function is replaced with specialized GROUPING function based on GROUP BY modifiers.
* For ROLLUP, CUBE, GROUPING SETS specialized GROUPING function take special __grouping_set column as argument.
*/
void resolveGroupingFunctions(QueryTreeNodePtr & query_node,
const Names & aggregation_keys,
const GroupingSetsParamsList & grouping_sets_parameters_list,
const PlannerContext & planner_context);
/// Extract aggregate descriptions from aggregate function nodes
AggregateDescriptions extractAggregateDescriptions(const QueryTreeNodes & aggregate_function_nodes, const PlannerContext & planner_context);

24
src/Planner/PlannerExpressionAnalysis.cpp
View File

@ -52,7 +52,7 @@ FilterAnalysisResult analyzeFilter(const QueryTreeNodePtr & filter_expression_no
/** Construct aggregation analysis result if query tree has GROUP BY or aggregates.
* Actions before aggregation are added into actions chain, if result is not null optional.
*/
std::optional<AggregationAnalysisResult> analyzeAggregation(QueryTreeNodePtr & query_tree,
std::optional<AggregationAnalysisResult> analyzeAggregation(const QueryTreeNodePtr & query_tree,
const ColumnsWithTypeAndName & join_tree_input_columns,
const PlannerContextPtr & planner_context,
ActionsChain & actions_chain)
@ -79,7 +79,6 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(QueryTreeNodePtr & q
GroupingSetsParamsList grouping_sets_parameters_list;
bool group_by_with_constant_keys = false;
bool disable_grouping_sets = false;
PlannerActionsVisitor actions_visitor(planner_context);
@ -137,13 +136,6 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(QueryTreeNodePtr & q
grouping_sets_parameter.used_keys = std::move(grouping_sets_keys);
}
/// It is expected by execution layer that if there are only 1 grouping sets it will be removed
if (grouping_sets_parameters_list.size() == 1)
{
disable_grouping_sets = true;
grouping_sets_parameters_list.clear();
}
}
else
{
@ -190,11 +182,9 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(QueryTreeNodePtr & q
/** For non ordinary GROUP BY we add virtual __grouping_set column
* With set number, which is used as an additional key at the stage of merging aggregating data.
*/
if (query_node.isGroupByWithRollup() || query_node.isGroupByWithCube() || (query_node.isGroupByWithGroupingSets() && !disable_grouping_sets))
if (query_node.isGroupByWithRollup() || query_node.isGroupByWithCube() || query_node.isGroupByWithGroupingSets())
aggregates_columns.emplace_back(nullptr, std::make_shared<DataTypeUInt64>(), "__grouping_set");
resolveGroupingFunctions(query_tree, aggregation_keys, grouping_sets_parameters_list, *planner_context);
/// Only aggregation keys and aggregates are available for next steps after GROUP BY step
auto aggregate_step = std::make_unique<ActionsChainStep>(before_aggregation_actions, ActionsChainStep::AvailableOutputColumnsStrategy::OUTPUT_NODES, aggregates_columns);
actions_chain.addStep(std::move(aggregate_step));
@ -212,7 +202,7 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(QueryTreeNodePtr & q
/** Construct window analysis result if query tree has window functions.
* Actions before window functions are added into actions chain, if result is not null optional.
*/
std::optional<WindowAnalysisResult> analyzeWindow(QueryTreeNodePtr & query_tree,
std::optional<WindowAnalysisResult> analyzeWindow(const QueryTreeNodePtr & query_tree,
const ColumnsWithTypeAndName & join_tree_input_columns,
const PlannerContextPtr & planner_context,
ActionsChain & actions_chain)
@ -417,7 +407,7 @@ LimitByAnalysisResult analyzeLimitBy(const QueryNode & query_node,
}
PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(QueryTreeNodePtr query_tree,
PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNodePtr & query_tree,
const ColumnsWithTypeAndName & join_tree_input_columns,
const PlannerContextPtr & planner_context)
{
@ -463,14 +453,8 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(QueryTreeNodePtr
project_names_actions->project(projection_analysis_result.projection_column_names_with_display_aliases);
actions_chain.addStep(std::make_unique<ActionsChainStep>(project_names_actions));
// std::cout << "Chain dump before finalize" << std::endl;
// std::cout << actions_chain.dump() << std::endl;
actions_chain.finalize();
// std::cout << "Chain dump after finalize" << std::endl;
// std::cout << actions_chain.dump() << std::endl;
projection_analysis_result.project_names_actions = std::move(project_names_actions);
PlannerExpressionsAnalysisResult expressions_analysis_result(std::move(projection_analysis_result));

2
src/Planner/PlannerExpressionAnalysis.h
View File

@ -168,7 +168,7 @@ private:
};
/// Build expression analysis result for query tree, join tree input columns and planner context
PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(QueryTreeNodePtr query_tree,
PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNodePtr & query_tree,
const ColumnsWithTypeAndName & join_tree_input_columns,
const PlannerContextPtr & planner_context);

18
src/Planner/PlannerJoinTree.cpp
View File

@ -248,7 +248,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(const QueryTreeNodePtr & tabl
if (max_streams == 0)
max_streams = 1;
/// If necessary, we request more sources than the number of threads - to distribute the work evenly over the threads.
/// If necessary, we request more sources than the number of threads - to distribute the work evenly over the threads
if (max_streams > 1 && !is_remote)
max_streams = static_cast<size_t>(max_streams * settings.max_streams_to_max_threads_ratio);
@ -841,7 +841,7 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node,
std::vector<ColumnIdentifierSet> table_expressions_outer_scope_columns(table_expressions_stack_size);
ColumnIdentifierSet current_outer_scope_columns = outer_scope_columns;
for (Int64 i = table_expressions_stack_size - 1; i >= 0; --i)
for (Int64 i = static_cast<Int64>(table_expressions_stack_size) - 1; i >= 0; --i)
{
table_expressions_outer_scope_columns[i] = current_outer_scope_columns;
@ -859,7 +859,8 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node,
{
if (query_plans_stack.empty())
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Expected at least 1 query plan on stack before ARRAY JOIN processing");
"Expected at least 1 query plan on stack before ARRAY JOIN processing. Actual {}",
query_plans_stack.size());
auto query_plan = std::move(query_plans_stack.back());
query_plans_stack.back() = buildQueryPlanForArrayJoinNode(table_expression,
@ -868,11 +869,10 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node,
}
else if (auto * join_node = table_expression->as<JoinNode>())
{
size_t table_expressions_column_nodes_with_names_stack_size = query_plans_stack.size();
if (table_expressions_column_nodes_with_names_stack_size < 2)
if (query_plans_stack.size() < 2)
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Expected at least 2 query plans on stack before JOIN processing. Actual {}",
table_expressions_column_nodes_with_names_stack_size);
query_plans_stack.size());
auto right_query_plan = std::move(query_plans_stack.back());
query_plans_stack.pop_back();
@ -901,8 +901,10 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node,
}
}
if (query_plans_stack.empty())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected at least 1 query plan for JOIN TREE");
if (query_plans_stack.size() != 1)
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Expected 1 query plan for JOIN TREE. Actual {}",
query_plans_stack.size());
return std::move(query_plans_stack.back());
}

4
src/Storages/StorageDistributed.cpp
View File

@ -578,9 +578,9 @@ std::optional<QueryProcessingStage::Enum> StorageDistributed::getOptimizedQueryP
bool has_aggregates = query_info.has_aggregates;
if (query_info.syntax_analyzer_result)
has_aggregates = query_info.syntax_analyzer_result->aggregates.empty();
has_aggregates = !query_info.syntax_analyzer_result->aggregates.empty();
if (!has_aggregates || group_by)
if (has_aggregates || group_by)
{
if (!optimize_sharding_key_aggregation || !group_by || !expr_contains_sharding_key(group_by->children))
return {};

10
tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.reference Normal file
View File

@ -0,0 +1,10 @@
45 0 0 0 1
45 0 1 0 1
45 0 2 0 1
45 0 3 0 1
45 0 4 0 1
45 0 5 0 1
45 0 6 0 1
45 0 7 0 1
45 0 8 0 1
45 0 9 0 1

20
tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.sql Normal file
View File

@ -0,0 +1,20 @@
SET allow_experimental_analyzer = 1;
SELECT
sum(a.number) AS total,
c.number AS cn,
b.number AS bn,
grouping(c.number) + grouping(b.number) AS l,
rank() OVER (PARTITION BY grouping(c.number) + grouping(b.number), multiIf(grouping(c.number) = 0, b.number, NULL) ORDER BY sum(a.number) DESC) AS r
FROM numbers(10) AS a, numbers(10) AS b, numbers(10) AS c
GROUP BY
cn,
bn
WITH ROLLUP
ORDER BY
total ASC,
cn ASC,
bn ASC,
l ASC,
r ASC
LIMIT 10;

141
tests/queries/0_stateless/02534_analyzer_grouping_function.reference Normal file
View File

@ -0,0 +1,141 @@
-- { echoOn }
EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY id, value;
QUERY id: 0
PROJECTION COLUMNS
grouping(id) UInt64
grouping(value) UInt64
PROJECTION
LIST id: 1, nodes: 2
FUNCTION id: 2, function_name: groupingOrdinary, function_type: ordinary, result_type: UInt64
ARGUMENTS
LIST id: 3, nodes: 1
COLUMN id: 4, column_name: id, result_type: UInt64, source_id: 5
FUNCTION id: 6, function_name: groupingOrdinary, function_type: ordinary, result_type: UInt64
ARGUMENTS
LIST id: 7, nodes: 1
COLUMN id: 8, column_name: value, result_type: String, source_id: 5
JOIN TREE
TABLE id: 5, table_name: default.test_table
GROUP BY
LIST id: 9, nodes: 2
COLUMN id: 4, column_name: id, result_type: UInt64, source_id: 5
COLUMN id: 8, column_name: value, result_type: String, source_id: 5
SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table
GROUP BY id, value ORDER BY grouping_id, grouping_value;
0 0 0 Value
EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY ROLLUP (id, value);
QUERY id: 0, group_by_type: rollup
PROJECTION COLUMNS
grouping(id) UInt64
grouping(value) UInt64
PROJECTION
LIST id: 1, nodes: 2
FUNCTION id: 2, function_name: groupingForRollup, function_type: ordinary, result_type: UInt64
ARGUMENTS
LIST id: 3, nodes: 2
COLUMN id: 4, column_name: __grouping_set, result_type: UInt64
COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6
FUNCTION id: 7, function_name: groupingForRollup, function_type: ordinary, result_type: UInt64
ARGUMENTS
LIST id: 8, nodes: 2
COLUMN id: 9, column_name: __grouping_set, result_type: UInt64
COLUMN id: 10, column_name: value, result_type: String, source_id: 6
JOIN TREE
TABLE id: 6, table_name: default.test_table
GROUP BY
LIST id: 11, nodes: 2
COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6
COLUMN id: 10, column_name: value, result_type: String, source_id: 6
SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table
GROUP BY ROLLUP (id, value) ORDER BY grouping_id, grouping_value;
0 0 0 Value
0 1 0
1 1 0
EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY CUBE (id, value);
QUERY id: 0, group_by_type: cube
PROJECTION COLUMNS
grouping(id) UInt64
grouping(value) UInt64
PROJECTION
LIST id: 1, nodes: 2
FUNCTION id: 2, function_name: groupingForCube, function_type: ordinary, result_type: UInt64
ARGUMENTS
LIST id: 3, nodes: 2
COLUMN id: 4, column_name: __grouping_set, result_type: UInt64
COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6
FUNCTION id: 7, function_name: groupingForCube, function_type: ordinary, result_type: UInt64
ARGUMENTS
LIST id: 8, nodes: 2
COLUMN id: 9, column_name: __grouping_set, result_type: UInt64
COLUMN id: 10, column_name: value, result_type: String, source_id: 6
JOIN TREE
TABLE id: 6, table_name: default.test_table
GROUP BY
LIST id: 11, nodes: 2
COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6
COLUMN id: 10, column_name: value, result_type: String, source_id: 6
SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table
GROUP BY CUBE (id, value) ORDER BY grouping_id, grouping_value;
0 0 0 Value
0 1 0
1 0 0 Value
1 1 0
EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY GROUPING SETS (id, value);
QUERY id: 0, group_by_type: grouping_sets
PROJECTION COLUMNS
grouping(id) UInt64
grouping(value) UInt64
PROJECTION
LIST id: 1, nodes: 2
FUNCTION id: 2, function_name: groupingForGroupingSets, function_type: ordinary, result_type: UInt64
ARGUMENTS
LIST id: 3, nodes: 2
COLUMN id: 4, column_name: __grouping_set, result_type: UInt64
COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6
FUNCTION id: 7, function_name: groupingForGroupingSets, function_type: ordinary, result_type: UInt64
ARGUMENTS
LIST id: 8, nodes: 2
COLUMN id: 9, column_name: __grouping_set, result_type: UInt64
COLUMN id: 10, column_name: value, result_type: String, source_id: 6
JOIN TREE
TABLE id: 6, table_name: default.test_table
GROUP BY
LIST id: 11, nodes: 2
LIST id: 12, nodes: 1
COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6
LIST id: 13, nodes: 1
COLUMN id: 10, column_name: value, result_type: String, source_id: 6
SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table
GROUP BY GROUPING SETS (id, value) ORDER BY grouping_id, grouping_value;
0 1 0
1 0 0 Value
EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY GROUPING SETS ((id), (value));
QUERY id: 0, group_by_type: grouping_sets
PROJECTION COLUMNS
grouping(id) UInt64
grouping(value) UInt64
PROJECTION
LIST id: 1, nodes: 2
FUNCTION id: 2, function_name: groupingForGroupingSets, function_type: ordinary, result_type: UInt64
ARGUMENTS
LIST id: 3, nodes: 2
COLUMN id: 4, column_name: __grouping_set, result_type: UInt64
COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6
FUNCTION id: 7, function_name: groupingForGroupingSets, function_type: ordinary, result_type: UInt64
ARGUMENTS
LIST id: 8, nodes: 2
COLUMN id: 9, column_name: __grouping_set, result_type: UInt64
COLUMN id: 10, column_name: value, result_type: String, source_id: 6
JOIN TREE
TABLE id: 6, table_name: default.test_table
GROUP BY
LIST id: 11, nodes: 2
LIST id: 12, nodes: 1
COLUMN id: 5, column_name: id, result_type: UInt64, source_id: 6
LIST id: 13, nodes: 1
COLUMN id: 10, column_name: value, result_type: String, source_id: 6
SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table
GROUP BY GROUPING SETS ((id), (value)) ORDER BY grouping_id, grouping_value;
0 1 0
1 0 0 Value

41
tests/queries/0_stateless/02534_analyzer_grouping_function.sql Normal file
View File

@ -0,0 +1,41 @@
SET allow_experimental_analyzer = 1;
DROP TABLE IF EXISTS test_table;
CREATE TABLE test_table
(
id UInt64,
value String
) ENGINE=MergeTree ORDER BY id;
INSERT INTO test_table VALUES (0, 'Value');
-- { echoOn }
EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY id, value;
SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table
GROUP BY id, value ORDER BY grouping_id, grouping_value;
EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY ROLLUP (id, value);
SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table
GROUP BY ROLLUP (id, value) ORDER BY grouping_id, grouping_value;
EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY CUBE (id, value);
SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table
GROUP BY CUBE (id, value) ORDER BY grouping_id, grouping_value;
EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY GROUPING SETS (id, value);
SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table
GROUP BY GROUPING SETS (id, value) ORDER BY grouping_id, grouping_value;
EXPLAIN QUERY TREE SELECT grouping(id), grouping(value) FROM test_table GROUP BY GROUPING SETS ((id), (value));
SELECT grouping(id) AS grouping_id, grouping(value) AS grouping_value, id, value FROM test_table
GROUP BY GROUPING SETS ((id), (value)) ORDER BY grouping_id, grouping_value;
-- { echoOff }
DROP TABLE test_table;