thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 08:32:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed tests

Browse Source
This commit is contained in:
Maksim Kita 2022-10-20 17:52:52 +02:00
parent 3862dd96fe
commit 9a02d25c0b
10 changed files with 96 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Analyzer/Passes/QueryAnalysisPass.cpp
View File

@ -1451,7 +1451,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, size
if (query_node)
query_node->performConstantFolding(std::move(constant_value));
else if (union_node)
query_node->performConstantFolding(std::move(constant_value));
union_node->performConstantFolding(std::move(constant_value));
return;
}

47
src/Planner/ActionsChain.cpp
View File

@ -11,26 +11,51 @@
namespace DB
{
ActionsChainStep::ActionsChainStep(ActionsDAGPtr actions_, AvailableOutputColumnsStrategy available_output_columns_stategy_)
: actions(std::move(actions_))
, available_output_columns_strategy(available_output_columns_stategy_)
{
initialize();
}
ActionsChainStep::ActionsChainStep(ActionsDAGPtr actions_,
AvailableOutputColumnsStrategy available_output_columns_stategy_,
ColumnsWithTypeAndName additional_output_columns_)
: actions(std::move(actions_))
, available_output_columns_strategy(available_output_columns_stategy_)
, additional_output_columns(std::move(additional_output_columns_))
{
initialize();
}
void ActionsChainStep::finalizeInputAndOutputColumns(const NameSet & child_input_columns)
{
child_required_output_columns_names.clear();
std::vector<const ActionsDAG::Node *> required_output_nodes;
auto child_input_columns_copy = child_input_columns;
std::unordered_set<std::string_view> output_nodes_names;
output_nodes_names.reserve(actions->getOutputs().size());
for (auto & output_node : actions->getOutputs())
output_nodes_names.insert(output_node->result_name);
for (const auto & node : actions->getNodes())
{
auto it = child_input_columns_copy.find(node.result_name);
if (it == child_input_columns_copy.end())
continue;
child_required_output_columns_names.insert(node.result_name);
required_output_nodes.push_back(&node);
child_input_columns_copy.erase(it);
}
child_required_output_columns_names.insert(node.result_name);
for (auto & required_output_node : required_output_nodes)
actions->addOrReplaceInOutputs(*required_output_node);
if (output_nodes_names.contains(node.result_name))
continue;
actions->getOutputs().push_back(&node);
output_nodes_names.insert(node.result_name);
}
actions->removeUnusedActions();
/// TODO: Analyzer fix ActionsDAG input and constant nodes with same name
@ -49,6 +74,7 @@ void ActionsChainStep::dump(WriteBuffer & buffer) const
for (const auto & column : additional_output_columns)
buffer << "Name " << column.name << " type " << column.type->getName() << '\n';
}
if (!child_required_output_columns_names.empty())
{
buffer << "Child required output columns " << boost::join(child_required_output_columns_names, ", ");
@ -121,13 +147,13 @@ void ActionsChain::finalize()
void ActionsChain::dump(WriteBuffer & buffer) const
{
size_t nodes_size = steps.size();
size_t steps_size = steps.size();
for (size_t i = 0; i < nodes_size; ++i)
for (size_t i = 0; i < steps_size; ++i)
{
const auto & node = steps[i];
const auto & step = steps[i];
buffer << "Step " << i << '\n';
node->dump(buffer);
step->dump(buffer);
buffer << '\n';
}
@ -137,6 +163,7 @@ String ActionsChain::dump() const
{
WriteBufferFromOwnString buffer;
dump(buffer);
return buffer.str();
}

23
src/Planner/ActionsChain.h
View File

@ -19,9 +19,9 @@ namespace ErrorCodes
* We want to reuse expr(id) from previous expressions step, and not recalculate it in projection.
* To do this we build a chain of all query action steps.
* For example:
* 1. before where
* 2. before order by
* 3. projection
* 1. Before where.
* 2. Before order by.
* 3. Projection.
*
* Initially root of chain is initialized with join tree query plan header.
* Each next chain step, must be initialized with previous step available output columns.
@ -56,22 +56,11 @@ public:
* If available output columns strategy is ALL_NODES, then available output columns initialized using actions dag nodes.
* If available output columns strategy is OUTPUT_NODES, then available output columns initialized using actions dag output nodes.
*/
explicit ActionsChainStep(ActionsDAGPtr actions_, AvailableOutputColumnsStrategy available_output_columns_stategy_ = AvailableOutputColumnsStrategy::ALL_NODES)
: actions(std::move(actions_))
, available_output_columns_strategy(available_output_columns_stategy_)
{
initialize();
}
explicit ActionsChainStep(ActionsDAGPtr actions_, AvailableOutputColumnsStrategy available_output_columns_stategy_ = AvailableOutputColumnsStrategy::ALL_NODES);
explicit ActionsChainStep(ActionsDAGPtr actions_,
AvailableOutputColumnsStrategy available_output_columns_stategy_,
ColumnsWithTypeAndName additional_output_columns_)
: actions(std::move(actions_))
, available_output_columns_strategy(available_output_columns_stategy_)
, additional_output_columns(std::move(additional_output_columns_))
{
initialize();
}
ColumnsWithTypeAndName additional_output_columns_);
/// Get actions
ActionsDAGPtr & getActions()
@ -225,7 +214,7 @@ public:
return steps.back()->getAvailableOutputColumns();
}
/// Get last step available output columns or throw exception if chain is empty
/// Get last step available output columns or null if chain is empty
const ColumnsWithTypeAndName * getLastStepAvailableOutputColumnsOrNull() const
{
if (steps.empty())

22
src/Planner/Planner.cpp
View File

@ -136,7 +136,7 @@ void addBuildSubqueriesForSetsStepIfNeeded(QueryPlan & query_plan, const SelectQ
PreparedSets::SubqueriesForSets subqueries_for_sets;
const auto & set_key_to_planner_set = planner_context->getRegisteredSets();
for (auto [key, planner_set] : set_key_to_planner_set)
for (const auto & [key, planner_set] : set_key_to_planner_set)
{
const auto subquery_node = planner_set.getSubqueryNode();
if (!subquery_node)
@ -188,19 +188,18 @@ Planner::Planner(const QueryTreeNodePtr & query_tree_,
ContextPtr context_)
: query_tree(query_tree_)
, select_query_options(select_query_options_)
, planner_context(std::make_shared<PlannerContext>(context_, std::make_shared<GlobalPlannerContext>()))
, planner_context(std::make_shared<PlannerContext>(std::move(context_), std::make_shared<GlobalPlannerContext>()))
{
initialize();
}
/// Initialize interpreter with query tree after query analysis phase and global planner context
Planner::Planner(const QueryTreeNodePtr & query_tree_,
const SelectQueryOptions & select_query_options_,
ContextPtr context_,
GlobalPlannerContextPtr global_planner_context_)
: query_tree(query_tree_)
, select_query_options(select_query_options_)
, planner_context(std::make_shared<PlannerContext>(context_, std::move(global_planner_context_)))
, planner_context(std::make_shared<PlannerContext>(std::move(context_), std::move(global_planner_context_)))
{
initialize();
}
@ -244,7 +243,7 @@ void Planner::initialize()
if (need_apply_query_settings)
updated_context->applySettingsChanges(query_node->getSettingsChanges());
/// Disable two-level aggregation due to version incompatibility.
/// Disable two-level aggregation due to version incompatibility
if (need_to_disable_two_level_aggregation)
{
updated_context->setSetting("group_by_two_level_threshold", Field(0));
@ -514,6 +513,7 @@ void Planner::buildQueryPlanIfNeeded()
{
const auto & having_analysis_result = expression_analysis_result.getHaving();
bool final = !query_node.isGroupByWithRollup() && !query_node.isGroupByWithCube();
having_executed = true;
auto totals_having_step = std::make_unique<TotalsHavingStep>(
query_plan.getCurrentDataStream(),
@ -575,7 +575,7 @@ void Planner::buildQueryPlanIfNeeded()
query_plan.getCurrentDataStream(),
window_description.full_sort_description,
settings.max_block_size,
0 /* LIMIT */,
0 /*limit*/,
SizeLimits(settings.max_rows_to_sort, settings.max_bytes_to_sort, settings.sort_overflow_mode),
settings.max_bytes_before_remerge_sort,
settings.remerge_sort_lowered_memory_bytes_ratio,
@ -664,16 +664,16 @@ void Planner::buildQueryPlanIfNeeded()
UInt64 partial_sorting_limit = 0;
/// Partial sort can be done if there is LIMIT, but no DISTINCT, LIMIT WITH TIES, LIMIT BY, ARRAY JOIN.
/// Partial sort can be done if there is LIMIT, but no DISTINCT, LIMIT WITH TIES, LIMIT BY, ARRAY JOIN
if (limit_length != 0 && !query_node.isDistinct() && !query_node.hasLimitBy() && !query_node.isLimitWithTies() &&
!query_has_array_join_in_join_tree && limit_length <= std::numeric_limits<UInt64>::max() - limit_offset)
{
partial_sorting_limit = limit_length + limit_offset;
}
const Settings & settings = planner_context->getQueryContext()->getSettingsRef();
const Settings & settings = query_context->getSettingsRef();
/// Merge the sorted blocks.
/// Merge the sorted blocks
auto sorting_step = std::make_unique<SortingStep>(
query_plan.getCurrentDataStream(),
sort_description,
@ -683,7 +683,7 @@ void Planner::buildQueryPlanIfNeeded()
settings.max_bytes_before_remerge_sort,
settings.remerge_sort_lowered_memory_bytes_ratio,
settings.max_bytes_before_external_sort,
planner_context->getQueryContext()->getTempDataOnDisk(),
query_context->getTempDataOnDisk(),
settings.min_free_disk_space_for_temporary_data,
settings.optimize_sorting_by_input_stream_properties);
@ -814,7 +814,7 @@ void Planner::buildQueryPlanIfNeeded()
if (query_node.hasLimit())
{
const Settings & settings = planner_context->getQueryContext()->getSettingsRef();
const Settings & settings = query_context->getSettingsRef();
bool always_read_till_end = settings.exact_rows_before_limit;
bool limit_with_ties = query_node.isLimitWithTies();

2
src/Planner/Planner.h
View File

@ -24,7 +24,7 @@ public:
const SelectQueryOptions & select_query_options_,
ContextPtr context_);
/// Initialize interpreter with query tree after query analysis phase and global planner context
/// Initialize planner with query tree after query analysis phase and global planner context
Planner(const QueryTreeNodePtr & query_tree_,
const SelectQueryOptions & select_query_options_,
ContextPtr context_,

17
src/Planner/PlannerActionsVisitor.cpp
View File

@ -223,7 +223,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
return visitQueryOrUnion(node);
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
"Expected only column, constant or function node. Actual {}",
"Expected column, constant, function, query or union node. Actual {}",
node->formatASTForErrorMessage());
}
@ -285,13 +285,16 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
"Lambda {} is not resolved during query analysis",
lambda_node.formatASTForErrorMessage());
auto & lambda_arguments_nodes = lambda_node.getArguments().getNodes();
size_t lambda_arguments_nodes_size = lambda_arguments_nodes.size();
NamesAndTypesList lambda_arguments_names_and_types;
for (auto & lambda_node_argument : lambda_node.getArguments().getNodes())
for (size_t i = 0; i < lambda_arguments_nodes_size; ++i)
{
auto lambda_argument_name = lambda_node_argument->getName();
auto lambda_argument_type = lambda_node_argument->getResultType();
lambda_arguments_names_and_types.emplace_back(lambda_argument_name, lambda_argument_type);
const auto & lambda_argument_name = lambda_node.getArgumentNames().at(i);
auto lambda_argument_type = lambda_arguments_nodes[i]->getResultType();
lambda_arguments_names_and_types.emplace_back(lambda_argument_name, std::move(lambda_argument_type));
}
size_t previous_scope_node_actions_stack_index = actions_stack.size() - 1;
@ -332,7 +335,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
if (level == actions_stack.size())
--level;
actions_stack[level].addFunctionIfNecessary(lambda_node_name, lambda_children, function_capture);
actions_stack[level].addFunctionIfNecessary(lambda_node_name, std::move(lambda_children), std::move(function_capture));
size_t actions_stack_size = actions_stack.size();
for (size_t i = level + 1; i < actions_stack_size; ++i)
@ -350,7 +353,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::ma
auto in_second_argument = function_node.getArguments().getNodes().at(1);
auto set_key = planner_context->createSetKey(in_second_argument);
auto planner_set = planner_context->getSetOrThrow(set_key);
const auto & planner_set = planner_context->getSetOrThrow(set_key);
ColumnWithTypeAndName column;
column.name = set_key;

6
src/Planner/PlannerActionsVisitor.h
View File

@ -24,11 +24,11 @@ using PlannerContextPtr = std::shared_ptr<PlannerContext>;
* 1. Table expression data for table expression nodes is collected in planner context.
* For column node, that has column table expression source, identifier for column name in table expression data
* is used as action dag node name.
* 2. Sets for IN functions are already collected in planner global context.
* 2. Sets for IN functions are already collected in planner context.
*
* During actions build, there is special handling for following functions:
* 1. Aggregate functions are added in actions dag as INPUT nodes. Aggregate functions arguments are not added.
* 2. For function `in` and its variants, already collected sets from global context are used.
* 2. For function `in` and its variants, already collected sets from planner context are used.
*/
class PlannerActionsVisitor
{
@ -45,7 +45,7 @@ private:
const PlannerContextPtr planner_context;
};
/** Calculate query tree expression node name action dag name and add them into node to name map.
/** Calculate query tree expression node action dag name and add them into node to name map.
* If node exists in map, name from map is used.
*
* For column node column node identifier from planner context is used.

54
src/Planner/PlannerAggregation.cpp
View File

@ -81,7 +81,7 @@ public:
if (it == aggegation_key_to_index.end())
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Argument of GROUPING function {} is not a part of GROUP BY clause",
argument->formatASTForErrorMessage());
argument->formatASTForErrorMessage());
arguments_indexes.push_back(it->second);
}
@ -98,30 +98,30 @@ public:
{
auto grouping_ordinary_function = std::make_shared<FunctionGroupingOrdinary>(arguments_indexes, force_grouping_standard_compatibility);
auto grouping_ordinary_function_adaptor = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_ordinary_function));
function_node->resolveAsFunction(grouping_ordinary_function_adaptor, std::make_shared<DataTypeUInt64>());
function_node->resolveAsFunction(std::move(grouping_ordinary_function_adaptor), std::make_shared<DataTypeUInt64>());
break;
}
case GroupByKind::ROLLUP:
{
auto grouping_ordinary_function = std::make_shared<FunctionGroupingForRollup>(arguments_indexes, aggregation_keys_size, force_grouping_standard_compatibility);
auto grouping_ordinary_function_adaptor = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_ordinary_function));
function_node->resolveAsFunction(grouping_ordinary_function_adaptor, std::make_shared<DataTypeUInt64>());
auto grouping_rollup_function = std::make_shared<FunctionGroupingForRollup>(arguments_indexes, aggregation_keys_size, force_grouping_standard_compatibility);
auto grouping_rollup_function_adaptor = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_rollup_function));
function_node->resolveAsFunction(std::move(grouping_rollup_function_adaptor), std::make_shared<DataTypeUInt64>());
function_node->getArguments().getNodes().push_back(std::move(grouping_set_argument_column));
break;
}
case GroupByKind::CUBE:
{
auto grouping_ordinary_function = std::make_shared<FunctionGroupingForCube>(arguments_indexes, aggregation_keys_size, force_grouping_standard_compatibility);
auto grouping_ordinary_function_adaptor = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_ordinary_function));
function_node->resolveAsFunction(grouping_ordinary_function_adaptor, std::make_shared<DataTypeUInt64>());
auto grouping_cube_function = std::make_shared<FunctionGroupingForCube>(arguments_indexes, aggregation_keys_size, force_grouping_standard_compatibility);
auto grouping_cube_function_adaptor = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_cube_function));
function_node->resolveAsFunction(std::move(grouping_cube_function_adaptor), std::make_shared<DataTypeUInt64>());
function_node->getArguments().getNodes().push_back(std::move(grouping_set_argument_column));
break;
}
case GroupByKind::GROUPING_SETS:
{
auto grouping_grouping_sets_function = std::make_shared<FunctionGroupingForGroupingSets>(arguments_indexes, grouping_sets_keys_indices, force_grouping_standard_compatibility);
auto grouping_ordinary_function_adaptor = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_grouping_sets_function));
function_node->resolveAsFunction(grouping_ordinary_function_adaptor, std::make_shared<DataTypeUInt64>());
auto grouping_grouping_sets_function_adaptor = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(grouping_grouping_sets_function));
function_node->resolveAsFunction(std::move(grouping_grouping_sets_function_adaptor), std::make_shared<DataTypeUInt64>());
function_node->getArguments().getNodes().push_back(std::move(grouping_set_argument_column));
break;
}
@ -147,8 +147,17 @@ void resolveGroupingFunctions(QueryTreeNodePtr & node,
const GroupingSetsParamsList & grouping_sets_parameters_list,
const PlannerContext & planner_context)
{
auto & query_node_typed = node->as<QueryNode &>();
GroupingFunctionResolveVisitor visitor(group_by_kind, aggregation_keys, grouping_sets_parameters_list, planner_context);
visitor.visit(node);
if (query_node_typed.hasHaving())
visitor.visit(query_node_typed.getHaving());
if (query_node_typed.hasOrderBy())
visitor.visit(query_node_typed.getOrderByNode());
visitor.visit(query_node_typed.getProjectionNode());
}
}
@ -168,26 +177,7 @@ void resolveGroupingFunctions(QueryTreeNodePtr & query_node,
else if (query_node_typed.isGroupByWithGroupingSets())
group_by_kind = GroupByKind::GROUPING_SETS;
if (query_node_typed.hasHaving())
{
resolveGroupingFunctions(query_node_typed.getHaving(),
group_by_kind,
aggregation_keys,
grouping_sets_parameters_list,
planner_context);
}
resolveGroupingFunctions(query_node_typed.getOrderByNode(),
group_by_kind,
aggregation_keys,
grouping_sets_parameters_list,
planner_context);
resolveGroupingFunctions(query_node_typed.getProjectionNode(),
group_by_kind,
aggregation_keys,
grouping_sets_parameters_list,
planner_context);
resolveGroupingFunctions(query_node, group_by_kind, aggregation_keys, grouping_sets_parameters_list, planner_context);
}
AggregateDescriptions extractAggregateDescriptions(const QueryTreeNodes & aggregate_function_nodes, const PlannerContext & planner_context)
@ -225,7 +215,7 @@ AggregateDescriptions extractAggregateDescriptions(const QueryTreeNodes & aggreg
aggregate_description.argument_names.emplace_back(std::move(argument_node_name));
}
aggregate_description.column_name = node_name;
aggregate_description.column_name = std::move(node_name);
aggregate_descriptions.push_back(std::move(aggregate_description));
}

4
src/Planner/PlannerContext.h
View File

@ -66,11 +66,13 @@ public:
, subquery_node(std::move(subquery_node_))
{}
/// Get set
const SetPtr & getSet() const
{
return set;
}
/// Get subquery node
const QueryTreeNodePtr & getSubqueryNode() const
{
return subquery_node;
@ -135,7 +137,7 @@ public:
*/
TableExpressionData * getTableExpressionDataOrNull(const QueryTreeNodePtr & table_expression_node);
/// Get table expression node to data map read only map
/// Get table expression node to data read only map
const std::unordered_map<QueryTreeNodePtr, TableExpressionData> & getTableExpressionNodeToData() const
{
return table_expression_node_to_data;

4
tests/queries/0_stateless/02342_analyzer_compound_types.sql
View File

@ -101,7 +101,7 @@ CREATE TABLE test_table
value Array(Tuple(value_0_level_0 Tuple(value_0_level_1 String, value_1_level_1 String), value_1_level_0 String))
) ENGINE=MergeTree ORDER BY id;
INSERT INTO test_table VALUES (0, [('value_0_level_1', 'value_1_level_1')], array('value_1_level_0'));
INSERT INTO test_table VALUES (0, [('value_0_level_1', 'value_1_level_1')], ['value_1_level_0']);
DESCRIBE (SELECT * FROM test_table);
SELECT * FROM test_table;
@ -152,7 +152,7 @@ CREATE TABLE test_table
value Nested (value_0_level_0 Nested(value_0_level_1 String, value_1_level_1 String), value_1_level_0 String)
) ENGINE=MergeTree ORDER BY id;
INSERT INTO test_table VALUES (0, [[('value_0_level_1', 'value_1_level_1')]], array('value_1_level_0'));
INSERT INTO test_table VALUES (0, [[('value_0_level_1', 'value_1_level_1')]], ['value_1_level_0']);
DESCRIBE (SELECT * FROM test_table);
SELECT * FROM test_table;