thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into Fix_endpoint_for_azureblobstorage

Browse Source
This commit is contained in:
Smita Kulkarni 2024-02-29 15:05:17 +01:00
commit bb7453a339
26 changed files with 1027 additions and 338 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitmessage
View File

@ -16,6 +16,7 @@
#ci_set_reduced
#ci_set_arm
#ci_set_integration
#ci_set_analyzer
## To run specified job in CI:
#job_<JOB NAME>

186
src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp Normal file
View File

@ -0,0 +1,186 @@
#include <Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <Analyzer/ArrayJoinNode.h>
#include <Analyzer/ColumnNode.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/QueryNode.h>
#include <Analyzer/TableNode.h>
#include <Analyzer/UnionNode.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
namespace
{
/// Try to eliminate min/max/any/anyLast.
class EliminateFunctionVisitor : public InDepthQueryTreeVisitorWithContext<EliminateFunctionVisitor>
{
public:
using Base = InDepthQueryTreeVisitorWithContext<EliminateFunctionVisitor>;
using Base::Base;
using GroupByKeysStack = std::vector<QueryTreeNodePtrWithHashSet>;
void enterImpl(QueryTreeNodePtr & node)
{
if (!getSettings().optimize_aggregators_of_group_by_keys)
return;
/// Collect group by keys.
auto * query_node = node->as<QueryNode>();
if (!query_node)
return;
if (!query_node->hasGroupBy())
{
group_by_keys_stack.push_back({});
}
else if (query_node->isGroupByWithTotals() || query_node->isGroupByWithCube() || query_node->isGroupByWithRollup())
{
/// Keep aggregator if group by is with totals/cube/rollup.
group_by_keys_stack.push_back({});
}
else
{
QueryTreeNodePtrWithHashSet group_by_keys;
for (auto & group_key : query_node->getGroupBy().getNodes())
{
/// For grouping sets case collect only keys that are presented in every set.
if (auto * list = group_key->as<ListNode>())
{
QueryTreeNodePtrWithHashSet common_keys_set;
for (auto & group_elem : list->getNodes())
{
if (group_by_keys.contains(group_elem))
common_keys_set.insert(group_elem);
}
group_by_keys = std::move(common_keys_set);
}
else
{
group_by_keys.insert(group_key);
}
}
group_by_keys_stack.push_back(std::move(group_by_keys));
}
}
/// Now we visit all nodes in QueryNode, we should remove group_by_keys from stack.
void leaveImpl(QueryTreeNodePtr & node)
{
if (!getSettings().optimize_aggregators_of_group_by_keys)
return;
if (node->getNodeType() == QueryTreeNodeType::FUNCTION)
{
if (aggregationCanBeEliminated(node, group_by_keys_stack.back()))
node = node->as<FunctionNode>()->getArguments().getNodes()[0];
}
else if (node->getNodeType() == QueryTreeNodeType::QUERY)
{
group_by_keys_stack.pop_back();
}
}
static bool needChildVisit(VisitQueryTreeNodeType & parent [[maybe_unused]], VisitQueryTreeNodeType & child)
{
/// Skip ArrayJoin.
return !child->as<ArrayJoinNode>();
}
private:
struct NodeWithInfo
{
QueryTreeNodePtr node;
bool parents_are_only_deterministic = false;
};
bool aggregationCanBeEliminated(QueryTreeNodePtr & node, const QueryTreeNodePtrWithHashSet & group_by_keys)
{
if (group_by_keys.empty())
return false;
auto * function = node->as<FunctionNode>();
if (!function || !function->isAggregateFunction())
return false;
if (!(function->getFunctionName() == "min"
|| function->getFunctionName() == "max"
|| function->getFunctionName() == "any"
|| function->getFunctionName() == "anyLast"))
return false;
std::vector<NodeWithInfo> candidates;
auto & function_arguments = function->getArguments().getNodes();
if (function_arguments.size() != 1)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected a single argument of function '{}' but received {}", function->getFunctionName(), function_arguments.size());
if (!function->getResultType()->equals(*function_arguments[0]->getResultType()))
return false;
candidates.push_back({ function_arguments[0], true });
/// Using DFS we traverse function tree and try to find if it uses other keys as function arguments.
while (!candidates.empty())
{
auto [candidate, parents_are_only_deterministic] = candidates.back();
candidates.pop_back();
bool found = group_by_keys.contains(candidate);
switch (candidate->getNodeType())
{
case QueryTreeNodeType::FUNCTION:
{
auto * func = candidate->as<FunctionNode>();
auto & arguments = func->getArguments().getNodes();
if (arguments.empty())
return false;
if (!found)
{
bool is_deterministic_function = parents_are_only_deterministic &&
func->getFunctionOrThrow()->isDeterministicInScopeOfQuery();
for (auto it = arguments.rbegin(); it != arguments.rend(); ++it)
candidates.push_back({ *it, is_deterministic_function });
}
break;
}
case QueryTreeNodeType::COLUMN:
if (!found)
return false;
break;
case QueryTreeNodeType::CONSTANT:
if (!parents_are_only_deterministic)
return false;
break;
default:
return false;
}
}
return true;
}
GroupByKeysStack group_by_keys_stack;
};
}
void AggregateFunctionOfGroupByKeysPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
EliminateFunctionVisitor eliminator(context);
eliminator.visit(query_tree_node);
}
};

28
src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.h Normal file
View File

@ -0,0 +1,28 @@
#pragma once
#include <Analyzer/IQueryTreePass.h>
namespace DB
{
/** Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section.
*
* Example: SELECT max(column) FROM table GROUP BY column;
* Result: SELECT column FROM table GROUP BY column;
*/
class AggregateFunctionOfGroupByKeysPass final : public IQueryTreePass
{
public:
String getName() override { return "AggregateFunctionOfGroupByKeys"; }
String getDescription() override
{
return "Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section.";
}
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};
}

2
src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp
View File

@ -92,7 +92,7 @@ private:
if (!found)
{
bool is_deterministic_function = parents_are_only_deterministic &&
function->getFunctionOrThrow()->isDeterministicInScopeOfQuery();
func->getFunctionOrThrow()->isDeterministicInScopeOfQuery();
for (auto it = arguments.rbegin(); it != arguments.rend(); ++it)
candidates.push_back({ *it, is_deterministic_function });
}

5
src/Analyzer/Passes/QueryAnalysisPass.cpp
View File

@ -6651,6 +6651,7 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table
if (column_default && column_default->kind == ColumnDefaultKind::Alias)
{
auto alias_expression = buildQueryTree(column_default->expression, scope.context);
alias_expression = buildCastFunction(alias_expression, column_name_and_type.type, scope.context, false /*resolve*/);
auto column_node = std::make_shared<ColumnNode>(column_name_and_type, std::move(alias_expression), table_expression_node);
column_name_to_column_node.emplace(column_name_and_type.name, column_node);
alias_columns_to_resolve.emplace_back(column_name_and_type.name, column_node);
@ -6683,9 +6684,7 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table
alias_column_resolve_scope,
false /*allow_lambda_expression*/,
false /*allow_table_expression*/);
auto & resolved_expression = alias_column_to_resolve->getExpression();
if (!resolved_expression->getResultType()->equals(*alias_column_to_resolve->getResultType()))
resolved_expression = buildCastFunction(resolved_expression, alias_column_to_resolve->getResultType(), scope.context, true);
column_name_to_column_node = std::move(alias_column_resolve_scope.column_name_to_column_node);
column_name_to_column_node[alias_column_to_resolve_name] = alias_column_to_resolve;
}

5
src/Analyzer/QueryTreePassManager.cpp
View File

@ -46,6 +46,7 @@
#include <Analyzer/Passes/CrossToInnerJoinPass.h>
#include <Analyzer/Passes/ShardNumColumnToFunctionPass.h>
#include <Analyzer/Passes/ConvertQueryToCNFPass.h>
#include <Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.h>
#include <Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.h>
@ -164,7 +165,6 @@ private:
/** ClickHouse query tree pass manager.
*
* TODO: Support setting optimize_aggregators_of_group_by_keys.
* TODO: Support setting optimize_monotonous_functions_in_order_by.
* TODO: Add optimizations based on function semantics. Example: SELECT * FROM test_table WHERE id != id. (id is not nullable column).
*/
@ -264,6 +264,9 @@ void addQueryTreePasses(QueryTreePassManager & manager)
manager.addPass(std::make_unique<RewriteArrayExistsToHasPass>());
manager.addPass(std::make_unique<NormalizeCountVariantsPass>());
/// should before AggregateFunctionsArithmericOperationsPass
manager.addPass(std::make_unique<AggregateFunctionOfGroupByKeysPass>());
manager.addPass(std::make_unique<AggregateFunctionsArithmericOperationsPass>());
manager.addPass(std::make_unique<UniqInjectiveFunctionsEliminationPass>());
manager.addPass(std::make_unique<OptimizeGroupByFunctionKeysPass>());

7
src/Interpreters/getHeaderForProcessingStage.cpp
View File

@ -121,12 +121,7 @@ Block getHeaderForProcessingStage(
auto & table_expression_data = query_info.planner_context->getTableExpressionDataOrThrow(left_table_expression);
const auto & query_context = query_info.planner_context->getQueryContext();
NamesAndTypes columns;
const auto & column_name_to_column = table_expression_data.getColumnNameToColumn();
for (const auto & column_name : table_expression_data.getSelectedColumnsNames())
columns.push_back(column_name_to_column.at(column_name));
auto columns = table_expression_data.getColumns();
auto new_query_node = buildSubqueryToReadColumnsFromTableExpression(columns, left_table_expression, query_context);
query = new_query_node->toAST();
}

166
src/Planner/CollectTableExpressionData.cpp
View File

@ -29,13 +29,34 @@ namespace
class CollectSourceColumnsVisitor : public InDepthQueryTreeVisitor<CollectSourceColumnsVisitor>
{
public:
explicit CollectSourceColumnsVisitor(PlannerContextPtr & planner_context_, bool keep_alias_columns_ = true)
explicit CollectSourceColumnsVisitor(PlannerContext & planner_context_)
: planner_context(planner_context_)
, keep_alias_columns(keep_alias_columns_)
{}
void visitImpl(QueryTreeNodePtr & node)
{
/// Special case for USING clause which contains references to ALIAS columns.
/// We can not modify such ColumnNode.
if (auto * join_node = node->as<JoinNode>())
{
if (!join_node->isUsingJoinExpression())
return;
auto & using_list = join_node->getJoinExpression()->as<ListNode&>();
for (auto & using_element : using_list)
{
auto & column_node = using_element->as<ColumnNode&>();
/// This list contains column nodes from left and right tables.
auto & columns_from_subtrees = column_node.getExpressionOrThrow()->as<ListNode&>().getNodes();
/// Visit left table column node.
visitUsingColumn(columns_from_subtrees[0]);
/// Visit right table column node.
visitUsingColumn(columns_from_subtrees[1]);
}
return;
}
auto * column_node = node->as<ColumnNode>();
if (!column_node)
return;
@ -51,55 +72,22 @@ public:
/// JOIN using expression
if (column_node->hasExpression() && column_source_node_type == QueryTreeNodeType::JOIN)
{
auto & columns_from_subtrees = column_node->getExpression()->as<ListNode &>().getNodes();
if (columns_from_subtrees.size() != 2)
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Expected two columns in JOIN using expression for column {}", column_node->dumpTree());
visit(columns_from_subtrees[0]);
visit(columns_from_subtrees[1]);
return;
}
auto & table_expression_data = planner_context->getOrCreateTableExpressionData(column_source_node);
auto & table_expression_data = planner_context.getOrCreateTableExpressionData(column_source_node);
if (isAliasColumn(node))
if (column_node->hasExpression() && column_source_node_type != QueryTreeNodeType::ARRAY_JOIN)
{
/// Column is an ALIAS column with expression
/// Replace ALIAS column with expression
bool column_already_exists = table_expression_data.hasColumn(column_node->getColumnName());
if (!column_already_exists)
{
CollectSourceColumnsVisitor visitor_for_alias_column(planner_context);
/// While we are processing expression of ALIAS columns we should not add source columns to selected.
/// See also comment for `select_added_columns`
visitor_for_alias_column.select_added_columns = false;
visitor_for_alias_column.keep_alias_columns = keep_alias_columns;
visitor_for_alias_column.visit(column_node->getExpression());
if (!keep_alias_columns)
{
/// For PREWHERE we can just replace ALIAS column with it's expression,
/// because ActionsDAG for PREWHERE applied right on top of table expression
/// and cannot affect subqueries or other table expressions.
node = column_node->getExpression();
return;
}
auto column_identifier = planner_context->getGlobalPlannerContext()->createColumnIdentifier(node);
ActionsDAGPtr alias_column_actions_dag = std::make_shared<ActionsDAG>();
PlannerActionsVisitor actions_visitor(planner_context, false);
auto outputs = actions_visitor.visit(alias_column_actions_dag, column_node->getExpression());
if (outputs.size() != 1)
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Expected single output in actions dag for alias column {}. Actual {}", column_node->dumpTree(), outputs.size());
const auto & column_name = column_node->getColumnName();
const auto & alias_node = alias_column_actions_dag->addAlias(*outputs[0], column_name);
alias_column_actions_dag->addOrReplaceInOutputs(alias_node);
table_expression_data.addAliasColumn(column_node->getColumn(), column_identifier, alias_column_actions_dag, select_added_columns);
auto column_identifier = planner_context.getGlobalPlannerContext()->createColumnIdentifier(node);
table_expression_data.addAliasColumnName(column_node->getColumnName(), column_identifier);
}
node = column_node->getExpression();
visitImpl(node);
return;
}
@ -114,58 +102,45 @@ public:
bool column_already_exists = table_expression_data.hasColumn(column_node->getColumnName());
if (column_already_exists)
{
/// Column may be added when we collected data for ALIAS column
/// But now we see it directly in the query, so make sure it's marked as selected
if (select_added_columns)
table_expression_data.markSelectedColumn(column_node->getColumnName());
return;
auto column_identifier = planner_context.getGlobalPlannerContext()->createColumnIdentifier(node);
table_expression_data.addColumn(column_node->getColumn(), column_identifier);
}
static bool needChildVisit(const QueryTreeNodePtr & parent, const QueryTreeNodePtr & child_node)
{
if (auto * join_node = parent->as<JoinNode>())
{
if (join_node->getJoinExpression() == child_node && join_node->isUsingJoinExpression())
return false;
}
auto column_identifier = planner_context->getGlobalPlannerContext()->createColumnIdentifier(node);
table_expression_data.addColumn(column_node->getColumn(), column_identifier, select_added_columns);
}
static bool isAliasColumn(const QueryTreeNodePtr & node)
{
const auto * column_node = node->as<ColumnNode>();
if (!column_node || !column_node->hasExpression())
return false;
const auto & column_source = column_node->getColumnSourceOrNull();
if (!column_source)
return false;
return column_source->getNodeType() != QueryTreeNodeType::JOIN &&
column_source->getNodeType() != QueryTreeNodeType::ARRAY_JOIN;
}
static bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node)
{
auto child_node_type = child_node->getNodeType();
return !(child_node_type == QueryTreeNodeType::QUERY ||
child_node_type == QueryTreeNodeType::UNION ||
isAliasColumn(parent_node));
}
void setKeepAliasColumns(bool keep_alias_columns_)
{
keep_alias_columns = keep_alias_columns_;
return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION);
}
private:
PlannerContextPtr & planner_context;
/// Replace ALIAS columns with their expressions or register them in table expression data.
/// Usually we can replace them when we build some "local" actions DAG
/// (for example Row Policy or PREWHERE) that is applied on top of the table expression.
/// In other cases, we keep ALIAS columns as ColumnNode with an expression child node,
/// and handle them in the Planner by inserting ActionsDAG to compute them after reading from storage.
bool keep_alias_columns = true;
void visitUsingColumn(QueryTreeNodePtr & node)
{
auto & column_node = node->as<ColumnNode&>();
if (column_node.hasExpression())
{
auto & table_expression_data = planner_context.getOrCreateTableExpressionData(column_node.getColumnSource());
bool column_already_exists = table_expression_data.hasColumn(column_node.getColumnName());
if (column_already_exists)
return;
/// Flag `select_added_columns` indicates if we should mark column as explicitly selected.
/// For example, for table with columns (a Int32, b ALIAS a+1) and query SELECT b FROM table
/// Column `b` is selected explicitly by user, but not `a` (that is also read though).
/// Distinguishing such columns is important for checking access rights for ALIAS columns.
bool select_added_columns = true;
auto column_identifier = planner_context.getGlobalPlannerContext()->createColumnIdentifier(node);
table_expression_data.addAliasColumnName(column_node.getColumnName(), column_identifier);
visitImpl(column_node.getExpressionOrThrow());
}
else
visitImpl(node);
}
PlannerContext & planner_context;
};
class CollectPrewhereTableExpressionVisitor : public ConstInDepthQueryTreeVisitor<CollectPrewhereTableExpressionVisitor>
@ -299,7 +274,7 @@ void collectTableExpressionData(QueryTreeNodePtr & query_node, PlannerContextPtr
}
}
CollectSourceColumnsVisitor collect_source_columns_visitor(planner_context);
CollectSourceColumnsVisitor collect_source_columns_visitor(*planner_context);
for (auto & node : query_node_typed.getChildren())
{
if (!node || node == query_node_typed.getPrewhere())
@ -325,26 +300,21 @@ void collectTableExpressionData(QueryTreeNodePtr & query_node, PlannerContextPtr
}
auto & table_expression_data = planner_context->getOrCreateTableExpressionData(prewhere_table_expression);
const auto & read_column_names = table_expression_data.getColumnNames();
NameSet required_column_names_without_prewhere(read_column_names.begin(), read_column_names.end());
const auto & selected_column_names = table_expression_data.getSelectedColumnsNames();
required_column_names_without_prewhere.insert(selected_column_names.begin(), selected_column_names.end());
const auto & column_names = table_expression_data.getColumnNames();
NameSet required_column_names_without_prewhere(column_names.begin(), column_names.end());
collect_source_columns_visitor.setKeepAliasColumns(false);
collect_source_columns_visitor.visit(query_node_typed.getPrewhere());
auto prewhere_actions_dag = std::make_shared<ActionsDAG>();
QueryTreeNodePtr query_tree_node = query_node_typed.getPrewhere();
PlannerActionsVisitor visitor(planner_context, false /*use_column_identifier_as_action_node_name*/);
auto expression_nodes = visitor.visit(prewhere_actions_dag, query_tree_node);
auto expression_nodes = visitor.visit(prewhere_actions_dag, query_node_typed.getPrewhere());
if (expression_nodes.size() != 1)
throw Exception(ErrorCodes::ILLEGAL_PREWHERE,
"Invalid PREWHERE. Expected single boolean expression. In query {}",
query_node->formatASTForErrorMessage());
prewhere_actions_dag->getOutputs().push_back(expression_nodes.back());
prewhere_actions_dag->getOutputs().push_back(expression_nodes[0]);
for (const auto & prewhere_input_node : prewhere_actions_dag->getInputs())
if (required_column_names_without_prewhere.contains(prewhere_input_node->result_name))
@ -354,9 +324,9 @@ void collectTableExpressionData(QueryTreeNodePtr & query_node, PlannerContextPtr
}
}
void collectSourceColumns(QueryTreeNodePtr & expression_node, PlannerContextPtr & planner_context, bool keep_alias_columns)
void collectSourceColumns(QueryTreeNodePtr & expression_node, PlannerContextPtr & planner_context)
{
CollectSourceColumnsVisitor collect_source_columns_visitor(planner_context, keep_alias_columns);
CollectSourceColumnsVisitor collect_source_columns_visitor(*planner_context);
collect_source_columns_visitor.visit(expression_node);
}

2
src/Planner/CollectTableExpressionData.h
View File

@ -19,6 +19,6 @@ void collectTableExpressionData(QueryTreeNodePtr & query_node, PlannerContextPtr
*
* ALIAS table column nodes are registered in table expression data and replaced in query tree with inner alias expression.
*/
void collectSourceColumns(QueryTreeNodePtr & expression_node, PlannerContextPtr & planner_context, bool keep_alias_columns = true);
void collectSourceColumns(QueryTreeNodePtr & expression_node, PlannerContextPtr & planner_context);
}

5
src/Planner/PlannerActionsVisitor.cpp
View File

@ -451,7 +451,6 @@ private:
std::unordered_map<QueryTreeNodePtr, std::string> node_to_node_name;
const PlannerContextPtr planner_context;
ActionNodeNameHelper action_node_name_helper;
bool use_column_identifier_as_action_node_name;
};
PlannerActionsVisitorImpl::PlannerActionsVisitorImpl(ActionsDAGPtr actions_dag,
@ -459,7 +458,6 @@ PlannerActionsVisitorImpl::PlannerActionsVisitorImpl(ActionsDAGPtr actions_dag,
bool use_column_identifier_as_action_node_name_)
: planner_context(planner_context_)
, action_node_name_helper(node_to_node_name, *planner_context, use_column_identifier_as_action_node_name_)
, use_column_identifier_as_action_node_name(use_column_identifier_as_action_node_name_)
{
actions_stack.emplace_back(std::move(actions_dag), nullptr);
}
@ -505,8 +503,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
{
auto column_node_name = action_node_name_helper.calculateActionNodeName(node);
const auto & column_node = node->as<ColumnNode &>();
if (column_node.hasExpression() && !use_column_identifier_as_action_node_name)
return visitImpl(column_node.getExpression());
Int64 actions_stack_size = static_cast<Int64>(actions_stack.size() - 1);
for (Int64 i = actions_stack_size; i >= 0; --i)
{

96
src/Planner/PlannerJoinTree.cpp
View File

@ -86,7 +86,7 @@ namespace
/// Check if current user has privileges to SELECT columns from table
/// Throws an exception if access to any column from `column_names` is not granted
/// If `column_names` is empty, check access to any columns and return names of accessible columns
NameSet checkAccessRights(const TableNode & table_node, const Names & column_names, const ContextPtr & query_context)
NameSet checkAccessRights(const TableNode & table_node, Names & column_names, const ContextPtr & query_context)
{
/// StorageDummy is created on preliminary stage, ignore access check for it.
if (typeid_cast<const StorageDummy *>(table_node.getStorage().get()))
@ -353,7 +353,9 @@ void prepareBuildQueryPlanForTableExpression(const QueryTreeNodePtr & table_expr
NameSet columns_names_allowed_to_select;
if (table_node)
{
const auto & column_names_with_aliases = table_expression_data.getSelectedColumnsNames();
auto column_names_with_aliases = columns_names;
const auto & alias_columns_names = table_expression_data.getAliasColumnsNames();
column_names_with_aliases.insert(column_names_with_aliases.end(), alias_columns_names.begin(), alias_columns_names.end());
columns_names_allowed_to_select = checkAccessRights(*table_node, column_names_with_aliases, query_context);
}
@ -862,28 +864,6 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
max_block_size,
max_streams);
const auto & alias_column_expressions = table_expression_data.getAliasColumnExpressions();
if (!alias_column_expressions.empty() && query_plan.isInitialized() && from_stage == QueryProcessingStage::FetchColumns)
{
ActionsDAGPtr merged_alias_columns_actions_dag = std::make_shared<ActionsDAG>(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
ActionsDAG::NodeRawConstPtrs action_dag_outputs = merged_alias_columns_actions_dag->getInputs();
for (const auto & [column_name, alias_column_actions_dag] : alias_column_expressions)
{
const auto & current_outputs = alias_column_actions_dag->getOutputs();
action_dag_outputs.insert(action_dag_outputs.end(), current_outputs.begin(), current_outputs.end());
merged_alias_columns_actions_dag->mergeNodes(std::move(*alias_column_actions_dag));
}
for (const auto * output_node : action_dag_outputs)
merged_alias_columns_actions_dag->addOrReplaceInOutputs(*output_node);
merged_alias_columns_actions_dag->removeUnusedActions(false);
auto alias_column_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(merged_alias_columns_actions_dag));
alias_column_step->setStepDescription("Compute alias columns");
query_plan.addStep(std::move(alias_column_step));
}
for (const auto & filter_info_and_description : where_filters)
{
const auto & [filter_info, description] = filter_info_and_description;
@ -927,8 +907,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
else
{
/// Create step which reads from empty source if storage has no data.
const auto & column_names = table_expression_data.getSelectedColumnsNames();
auto source_header = storage_snapshot->getSampleBlockForColumns(column_names);
auto source_header = storage_snapshot->getSampleBlockForColumns(table_expression_data.getColumnNames());
Pipe pipe(std::make_shared<NullSource>(source_header));
auto read_from_pipe = std::make_unique<ReadFromPreparedSource>(std::move(pipe));
read_from_pipe->setStepDescription("Read from NullSource");
@ -1045,6 +1024,57 @@ void joinCastPlanColumnsToNullable(QueryPlan & plan_to_add_cast, PlannerContextP
plan_to_add_cast.addStep(std::move(cast_join_columns_step));
}
/// Actions to calculate table columns that have a functional representation (ALIASes and subcolumns)
/// and used in USING clause of JOIN expression.
struct UsingAliasKeyActions
{
UsingAliasKeyActions(
const ColumnsWithTypeAndName & left_plan_output_columns,
const ColumnsWithTypeAndName & right_plan_output_columns
)
: left_alias_columns_keys(std::make_shared<ActionsDAG>(left_plan_output_columns))
, right_alias_columns_keys(std::make_shared<ActionsDAG>(right_plan_output_columns))
{}
void addLeftColumn(QueryTreeNodePtr & node, const ColumnsWithTypeAndName & plan_output_columns, const PlannerContextPtr & planner_context)
{
addColumnImpl(left_alias_columns_keys, node, plan_output_columns, planner_context);
}
void addRightColumn(QueryTreeNodePtr & node, const ColumnsWithTypeAndName & plan_output_columns, const PlannerContextPtr & planner_context)
{
addColumnImpl(right_alias_columns_keys, node, plan_output_columns, planner_context);
}
ActionsDAGPtr getLeftActions()
{
left_alias_columns_keys->projectInput();
return std::move(left_alias_columns_keys);
}
ActionsDAGPtr getRightActions()
{
right_alias_columns_keys->projectInput();
return std::move(right_alias_columns_keys);
}
private:
void addColumnImpl(ActionsDAGPtr & alias_columns_keys, QueryTreeNodePtr & node, const ColumnsWithTypeAndName & plan_output_columns, const PlannerContextPtr & planner_context)
{
auto & column_node = node->as<ColumnNode&>();
if (column_node.hasExpression())
{
auto dag = buildActionsDAGFromExpressionNode(column_node.getExpressionOrThrow(), plan_output_columns, planner_context);
const auto & left_inner_column_identifier = planner_context->getColumnNodeIdentifierOrThrow(node);
dag->addOrReplaceInOutputs(dag->addAlias(*dag->getOutputs().front(), left_inner_column_identifier));
alias_columns_keys->mergeInplace(std::move(*dag));
}
}
ActionsDAGPtr left_alias_columns_keys;
ActionsDAGPtr right_alias_columns_keys;
};
JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_expression,
JoinTreeQueryPlan left_join_tree_query_plan,
JoinTreeQueryPlan right_join_tree_query_plan,
@ -1113,6 +1143,8 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
if (join_node.isUsingJoinExpression())
{
UsingAliasKeyActions using_alias_key_actions{left_plan_output_columns, right_plan_output_columns};
auto & join_node_using_columns_list = join_node.getJoinExpression()->as<ListNode &>();
for (auto & join_node_using_node : join_node_using_columns_list.getNodes())
{
@ -1122,9 +1154,13 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
auto & left_inner_column_node = inner_columns_list.getNodes().at(0);
auto & left_inner_column = left_inner_column_node->as<ColumnNode &>();
using_alias_key_actions.addLeftColumn(left_inner_column_node, left_plan_output_columns, planner_context);
auto & right_inner_column_node = inner_columns_list.getNodes().at(1);
auto & right_inner_column = right_inner_column_node->as<ColumnNode &>();
using_alias_key_actions.addRightColumn(right_inner_column_node, right_plan_output_columns, planner_context);
const auto & join_node_using_column_node_type = join_node_using_column_node.getColumnType();
if (!left_inner_column.getColumnType()->equals(*join_node_using_column_node_type))
{
@ -1138,6 +1174,14 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
right_plan_column_name_to_cast_type.emplace(right_inner_column_identifier, join_node_using_column_node_type);
}
}
auto left_alias_columns_keys_step = std::make_unique<ExpressionStep>(left_plan.getCurrentDataStream(), using_alias_key_actions.getLeftActions());
left_alias_columns_keys_step->setStepDescription("Actions for left table alias column keys");
left_plan.addStep(std::move(left_alias_columns_keys_step));
auto right_alias_columns_keys_step = std::make_unique<ExpressionStep>(right_plan.getCurrentDataStream(), using_alias_key_actions.getRightActions());
right_alias_columns_keys_step->setStepDescription("Actions for right table alias column keys");
right_plan.addStep(std::move(right_alias_columns_keys_step));
}
auto join_cast_plan_output_nodes = [&](QueryPlan & plan_to_add_cast, std::unordered_map<std::string, DataTypePtr> & plan_column_name_to_cast_type)

115
src/Planner/TableExpressionData.h
View File

@ -55,7 +55,7 @@ public:
/// Return true if column with name exists, false otherwise
bool hasColumn(const std::string & column_name) const
{
return column_name_to_column.contains(column_name);
return alias_columns_names.contains(column_name) || column_name_to_column.contains(column_name);
}
/** Add column in table expression data.
@ -63,40 +63,37 @@ public:
*
* Logical error exception is thrown if column already exists.
*/
void addColumn(const NameAndTypePair & column, const ColumnIdentifier & column_identifier, bool is_selected_column = true)
void addColumn(const NameAndTypePair & column, const ColumnIdentifier & column_identifier)
{
if (hasColumn(column.name))
throw Exception(ErrorCodes::LOGICAL_ERROR, "Column with name {} already exists", column.name);
column_names.push_back(column.name);
addColumnImpl(column, column_identifier, is_selected_column);
addColumnImpl(column, column_identifier);
}
/// Add alias column
void addAliasColumn(const NameAndTypePair & column, const ColumnIdentifier & column_identifier, ActionsDAGPtr actions_dag, bool is_selected_column = true)
/** Add column if it does not exists in table expression data.
* Column identifier must be created using global planner context.
*/
void addColumnIfNotExists(const NameAndTypePair & column, const ColumnIdentifier & column_identifier)
{
alias_column_expressions.emplace(column.name, std::move(actions_dag));
addColumnImpl(column, column_identifier, is_selected_column);
if (hasColumn(column.name))
return;
addColumnImpl(column, column_identifier);
}
/// Mark existing column as selected
void markSelectedColumn(const std::string & column_name)
/// Add alias column name
void addAliasColumnName(const std::string & column_name, const ColumnIdentifier & column_identifier)
{
auto [_, inserted] = selected_column_names_set.emplace(column_name);
if (inserted)
selected_column_names.push_back(column_name);
alias_columns_names.insert(column_name);
column_name_to_column_identifier.emplace(column_name, column_identifier);
}
/// Get columns that are requested from table expression, including ALIAS columns
const Names & getSelectedColumnsNames() const
/// Get alias columns names
const NameSet & getAliasColumnsNames() const
{
return selected_column_names;
}
/// Get ALIAS columns names mapped to expressions
const std::unordered_map<std::string, ActionsDAGPtr> & getAliasColumnExpressions() const
{
return alias_column_expressions;
return alias_columns_names;
}
/// Get column name to column map
@ -105,7 +102,7 @@ public:
return column_name_to_column;
}
/// Get column names that are read from table expression
/// Get column names
const Names & getColumnNames() const
{
return column_names;
@ -122,6 +119,23 @@ public:
return result;
}
ColumnIdentifiers getColumnIdentifiers() const
{
ColumnIdentifiers result;
result.reserve(column_identifier_to_column_name.size());
for (const auto & [column_identifier, _] : column_identifier_to_column_name)
result.push_back(column_identifier);
return result;
}
/// Get column name to column identifier map
const ColumnNameToColumnIdentifier & getColumnNameToIdentifier() const
{
return column_name_to_column_identifier;
}
/// Get column identifier to column name map
const ColumnNameToColumnIdentifier & getColumnIdentifierToColumnName() const
{
@ -145,6 +159,18 @@ public:
return it->second;
}
/** Get column for column name.
* Null is returned if there are no column for column name.
*/
const NameAndTypePair * getColumnOrNull(const std::string & column_name) const
{
auto it = column_name_to_column.find(column_name);
if (it == column_name_to_column.end())
return nullptr;
return &it->second;
}
/** Get column identifier for column name.
* Exception is thrown if there are no column identifier for column name.
*/
@ -174,6 +200,24 @@ public:
return &it->second;
}
/** Get column name for column identifier.
* Exception is thrown if there are no column name for column identifier.
*/
const std::string & getColumnNameOrThrow(const ColumnIdentifier & column_identifier) const
{
auto it = column_identifier_to_column_name.find(column_identifier);
if (it == column_identifier_to_column_name.end())
{
auto column_identifiers = getColumnIdentifiers();
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Column name for column identifier {} does not exists. There are only column identifiers: {}",
column_identifier,
fmt::join(column_identifiers.begin(), column_identifiers.end(), ", "));
}
return it->second;
}
/** Get column name for column identifier.
* Null is returned if there are no column name for column identifier.
*/
@ -252,36 +296,23 @@ public:
}
private:
void addColumnImpl(const NameAndTypePair & column, const ColumnIdentifier & column_identifier, bool add_to_selected_columns)
void addColumnImpl(const NameAndTypePair & column, const ColumnIdentifier & column_identifier)
{
if (add_to_selected_columns)
markSelectedColumn(column.name);
column_names.push_back(column.name);
column_name_to_column.emplace(column.name, column);
column_name_to_column_identifier.emplace(column.name, column_identifier);
column_identifier_to_column_name.emplace(column_identifier, column.name);
}
/// Set of columns that are physically read from table expression
/// In case of ALIAS columns it contains source column names that are used to calculate alias
/// This source column may be not used by user
/// Valid for table, table function, array join, query, union nodes
Names column_names;
/// Set of columns that are SELECTed from table expression
/// It may contain ALIAS columns.
/// Mainly it's used to determine access to which columns to check
/// For example user may have an access to column `a ALIAS x + y` but not to `x` and `y`
/// In that case we can read `x` and `y` and calculate `a`, but not return `x` and `y` to user
Names selected_column_names;
/// To deduplicate columns in `selected_column_names`
NameSet selected_column_names_set;
/// Expression to calculate ALIAS columns
std::unordered_map<std::string, ActionsDAGPtr> alias_column_expressions;
/// Valid for table, table function, array join, query, union nodes
ColumnNameToColumn column_name_to_column;
/// Valid only for table node
NameSet alias_columns_names;
/// Valid for table, table function, array join, query, union nodes
ColumnNameToColumnIdentifier column_name_to_column_identifier;

11
src/Planner/Utils.cpp
View File

@ -469,19 +469,12 @@ FilterDAGInfo buildFilterInfo(ASTPtr filter_expression,
NameSet table_expression_required_names_without_filter)
{
const auto & query_context = planner_context->getQueryContext();
auto filter_query_tree = buildQueryTree(filter_expression, query_context);
QueryAnalysisPass query_analysis_pass(table_expression);
query_analysis_pass.run(filter_query_tree, query_context);
return buildFilterInfo(std::move(filter_query_tree), table_expression, planner_context, std::move(table_expression_required_names_without_filter));
}
FilterDAGInfo buildFilterInfo(QueryTreeNodePtr filter_query_tree,
const QueryTreeNodePtr & table_expression,
PlannerContextPtr & planner_context,
NameSet table_expression_required_names_without_filter)
{
if (table_expression_required_names_without_filter.empty())
{
auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression);
@ -489,7 +482,7 @@ FilterDAGInfo buildFilterInfo(QueryTreeNodePtr filter_query_tree,
table_expression_required_names_without_filter.insert(table_expression_names.begin(), table_expression_names.end());
}
collectSourceColumns(filter_query_tree, planner_context, false /*keep_alias_columns*/);
collectSourceColumns(filter_query_tree, planner_context);
collectSets(filter_query_tree, *planner_context);
auto filter_actions_dag = std::make_shared<ActionsDAG>();

5
src/Planner/Utils.h
View File

@ -89,11 +89,6 @@ FilterDAGInfo buildFilterInfo(ASTPtr filter_expression,
PlannerContextPtr & planner_context,
NameSet table_expression_required_names_without_filter = {});
FilterDAGInfo buildFilterInfo(QueryTreeNodePtr filter_query_tree,
const QueryTreeNodePtr & table_expression,
PlannerContextPtr & planner_context,
NameSet table_expression_required_names_without_filter = {});
ASTPtr parseAdditionalResultFilter(const Settings & settings);
}

5
src/Processors/QueryPlan/ReadFromMergeTree.cpp
View File

@ -1434,13 +1434,8 @@ void ReadFromMergeTree::applyFilters(ActionDAGNodes added_filter_nodes)
if (query_info.planner_context)
{
const auto & table_expression_data = query_info.planner_context->getTableExpressionDataOrThrow(query_info.table_expression);
const auto & alias_column_expressions = table_expression_data.getAliasColumnExpressions();
for (const auto & [column_identifier, column_name] : table_expression_data.getColumnIdentifierToColumnName())
{
/// ALIAS columns cannot be used in the filter expression without being calculated in ActionsDAG,
/// so they should not be added to the input nodes.
if (alias_column_expressions.contains(column_name))
continue;
const auto & column = table_expression_data.getColumnOrThrow(column_name);
node_name_to_input_node_column.emplace(column_identifier, ColumnWithTypeAndName(column.type, column_name));
}

28
src/Storages/StorageDistributed.cpp
View File

@ -744,32 +744,6 @@ StorageSnapshotPtr StorageDistributed::getStorageSnapshotForQuery(
namespace
{
class ReplaseAliasColumnsVisitor : public InDepthQueryTreeVisitor<ReplaseAliasColumnsVisitor>
{
static QueryTreeNodePtr getColumnNodeAliasExpression(const QueryTreeNodePtr & node)
{
const auto * column_node = node->as<ColumnNode>();
if (!column_node || !column_node->hasExpression())
return nullptr;
const auto & column_source = column_node->getColumnSourceOrNull();
if (!column_source || column_source->getNodeType() == QueryTreeNodeType::JOIN
|| column_source->getNodeType() == QueryTreeNodeType::ARRAY_JOIN)
return nullptr;
auto column_expression = column_node->getExpression();
column_expression->setAlias(column_node->getColumnName());
return column_expression;
}
public:
void visitImpl(QueryTreeNodePtr & node)
{
if (auto column_expression = getColumnNodeAliasExpression(node))
node = column_expression;
}
};
QueryTreeNodePtr buildQueryTreeDistributed(SelectQueryInfo & query_info,
const StorageSnapshotPtr & distributed_storage_snapshot,
const StorageID & remote_storage_id,
@ -822,8 +796,6 @@ QueryTreeNodePtr buildQueryTreeDistributed(SelectQueryInfo & query_info,
replacement_table_expression->setAlias(query_info.table_expression->getAlias());
auto query_tree_to_modify = query_info.query_tree->cloneAndReplace(query_info.table_expression, std::move(replacement_table_expression));
ReplaseAliasColumnsVisitor replase_alias_columns_visitor;
replase_alias_columns_visitor.visit(query_tree_to_modify);
return buildQueryTreeForShard(query_info.planner_context, query_tree_to_modify);
}

1
tests/analyzer_integration_broken_tests.txt
View File

@ -4,3 +4,4 @@ test_distributed_type_object/test.py::test_distributed_type_object
test_merge_table_over_distributed/test.py::test_global_in
test_merge_table_over_distributed/test.py::test_select_table_name_from_merge_over_distributed
test_passing_max_partitions_to_read_remotely/test.py::test_default_database_on_cluster
test_select_access_rights/test_main.py::test_alias_columns

11
tests/ci/ci_config.py
View File

@ -21,6 +21,7 @@ class Labels(metaclass=WithIter):
CI_SET_REDUCED = "ci_set_reduced"
CI_SET_ARM = "ci_set_arm"
CI_SET_INTEGRATION = "ci_set_integration"
CI_SET_ANALYZER = "ci_set_analyzer"
libFuzzer = "libFuzzer"
@ -647,6 +648,16 @@ CI_CONFIG = CIConfig(
JobNames.INTEGRATION_TEST,
]
),
Labels.CI_SET_ANALYZER: LabelConfig(
run_jobs=[
JobNames.STYLE_CHECK,
JobNames.FAST_TEST,
Build.PACKAGE_RELEASE,
Build.PACKAGE_ASAN,
JobNames.STATELESS_TEST_ANALYZER_S3_REPLICATED_RELEASE,
JobNames.INTEGRATION_TEST_ASAN_ANALYZER,
]
),
Labels.CI_SET_REDUCED: LabelConfig(
run_jobs=[
job

2
tests/integration/test_disabled_access_control_improvements/test_row_policy.py
View File

@ -41,7 +41,7 @@ def started_cluster():
CREATE TABLE mydb.filtered_table2 (a UInt8, b UInt8, c UInt8, d UInt8) ENGINE MergeTree ORDER BY a;
INSERT INTO mydb.filtered_table2 values (0, 0, 0, 0), (1, 2, 3, 4), (4, 3, 2, 1), (0, 0, 6, 0);
CREATE TABLE mydb.filtered_table3 (a UInt8, b UInt8, bb ALIAS b + 1, c UInt16 ALIAS a + bb - 1) ENGINE MergeTree ORDER BY a;
CREATE TABLE mydb.filtered_table3 (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a;
INSERT INTO mydb.filtered_table3 values (0, 0), (0, 1), (1, 0), (1, 1);
CREATE TABLE mydb.`.filtered_table4` (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a;

3
tests/integration/test_row_policy/test.py
View File

@ -60,7 +60,7 @@ def started_cluster():
CREATE TABLE mydb.filtered_table2 (a UInt8, b UInt8, c UInt8, d UInt8) ENGINE MergeTree ORDER BY a;
INSERT INTO mydb.filtered_table2 values (0, 0, 0, 0), (1, 2, 3, 4), (4, 3, 2, 1), (0, 0, 6, 0);
CREATE TABLE mydb.filtered_table3 (a UInt8, b UInt8, bb ALIAS b + 1, c UInt16 ALIAS a + bb - 1) ENGINE MergeTree ORDER BY a;
CREATE TABLE mydb.filtered_table3 (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a;
INSERT INTO mydb.filtered_table3 values (0, 0), (0, 1), (1, 0), (1, 1);
CREATE TABLE mydb.`.filtered_table4` (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a;
@ -113,7 +113,6 @@ def test_smoke():
assert node.query("SELECT a FROM mydb.filtered_table3") == TSV([[0], [1]])
assert node.query("SELECT b FROM mydb.filtered_table3") == TSV([[1], [0]])
assert node.query("SELECT bb FROM mydb.filtered_table3") == TSV([[2], [1]])
assert node.query("SELECT c FROM mydb.filtered_table3") == TSV([[1], [1]])
assert node.query("SELECT a + b FROM mydb.filtered_table3") == TSV([[1], [1]])
assert node.query("SELECT a FROM mydb.filtered_table3 WHERE c = 1") == TSV(

12
tests/queries/0_stateless/02514_analyzer_drop_join_on.reference
View File

@ -55,33 +55,33 @@ Header: a2 String
Header: __table1.a2 String
__table1.k UInt64
__table4.d2 String
Expression (DROP unused columns after JOIN)
Expression ((Actions for left table alias column keys + DROP unused columns after JOIN))
Header: __table1.a2 String
__table1.k UInt64
Join (JOIN FillRightFirst)
Header: __table1.a2 String
__table1.k UInt64
Expression (DROP unused columns after JOIN)
Expression ((Actions for left table alias column keys + DROP unused columns after JOIN))
Header: __table1.a2 String
__table1.k UInt64
Join (JOIN FillRightFirst)
Header: __table1.a2 String
__table1.k UInt64
Expression (Change column names to column identifiers)
Expression ((Actions for left table alias column keys + Change column names to column identifiers))
Header: __table1.a2 String
__table1.k UInt64
ReadFromMemoryStorage
Header: a2 String
k UInt64
Expression (Change column names to column identifiers)
Expression ((Actions for right table alias column keys + Change column names to column identifiers))
Header: __table2.k UInt64
ReadFromMemoryStorage
Header: k UInt64
Expression (Change column names to column identifiers)
Expression ((Actions for right table alias column keys + Change column names to column identifiers))
Header: __table3.k UInt64
ReadFromMemoryStorage
Header: k UInt64
Expression (Change column names to column identifiers)
Expression ((Actions for right table alias column keys + Change column names to column identifiers))
Header: __table4.d2 String
__table4.k UInt64
ReadFromMemoryStorage

37
tests/queries/0_stateless/02783_parsedatetimebesteffort_syslog.reference
View File

@ -1,37 +0,0 @@
The reference time point is 2023-06-30 23:59:30
───────────────────────────────────────────────
The argument is before the reference time point
───────────────────────────────────────────────
Row 1:
──────
syslog_arg: Jun 30 23:58:30
res: 2023-06-30 23:58:30
res_null: 2023-06-30 23:58:30
res_zero: 2023-06-30 23:58:30
res_us: 2023-06-30 23:58:30
res_us_null: 2023-06-30 23:58:30
res_us_zero: 2023-06-30 23:58:30
res64: 2023-06-30 23:58:30.000
res64_null: 2023-06-30 23:58:30.000
res64_zero: 2023-06-30 23:58:30.000
res64_us: 2023-06-30 23:58:30.000
res64_us_null: 2023-06-30 23:58:30.000
res64_us_zero: 2023-06-30 23:58:30.000
──────────────────────────────────────────────
The argument is after the reference time point
──────────────────────────────────────────────
Row 1:
──────
syslog_arg: Jul 1 00:00:30
res: 2022-07-01 00:00:30
res_null: 2022-07-01 00:00:30
res_zero: 2022-07-01 00:00:30
res_us: 2022-07-01 00:00:30
res_us_null: 2022-07-01 00:00:30
res_us_zero: 2022-07-01 00:00:30
res64: 2022-07-01 00:00:30.000
res64_null: 2022-07-01 00:00:30.000
res64_zero: 2022-07-01 00:00:30.000
res64_us: 2022-07-01 00:00:30.000
res64_us_null: 2022-07-01 00:00:30.000
res64_us_zero: 2022-07-01 00:00:30.000

54
tests/queries/0_stateless/02783_parsedatetimebesteffort_syslog.sql
View File

@ -1,54 +0,0 @@
SET session_timezone = 'UTC';
SELECT 'The reference time point is 2023-06-30 23:59:30';
SELECT '───────────────────────────────────────────────';
SELECT 'The argument is before the reference time point';
SELECT '───────────────────────────────────────────────';
WITH
toDateTime('2023-06-30 23:59:30') AS dt_ref,
now() AS dt_now,
date_sub(MINUTE, 1, dt_now) as dt_before,
dateDiff(SECOND, dt_ref, dt_now) AS time_shift,
formatDateTime(dt_before, '%b %e %T') AS syslog_before
SELECT
formatDateTime(dt_before - time_shift, '%b %e %T') AS syslog_arg,
parseDateTimeBestEffort(syslog_before) - time_shift AS res,
parseDateTimeBestEffortOrNull(syslog_before) - time_shift AS res_null,
parseDateTimeBestEffortOrZero(syslog_before) - time_shift AS res_zero,
parseDateTimeBestEffortUS(syslog_before) - time_shift AS res_us,
parseDateTimeBestEffortUSOrNull(syslog_before) - time_shift AS res_us_null,
parseDateTimeBestEffortUSOrZero(syslog_before) - time_shift AS res_us_zero,
parseDateTime64BestEffort(syslog_before) - time_shift AS res64,
parseDateTime64BestEffortOrNull(syslog_before) - time_shift AS res64_null,
parseDateTime64BestEffortOrZero(syslog_before) - time_shift AS res64_zero,
parseDateTime64BestEffortUS(syslog_before) - time_shift AS res64_us,
parseDateTime64BestEffortUSOrNull(syslog_before) - time_shift AS res64_us_null,
parseDateTime64BestEffortUSOrZero(syslog_before) - time_shift AS res64_us_zero
FORMAT Vertical;
SELECT '──────────────────────────────────────────────';
SELECT 'The argument is after the reference time point';
SELECT '──────────────────────────────────────────────';
WITH
toDateTime('2023-06-30 23:59:30') AS dt_ref,
now() AS dt_now,
date_add(MINUTE, 1, dt_now) as dt_after,
dateDiff(SECOND, dt_ref, dt_now) AS time_shift,
formatDateTime(dt_after, '%b %e %T') AS syslog_after
SELECT
formatDateTime(dt_after - time_shift, '%b %e %T') AS syslog_arg,
parseDateTimeBestEffort(syslog_after) - time_shift AS res,
parseDateTimeBestEffortOrNull(syslog_after) - time_shift AS res_null,
parseDateTimeBestEffortOrZero(syslog_after) - time_shift AS res_zero,
parseDateTimeBestEffortUS(syslog_after) - time_shift AS res_us,
parseDateTimeBestEffortUSOrNull(syslog_after) - time_shift AS res_us_null,
parseDateTimeBestEffortUSOrZero(syslog_after) - time_shift AS res_us_zero,
parseDateTime64BestEffort(syslog_after) - time_shift AS res64,
parseDateTime64BestEffortOrNull(syslog_after) - time_shift AS res64_null,
parseDateTime64BestEffortOrZero(syslog_after) - time_shift AS res64_zero,
parseDateTime64BestEffortUS(syslog_after) - time_shift AS res64_us,
parseDateTime64BestEffortUSOrNull(syslog_after) - time_shift AS res64_us_null,
parseDateTime64BestEffortUSOrZero(syslog_after) - time_shift AS res64_us_zero
FORMAT Vertical;

526
tests/queries/0_stateless/02815_analyzer_aggregate_functions_of_group_by_keys.reference Normal file
View File

@ -0,0 +1,526 @@
set optimize_aggregators_of_group_by_keys = 1
0 0
0 1
0 2
1 0
1 1
1 2
0 0
0 1
0 2
1 0
1 1
1 2
0
0
0
0
0
0
0
0
0
0
0
1
2
2
3
3
4
4
4
5
6
6
6
8
8
9
10
12
12
12
15
16
18
20
24
0
0
QUERY id: 0
PROJECTION COLUMNS
a UInt8
b UInt8
PROJECTION
LIST id: 1, nodes: 2
FUNCTION id: 2, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 3, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 6, constant_value: UInt64_2, constant_value_type: UInt8
FUNCTION id: 7, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 8, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 9, constant_value: UInt64_3, constant_value_type: UInt8
JOIN TREE
TABLE_FUNCTION id: 5, alias: __table1, table_function_name: numbers
ARGUMENTS
LIST id: 10, nodes: 1
CONSTANT id: 11, constant_value: UInt64_10000000, constant_value_type: UInt32
GROUP BY
LIST id: 12, nodes: 2
FUNCTION id: 13, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 14, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 15, constant_value: UInt64_2, constant_value_type: UInt8
FUNCTION id: 16, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 17, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 18, constant_value: UInt64_3, constant_value_type: UInt8
ORDER BY
LIST id: 19, nodes: 2
SORT id: 20, sort_direction: ASCENDING, with_fill: 0
EXPRESSION
FUNCTION id: 2, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 3, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 6, constant_value: UInt64_2, constant_value_type: UInt8
SORT id: 21, sort_direction: ASCENDING, with_fill: 0
EXPRESSION
FUNCTION id: 7, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 8, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 9, constant_value: UInt64_3, constant_value_type: UInt8
QUERY id: 0
PROJECTION COLUMNS
a UInt8
b UInt8
PROJECTION
LIST id: 1, nodes: 2
FUNCTION id: 2, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 3, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 6, constant_value: UInt64_2, constant_value_type: UInt8
FUNCTION id: 7, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 8, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 9, constant_value: UInt64_3, constant_value_type: UInt8
JOIN TREE
TABLE_FUNCTION id: 5, alias: __table1, table_function_name: numbers
ARGUMENTS
LIST id: 10, nodes: 1
CONSTANT id: 11, constant_value: UInt64_10000000, constant_value_type: UInt32
GROUP BY
LIST id: 12, nodes: 2
FUNCTION id: 13, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 14, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 15, constant_value: UInt64_2, constant_value_type: UInt8
FUNCTION id: 16, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 17, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 18, constant_value: UInt64_3, constant_value_type: UInt8
ORDER BY
LIST id: 19, nodes: 2
SORT id: 20, sort_direction: ASCENDING, with_fill: 0
EXPRESSION
FUNCTION id: 2, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 3, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 6, constant_value: UInt64_2, constant_value_type: UInt8
SORT id: 21, sort_direction: ASCENDING, with_fill: 0
EXPRESSION
FUNCTION id: 7, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 8, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 9, constant_value: UInt64_3, constant_value_type: UInt8
QUERY id: 0
PROJECTION COLUMNS
a UInt16
PROJECTION
LIST id: 1, nodes: 1
FUNCTION id: 2, function_name: multiply, function_type: ordinary, result_type: UInt16
ARGUMENTS
LIST id: 3, nodes: 2
FUNCTION id: 4, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 5, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 8, constant_value: UInt64_5, constant_value_type: UInt8
FUNCTION id: 9, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 10, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 11, constant_value: UInt64_7, constant_value_type: UInt8
JOIN TREE
TABLE_FUNCTION id: 7, alias: __table1, table_function_name: numbers
ARGUMENTS
LIST id: 12, nodes: 1
CONSTANT id: 13, constant_value: UInt64_10000000, constant_value_type: UInt32
GROUP BY
LIST id: 14, nodes: 2
FUNCTION id: 15, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 16, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 17, constant_value: UInt64_7, constant_value_type: UInt8
FUNCTION id: 18, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 19, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 20, constant_value: UInt64_5, constant_value_type: UInt8
ORDER BY
LIST id: 21, nodes: 1
SORT id: 22, sort_direction: ASCENDING, with_fill: 0
EXPRESSION
FUNCTION id: 2, function_name: multiply, function_type: ordinary, result_type: UInt16
ARGUMENTS
LIST id: 3, nodes: 2
FUNCTION id: 4, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 5, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 8, constant_value: UInt64_5, constant_value_type: UInt8
FUNCTION id: 9, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 10, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 11, constant_value: UInt64_7, constant_value_type: UInt8
QUERY id: 0
PROJECTION COLUMNS
foo UInt64
PROJECTION
LIST id: 1, nodes: 1
COLUMN id: 2, column_name: foo, result_type: UInt64, source_id: 3
JOIN TREE
QUERY id: 3, alias: __table1, is_subquery: 1
PROJECTION COLUMNS
foo UInt64
PROJECTION
LIST id: 4, nodes: 1
COLUMN id: 5, column_name: number, result_type: UInt64, source_id: 6
JOIN TREE
TABLE_FUNCTION id: 6, alias: __table2, table_function_name: numbers
ARGUMENTS
LIST id: 7, nodes: 1
CONSTANT id: 8, constant_value: UInt64_1, constant_value_type: UInt8
GROUP BY
LIST id: 9, nodes: 1
COLUMN id: 5, column_name: number, result_type: UInt64, source_id: 6
QUERY id: 0
PROJECTION COLUMNS
min(number) OVER (PARTITION BY modulo(number, 2)) UInt64
PROJECTION
LIST id: 1, nodes: 1
FUNCTION id: 2, function_name: min, function_type: window, result_type: UInt64
ARGUMENTS
LIST id: 3, nodes: 1
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
WINDOW
WINDOW id: 6, frame_type: RANGE, frame_begin_type: unbounded preceding, frame_end_type: current
PARTITION BY
LIST id: 7, nodes: 1
FUNCTION id: 8, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 9, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 10, constant_value: UInt64_2, constant_value_type: UInt8
JOIN TREE
TABLE_FUNCTION id: 5, alias: __table1, table_function_name: numbers
ARGUMENTS
LIST id: 11, nodes: 1
CONSTANT id: 12, constant_value: UInt64_3, constant_value_type: UInt8
GROUP BY
LIST id: 13, nodes: 1
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
set optimize_aggregators_of_group_by_keys = 0
0 0
0 1
0 2
1 0
1 1
1 2
0 0
0 1
0 2
1 0
1 1
1 2
0
0
0
0
0
0
0
0
0
0
0
1
2
2
3
3
4
4
4
5
6
6
6
8
8
9
10
12
12
12
15
16
18
20
24
0
QUERY id: 0
PROJECTION COLUMNS
a UInt8
b UInt8
PROJECTION
LIST id: 1, nodes: 2
FUNCTION id: 2, function_name: min, function_type: aggregate, result_type: UInt8
ARGUMENTS
LIST id: 3, nodes: 1
FUNCTION id: 4, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 5, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 8, constant_value: UInt64_2, constant_value_type: UInt8
FUNCTION id: 9, function_name: max, function_type: aggregate, result_type: UInt8
ARGUMENTS
LIST id: 10, nodes: 1
FUNCTION id: 11, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 12, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 13, constant_value: UInt64_3, constant_value_type: UInt8
JOIN TREE
TABLE_FUNCTION id: 7, alias: __table1, table_function_name: numbers
ARGUMENTS
LIST id: 14, nodes: 1
CONSTANT id: 15, constant_value: UInt64_10000000, constant_value_type: UInt32
GROUP BY
LIST id: 16, nodes: 2
FUNCTION id: 17, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 18, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 19, constant_value: UInt64_2, constant_value_type: UInt8
FUNCTION id: 20, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 21, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 22, constant_value: UInt64_3, constant_value_type: UInt8
ORDER BY
LIST id: 23, nodes: 2
SORT id: 24, sort_direction: ASCENDING, with_fill: 0
EXPRESSION
FUNCTION id: 2, function_name: min, function_type: aggregate, result_type: UInt8
ARGUMENTS
LIST id: 3, nodes: 1
FUNCTION id: 4, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 5, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 8, constant_value: UInt64_2, constant_value_type: UInt8
SORT id: 25, sort_direction: ASCENDING, with_fill: 0
EXPRESSION
FUNCTION id: 9, function_name: max, function_type: aggregate, result_type: UInt8
ARGUMENTS
LIST id: 10, nodes: 1
FUNCTION id: 11, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 12, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 13, constant_value: UInt64_3, constant_value_type: UInt8
QUERY id: 0
PROJECTION COLUMNS
a UInt8
b UInt8
PROJECTION
LIST id: 1, nodes: 2
FUNCTION id: 2, function_name: any, function_type: aggregate, result_type: UInt8
ARGUMENTS
LIST id: 3, nodes: 1
FUNCTION id: 4, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 5, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 8, constant_value: UInt64_2, constant_value_type: UInt8
FUNCTION id: 9, function_name: anyLast, function_type: aggregate, result_type: UInt8
ARGUMENTS
LIST id: 10, nodes: 1
FUNCTION id: 11, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 12, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 13, constant_value: UInt64_3, constant_value_type: UInt8
JOIN TREE
TABLE_FUNCTION id: 7, alias: __table1, table_function_name: numbers
ARGUMENTS
LIST id: 14, nodes: 1
CONSTANT id: 15, constant_value: UInt64_10000000, constant_value_type: UInt32
GROUP BY
LIST id: 16, nodes: 2
FUNCTION id: 17, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 18, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 19, constant_value: UInt64_2, constant_value_type: UInt8
FUNCTION id: 20, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 21, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 22, constant_value: UInt64_3, constant_value_type: UInt8
ORDER BY
LIST id: 23, nodes: 2
SORT id: 24, sort_direction: ASCENDING, with_fill: 0
EXPRESSION
FUNCTION id: 2, function_name: any, function_type: aggregate, result_type: UInt8
ARGUMENTS
LIST id: 3, nodes: 1
FUNCTION id: 4, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 5, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 8, constant_value: UInt64_2, constant_value_type: UInt8
SORT id: 25, sort_direction: ASCENDING, with_fill: 0
EXPRESSION
FUNCTION id: 9, function_name: anyLast, function_type: aggregate, result_type: UInt8
ARGUMENTS
LIST id: 10, nodes: 1
FUNCTION id: 11, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 12, nodes: 2
COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
CONSTANT id: 13, constant_value: UInt64_3, constant_value_type: UInt8
QUERY id: 0
PROJECTION COLUMNS
a UInt16
PROJECTION
LIST id: 1, nodes: 1
FUNCTION id: 2, function_name: max, function_type: aggregate, result_type: UInt16
ARGUMENTS
LIST id: 3, nodes: 1
FUNCTION id: 4, function_name: multiply, function_type: ordinary, result_type: UInt16
ARGUMENTS
LIST id: 5, nodes: 2
FUNCTION id: 6, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 7, nodes: 2
COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9
CONSTANT id: 10, constant_value: UInt64_5, constant_value_type: UInt8
FUNCTION id: 11, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 12, nodes: 2
COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9
CONSTANT id: 13, constant_value: UInt64_7, constant_value_type: UInt8
JOIN TREE
TABLE_FUNCTION id: 9, alias: __table1, table_function_name: numbers
ARGUMENTS
LIST id: 14, nodes: 1
CONSTANT id: 15, constant_value: UInt64_10000000, constant_value_type: UInt32
GROUP BY
LIST id: 16, nodes: 2
FUNCTION id: 17, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 18, nodes: 2
COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9
CONSTANT id: 19, constant_value: UInt64_7, constant_value_type: UInt8
FUNCTION id: 20, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 21, nodes: 2
COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9
CONSTANT id: 22, constant_value: UInt64_5, constant_value_type: UInt8
ORDER BY
LIST id: 23, nodes: 1
SORT id: 24, sort_direction: ASCENDING, with_fill: 0
EXPRESSION
FUNCTION id: 2, function_name: max, function_type: aggregate, result_type: UInt16
ARGUMENTS
LIST id: 3, nodes: 1
FUNCTION id: 4, function_name: multiply, function_type: ordinary, result_type: UInt16
ARGUMENTS
LIST id: 5, nodes: 2
FUNCTION id: 6, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 7, nodes: 2
COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9
CONSTANT id: 10, constant_value: UInt64_5, constant_value_type: UInt8
FUNCTION id: 11, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 12, nodes: 2
COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9
CONSTANT id: 13, constant_value: UInt64_7, constant_value_type: UInt8
QUERY id: 0
PROJECTION COLUMNS
foo UInt64
PROJECTION
LIST id: 1, nodes: 1
COLUMN id: 2, column_name: foo, result_type: UInt64, source_id: 3
JOIN TREE
QUERY id: 3, alias: __table1, is_subquery: 1
PROJECTION COLUMNS
foo UInt64
PROJECTION
LIST id: 4, nodes: 1
FUNCTION id: 5, function_name: anyLast, function_type: aggregate, result_type: UInt64
ARGUMENTS
LIST id: 6, nodes: 1
COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
JOIN TREE
TABLE_FUNCTION id: 8, alias: __table2, table_function_name: numbers
ARGUMENTS
LIST id: 9, nodes: 1
CONSTANT id: 10, constant_value: UInt64_1, constant_value_type: UInt8
GROUP BY
LIST id: 11, nodes: 1
COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
QUERY id: 0
PROJECTION COLUMNS
min(number) OVER (PARTITION BY modulo(number, 2)) UInt64
PROJECTION
LIST id: 1, nodes: 1
FUNCTION id: 2, function_name: min, function_type: window, result_type: UInt64
ARGUMENTS
LIST id: 3, nodes: 1
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
WINDOW
WINDOW id: 6, frame_type: RANGE, frame_begin_type: unbounded preceding, frame_end_type: current
PARTITION BY
LIST id: 7, nodes: 1
FUNCTION id: 8, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 9, nodes: 2
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
CONSTANT id: 10, constant_value: UInt64_2, constant_value_type: UInt8
JOIN TREE
TABLE_FUNCTION id: 5, alias: __table1, table_function_name: numbers
ARGUMENTS
LIST id: 11, nodes: 1
CONSTANT id: 12, constant_value: UInt64_3, constant_value_type: UInt8
GROUP BY
LIST id: 13, nodes: 1
COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5

39
tests/queries/0_stateless/02815_analyzer_aggregate_functions_of_group_by_keys.sql Normal file
View File

@ -0,0 +1,39 @@
set allow_experimental_analyzer = 1;
set optimize_move_functions_out_of_any = 0;
SELECT 'set optimize_aggregators_of_group_by_keys = 1';
set optimize_aggregators_of_group_by_keys = 1;
SELECT min(number % 2) AS a, max(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
SELECT any(number % 2) AS a, anyLast(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
SELECT max((number % 5) * (number % 7)) AS a FROM numbers(10000000) GROUP BY number % 7, number % 5 ORDER BY a;
SELECT foo FROM (SELECT anyLast(number) AS foo FROM numbers(1) GROUP BY number);
SELECT anyLast(number) FROM numbers(1) GROUP BY number;
EXPLAIN QUERY TREE SELECT min(number % 2) AS a, max(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
EXPLAIN QUERY TREE SELECT any(number % 2) AS a, anyLast(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
EXPLAIN QUERY TREE SELECT max((number % 5) * (number % 7)) AS a FROM numbers(10000000) GROUP BY number % 7, number % 5 ORDER BY a;
EXPLAIN QUERY TREE SELECT foo FROM (SELECT anyLast(number) AS foo FROM numbers(1) GROUP BY number);
EXPLAIN QUERY TREE
SELECT min(number) OVER (PARTITION BY number % 2)
FROM numbers(3)
GROUP BY number;
SELECT 'set optimize_aggregators_of_group_by_keys = 0';
set optimize_aggregators_of_group_by_keys = 0;
SELECT min(number % 2) AS a, max(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
SELECT any(number % 2) AS a, anyLast(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
SELECT max((number % 5) * (number % 7)) AS a FROM numbers(10000000) GROUP BY number % 7, number % 5 ORDER BY a;
SELECT foo FROM (SELECT anyLast(number) AS foo FROM numbers(1) GROUP BY number);
EXPLAIN QUERY TREE SELECT min(number % 2) AS a, max(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
EXPLAIN QUERY TREE SELECT any(number % 2) AS a, anyLast(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
EXPLAIN QUERY TREE SELECT max((number % 5) * (number % 7)) AS a FROM numbers(10000000) GROUP BY number % 7, number % 5 ORDER BY a;
EXPLAIN QUERY TREE SELECT foo FROM (SELECT anyLast(number) AS foo FROM numbers(1) GROUP BY number);
EXPLAIN QUERY TREE
SELECT min(number) OVER (PARTITION BY number % 2)
FROM numbers(3)
GROUP BY number;

13
tests/queries/0_stateless/02911_support_alias_column_in_indices.reference
View File

@ -14,13 +14,13 @@ Expression ((Projection + Before ORDER BY))
Parts: 1/1
Granules: 1/1
Expression ((Project names + Projection))
Filter ((WHERE + (Change column names to column identifiers + Compute alias columns)))
Filter ((WHERE + Change column names to column identifiers))
ReadFromMergeTree (02911_support_alias_column_in_indices.test1)
Indexes:
PrimaryKey
Keys:
c
Condition: (plus(c, 1) in [11, +Inf))
Condition: (_CAST(plus(c, \'UInt64\'), 1) in [11, +Inf))
Parts: 1/2
Granules: 1/2
Skip
@ -44,17 +44,12 @@ Expression ((Projection + Before ORDER BY))
Parts: 1/1
Granules: 1/1
Expression ((Project names + Projection))
Filter ((WHERE + (Change column names to column identifiers + Compute alias columns)))
Filter ((WHERE + Change column names to column identifiers))
ReadFromMergeTree (02911_support_alias_column_in_indices.test2)
Indexes:
PrimaryKey
Keys:
c
Condition: (plus(plus(c, 1), 1) in [16, +Inf))
Condition: (_CAST(plus(_CAST(plus(c, \'UInt64\'), 1), \'UInt64\'), 1) in [16, +Inf))
Parts: 1/2
Granules: 1/2
Skip
Name: i
Description: minmax GRANULARITY 1
Parts: 1/1
Granules: 1/1