Merge pull request #50894 from ClickHouse/storage-merge-aliases-analyzer

Analyzer: support aliases and distributed JOINs in StorageMerge
This commit is contained in:
Dmitry Novik 2024-02-08 11:22:00 +01:00 committed by GitHub
commit b8f74b5b05
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
69 changed files with 413 additions and 106 deletions

View File

@ -108,6 +108,11 @@ public:
*/
QueryTreeNodePtr getColumnSourceOrNull() const;
void setColumnSource(const QueryTreeNodePtr & source)
{
getSourceWeakPointer() = source;
}
QueryTreeNodeType getNodeType() const override
{
return QueryTreeNodeType::COLUMN;

View File

@ -31,7 +31,7 @@ public:
virtual String getDescription() = 0;
/// Run pass over query tree
virtual void run(QueryTreeNodePtr query_tree_node, ContextPtr context) = 0;
virtual void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) = 0;
};

View File

@ -194,7 +194,7 @@ private:
}
void AggregateFunctionsArithmericOperationsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void AggregateFunctionsArithmericOperationsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
AggregateFunctionsArithmericOperationsVisitor visitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -17,7 +17,7 @@ public:
String getDescription() override { return "Extract arithmeric operations from aggregate functions."; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -92,7 +92,7 @@ public:
}
void RewriteArrayExistsToHasPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void RewriteArrayExistsToHasPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
RewriteArrayExistsToHasVisitor visitor(context);
visitor.visit(query_tree_node);

View File

@ -20,7 +20,7 @@ public:
String getDescription() override { return "Rewrite arrayExists(func, arr) functions to has(arr, elem) when logically equivalent"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};
}

View File

@ -67,7 +67,7 @@ private:
}
void AutoFinalOnQueryPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void AutoFinalOnQueryPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
auto visitor = AutoFinalOnQueryPassVisitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -25,7 +25,7 @@ public:
return "Automatically applies final modifier to table expressions in queries if it is supported and if user level final setting is set";
}
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};
}

View File

@ -213,7 +213,7 @@ private:
}
void ComparisonTupleEliminationPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void ComparisonTupleEliminationPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
ComparisonTupleEliminationPassVisitor visitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -17,7 +17,7 @@ public:
String getDescription() override { return "Rewrite tuples comparison into equivalent comparison of tuples arguments"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -132,7 +132,7 @@ private:
}
void ConvertOrLikeChainPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void ConvertOrLikeChainPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
auto or_function_resolver = FunctionFactory::instance().get("or", context);
auto match_function_resolver = FunctionFactory::instance().get("multiMatchAny", context);

View File

@ -14,7 +14,7 @@ public:
String getDescription() override { return "Replaces all the 'or's with {i}like to multiMatchAny"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};
}

View File

@ -718,7 +718,7 @@ public:
}
void ConvertLogicalExpressionToCNFPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void ConvertLogicalExpressionToCNFPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
const auto & settings = context->getSettingsRef();
if (!settings.convert_query_to_cnf)

View File

@ -12,7 +12,7 @@ public:
String getDescription() override { return "Convert logical expression to CNF and apply optimizations using constraints"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};
}

View File

@ -87,7 +87,7 @@ public:
}
void CountDistinctPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void CountDistinctPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
CountDistinctVisitor visitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -20,7 +20,7 @@ public:
return "Optimize single countDistinct into count over subquery";
}
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -264,7 +264,7 @@ private:
}
void CrossToInnerJoinPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void CrossToInnerJoinPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
CrossToInnerJoinVisitor visitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -22,7 +22,7 @@ public:
return "Replace CROSS JOIN with INNER JOIN";
}
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};
}

View File

@ -224,7 +224,7 @@ private:
}
void FunctionToSubcolumnsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void FunctionToSubcolumnsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
FunctionToSubcolumnsVisitor visitor(context);
visitor.visit(query_tree_node);

View File

@ -24,7 +24,7 @@ public:
String getDescription() override { return "Rewrite function to subcolumns, for example tupleElement(column, subcolumn) into column.subcolumn"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -256,7 +256,7 @@ void tryFuseQuantiles(QueryTreeNodePtr query_tree_node, ContextPtr context)
}
void FuseFunctionsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void FuseFunctionsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
tryFuseSumCountAvg(query_tree_node, context);
tryFuseQuantiles(query_tree_node, context);

View File

@ -20,7 +20,7 @@ public:
String getDescription() override { return "Replaces several calls of aggregate functions of the same family into one call"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};
}

View File

@ -249,7 +249,7 @@ private:
}
void GroupingFunctionsResolvePass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void GroupingFunctionsResolvePass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
GroupingFunctionsResolveVisitor visitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -24,7 +24,7 @@ public:
String getDescription() override { return "Resolve GROUPING functions based on GROUP BY modifiers"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -73,7 +73,7 @@ private:
}
void IfChainToMultiIfPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void IfChainToMultiIfPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
auto multi_if_function_ptr = FunctionFactory::instance().get("multiIf", context);
IfChainToMultiIfPassVisitor visitor(std::move(multi_if_function_ptr), std::move(context));

View File

@ -18,7 +18,7 @@ public:
String getDescription() override { return "Optimize if chain to multiIf"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -57,7 +57,7 @@ public:
}
void IfConstantConditionPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void IfConstantConditionPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
IfConstantConditionVisitor visitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -21,7 +21,7 @@ public:
String getDescription() override { return "Optimize if, multiIf for constant condition."; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -190,7 +190,7 @@ public:
}
void IfTransformStringsToEnumPass::run(QueryTreeNodePtr query, ContextPtr context)
void IfTransformStringsToEnumPass::run(QueryTreeNodePtr & query, ContextPtr context)
{
ConvertStringsToEnumVisitor visitor(std::move(context));
visitor.visit(query);

View File

@ -33,7 +33,7 @@ public:
String getDescription() override { return "Replaces string-type arguments in If and Transform to enum"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};
}

View File

@ -554,7 +554,7 @@ private:
}
};
void LogicalExpressionOptimizerPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void LogicalExpressionOptimizerPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
LogicalExpressionOptimizerVisitor visitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -109,7 +109,7 @@ public:
"replace chains of equality functions inside an OR with a single IN operator";
}
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};
}

View File

@ -52,7 +52,7 @@ private:
}
void MultiIfToIfPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void MultiIfToIfPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
auto if_function_ptr = FunctionFactory::instance().get("if", context);
MultiIfToIfVisitor visitor(std::move(if_function_ptr), std::move(context));

View File

@ -17,7 +17,7 @@ public:
String getDescription() override { return "Optimize multiIf with single condition to if."; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -64,7 +64,7 @@ private:
}
void NormalizeCountVariantsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void NormalizeCountVariantsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
NormalizeCountVariantsVisitor visitor(context);
visitor.visit(query_tree_node);

View File

@ -20,7 +20,7 @@ public:
String getDescription() override { return "Optimize count(literal), sum(1) into count()."; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -215,7 +215,7 @@ private:
}
void OptimizeDateOrDateTimeConverterWithPreimagePass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void OptimizeDateOrDateTimeConverterWithPreimagePass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
OptimizeDateOrDateTimeConverterWithPreimageVisitor visitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -17,7 +17,7 @@ public:
String getDescription() override { return "Replace predicate having Date/DateTime converters with their preimages"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -130,7 +130,7 @@ private:
}
};
void OptimizeGroupByFunctionKeysPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void OptimizeGroupByFunctionKeysPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
OptimizeGroupByFunctionKeysVisitor visitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -16,7 +16,7 @@ public:
String getDescription() override { return "Eliminates functions of other keys in GROUP BY section."; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};
}

View File

@ -115,7 +115,7 @@ private:
}
void OptimizeGroupByInjectiveFunctionsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void OptimizeGroupByInjectiveFunctionsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
OptimizeGroupByInjectiveFunctionsVisitor visitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -14,7 +14,7 @@ public:
String getDescription() override { return "Replaces injective functions by it's arguments in GROUP BY section."; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};
}

View File

@ -124,7 +124,7 @@ private:
}
void OptimizeRedundantFunctionsInOrderByPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void OptimizeRedundantFunctionsInOrderByPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
OptimizeRedundantFunctionsInOrderByVisitor visitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -17,7 +17,7 @@ public:
String getDescription() override { return "If ORDER BY has argument x followed by f(x) transforms it to ORDER BY x."; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};
}

View File

@ -70,7 +70,7 @@ private:
}
void OrderByLimitByDuplicateEliminationPass::run(QueryTreeNodePtr query_tree_node, ContextPtr)
void OrderByLimitByDuplicateEliminationPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr)
{
OrderByLimitByDuplicateEliminationVisitor visitor;
visitor.visit(query_tree_node);

View File

@ -20,7 +20,7 @@ public:
String getDescription() override { return "Remove duplicate columns from ORDER BY, LIMIT BY."; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -50,7 +50,7 @@ public:
}
void OrderByTupleEliminationPass::run(QueryTreeNodePtr query_tree_node, ContextPtr)
void OrderByTupleEliminationPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr)
{
OrderByTupleEliminationVisitor visitor;
visitor.visit(query_tree_node);

View File

@ -17,7 +17,7 @@ public:
String getDescription() override { return "Remove tuple from ORDER BY."; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -79,6 +79,8 @@
#include <Analyzer/QueryTreeBuilder.h>
#include <Analyzer/IQueryTreeNode.h>
#include <Analyzer/Identifier.h>
#include <Poco/Logger.h>
#include <Common/logger_useful.h>
namespace ProfileEvents
{
@ -1066,7 +1068,7 @@ private:
class QueryAnalyzer
{
public:
void resolve(QueryTreeNodePtr node, const QueryTreeNodePtr & table_expression, ContextPtr context)
void resolve(QueryTreeNodePtr & node, const QueryTreeNodePtr & table_expression, ContextPtr context)
{
IdentifierResolveScope scope(node, nullptr /*parent_scope*/);
@ -7649,7 +7651,7 @@ QueryAnalysisPass::QueryAnalysisPass(QueryTreeNodePtr table_expression_)
: table_expression(std::move(table_expression_))
{}
void QueryAnalysisPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void QueryAnalysisPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
QueryAnalyzer analyzer;
analyzer.resolve(query_tree_node, table_expression, context);

View File

@ -89,7 +89,7 @@ public:
return "Resolve type for each query expression. Replace identifiers, matchers with query expressions. Perform constant folding. Evaluate scalar subqueries.";
}
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
private:
QueryTreeNodePtr table_expression;

View File

@ -132,7 +132,7 @@ void updateUsedProjectionIndexes(const QueryTreeNodePtr & query_or_union_node, s
}
void RemoveUnusedProjectionColumnsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void RemoveUnusedProjectionColumnsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
std::vector<QueryTreeNodePtr> nodes_to_visit;
nodes_to_visit.push_back(query_tree_node);

View File

@ -17,7 +17,7 @@ public:
String getDescription() override { return "Remove unused projection columns in subqueries."; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -109,7 +109,7 @@ private:
}
void RewriteAggregateFunctionWithIfPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void RewriteAggregateFunctionWithIfPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
RewriteAggregateFunctionWithIfVisitor visitor(context);
visitor.visit(query_tree_node);

View File

@ -20,7 +20,7 @@ public:
return "Rewrite aggregate functions with if expression as argument when logically equivalent";
}
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -120,7 +120,7 @@ private:
}
void RewriteSumFunctionWithSumAndCountPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void RewriteSumFunctionWithSumAndCountPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
RewriteSumFunctionWithSumAndCountVisitor visitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -20,7 +20,7 @@ public:
String getDescription() override { return "Rewrite sum(column +/- literal) into sum(column) and literal * count(column)"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -58,7 +58,7 @@ public:
}
void ShardNumColumnToFunctionPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void ShardNumColumnToFunctionPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
ShardNumColumnToFunctionVisitor visitor(context);
visitor.visit(query_tree_node);

View File

@ -17,7 +17,7 @@ public:
String getDescription() override { return "Rewrite _shard_num column into shardNum() function"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -178,7 +178,7 @@ private:
}
void SumIfToCountIfPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void SumIfToCountIfPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
SumIfToCountIfVisitor visitor(context);
visitor.visit(query_tree_node);

View File

@ -23,7 +23,7 @@ public:
String getDescription() override { return "Rewrite sum(if) and sumIf into countIf"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -89,7 +89,7 @@ public:
}
void UniqInjectiveFunctionsEliminationPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void UniqInjectiveFunctionsEliminationPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
UniqInjectiveFunctionsEliminationVisitor visitor(std::move(context));
visitor.visit(query_tree_node);

View File

@ -17,7 +17,7 @@ public:
String getDescription() override { return "Remove injective functions from uniq functions arguments."; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};

View File

@ -185,7 +185,7 @@ public:
};
void UniqToCountPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
void UniqToCountPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
UniqToCountVisitor visitor(context);
visitor.visit(query_tree_node);

View File

@ -24,7 +24,7 @@ public:
return "Rewrite uniq and its variants(except uniqUpTo) to count if subquery has distinct or group by clause.";
}
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
};
}

View File

@ -495,8 +495,8 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
return visitFunction(node);
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
"Expected column, constant, function. Actual {}",
node->formatASTForErrorMessage());
"Expected column, constant, function. Actual {} with type: {}",
node->formatASTForErrorMessage(), node_type);
}
PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitColumn(const QueryTreeNodePtr & node)

View File

@ -773,7 +773,8 @@ QueryTreeNodePtr buildQueryTreeDistributed(SelectQueryInfo & query_info,
table_function_node->setTableExpressionModifiers(*table_expression_modifiers);
QueryAnalysisPass query_analysis_pass;
query_analysis_pass.run(table_function_node, query_context);
QueryTreeNodePtr node = table_function_node;
query_analysis_pass.run(node, query_context);
replacement_table_expression = std::move(table_function_node);
}

View File

@ -1,6 +1,13 @@
#include <algorithm>
#include <functional>
#include <iterator>
#include <Analyzer/ConstantNode.h>
#include <Analyzer/ColumnNode.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/IdentifierNode.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/Passes/QueryAnalysisPass.h>
#include <Analyzer/QueryTreeBuilder.h>
#include <Analyzer/TableNode.h>
#include <Analyzer/Utils.h>
#include <Columns/ColumnSet.h>
@ -25,6 +32,7 @@
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTSelectQuery.h>
#include <Planner/PlannerActionsVisitor.h>
#include <Planner/Utils.h>
#include <Processors/ConcatProcessor.h>
#include <Processors/QueryPlan/BuildQueryPipelineSettings.h>
@ -40,6 +48,7 @@
#include <QueryPipeline/narrowPipe.h>
#include <Storages/AlterCommands.h>
#include <Storages/SelectQueryInfo.h>
#include <Storages/StorageDistributed.h>
#include <Storages/StorageFactory.h>
#include <Storages/StorageMerge.h>
#include <Storages/StorageView.h>
@ -51,6 +60,8 @@
#include <Common/assert_cast.h>
#include <Common/checkStackSize.h>
#include <Common/typeid_cast.h>
#include <Core/NamesAndTypes.h>
#include <Functions/FunctionFactory.h>
namespace
{
@ -78,13 +89,13 @@ namespace DB
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int BAD_ARGUMENTS;
extern const int NOT_IMPLEMENTED;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int SAMPLING_NOT_SUPPORTED;
extern const int ALTER_OF_COLUMN_IS_FORBIDDEN;
extern const int CANNOT_EXTRACT_TABLE_STRUCTURE;
extern const int LOGICAL_ERROR;
}
StorageMerge::DatabaseNameOrRegexp::DatabaseNameOrRegexp(
@ -379,7 +390,14 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu
const auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr();
const auto nested_storage_snaphsot = storage->getStorageSnapshot(storage_metadata_snapshot, context);
auto modified_query_info = getModifiedQueryInfo(query_info, context, table, nested_storage_snaphsot);
Names column_names_as_aliases;
Aliases aliases;
Names real_column_names = column_names;
if (child_plan.row_policy_data_opt)
child_plan.row_policy_data_opt->extendNames(real_column_names);
auto modified_query_info = getModifiedQueryInfo(context, table, nested_storage_snaphsot, real_column_names, column_names_as_aliases, aliases);
auto source_pipeline = createSources(
child_plan.plan,
@ -512,7 +530,6 @@ std::vector<ReadFromMerge::ChildPlan> ReadFromMerge::createChildrenPlans(SelectQ
auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr();
auto nested_storage_snaphsot = storage->getStorageSnapshot(storage_metadata_snapshot, context);
auto modified_query_info = getModifiedQueryInfo(query_info, context, table, nested_storage_snaphsot);
Names column_names_as_aliases;
Names real_column_names = column_names;
@ -528,6 +545,8 @@ std::vector<ReadFromMerge::ChildPlan> ReadFromMerge::createChildrenPlans(SelectQ
row_policy_data_opt->extendNames(real_column_names);
}
auto modified_query_info = getModifiedQueryInfo(context, table, nested_storage_snaphsot, real_column_names, column_names_as_aliases, aliases);
if (!context->getSettingsRef().allow_experimental_analyzer)
{
auto storage_columns = storage_metadata_snapshot->getColumns();
@ -580,6 +599,10 @@ std::vector<ReadFromMerge::ChildPlan> ReadFromMerge::createChildrenPlans(SelectQ
column_names_as_aliases.push_back(ExpressionActions::getSmallestColumn(storage_metadata_snapshot->getColumns().getAllPhysical()).name);
}
}
else
{
}
res.back().plan = createPlanForTable(
nested_storage_snaphsot,
@ -596,10 +619,198 @@ std::vector<ReadFromMerge::ChildPlan> ReadFromMerge::createChildrenPlans(SelectQ
return res;
}
SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & query_info,
const ContextPtr & modified_context,
namespace
{
class ApplyAliasColumnExpressionsVisitor : public InDepthQueryTreeVisitor<ApplyAliasColumnExpressionsVisitor>
{
public:
explicit ApplyAliasColumnExpressionsVisitor(QueryTreeNodePtr replacement_table_expression_)
: replacement_table_expression(replacement_table_expression_)
{}
void visitImpl(QueryTreeNodePtr & node)
{
if (auto * column = node->as<ColumnNode>(); column != nullptr)
{
if (column->hasExpression())
{
node = column->getExpressionOrThrow();
node->setAlias(column->getColumnName());
}
else
column->setColumnSource(replacement_table_expression);
}
}
private:
QueryTreeNodePtr replacement_table_expression;
};
bool hasUnknownColumn(const QueryTreeNodePtr & node, QueryTreeNodePtr replacement_table_expression)
{
QueryTreeNodes stack = { node };
while (!stack.empty())
{
auto current = stack.back();
stack.pop_back();
switch (current->getNodeType())
{
case QueryTreeNodeType::CONSTANT:
break;
case QueryTreeNodeType::COLUMN:
{
auto * column_node = current->as<ColumnNode>();
auto source = column_node->getColumnSourceOrNull();
if (source != replacement_table_expression)
return true;
break;
}
default:
{
for (const auto & child : current->getChildren())
{
if (child)
stack.push_back(child);
}
}
}
}
return false;
}
void replaceFilterExpression(
QueryTreeNodePtr & expression,
const QueryTreeNodePtr & replacement_table_expression,
const ContextPtr & context)
{
auto * function = expression->as<FunctionNode>();
if (!function)
return;
if (function->getFunctionName() != "and")
{
if (hasUnknownColumn(expression, replacement_table_expression))
expression = nullptr;
return;
}
QueryTreeNodes conjunctions;
QueryTreeNodes processing{ expression };
while (!processing.empty())
{
auto node = std::move(processing.back());
processing.pop_back();
if (auto * function_node = node->as<FunctionNode>())
{
if (function_node->getFunctionName() == "and")
std::copy(
function_node->getArguments().begin(),
function_node->getArguments().end(),
std::back_inserter(processing)
);
else
conjunctions.push_back(node);
}
else
{
conjunctions.push_back(node);
}
}
std::swap(processing, conjunctions);
for (const auto & node : processing)
{
if (!hasUnknownColumn(node, replacement_table_expression))
conjunctions.push_back(node);
}
if (conjunctions.empty())
{
expression = {};
return;
}
if (conjunctions.size() == 1)
{
expression = conjunctions[0];
return;
}
function->getArguments().getNodes() = std::move(conjunctions);
const auto function_impl = FunctionFactory::instance().get("and", context);
function->resolveAsFunction(function_impl->build(function->getArgumentColumns()));
}
QueryTreeNodePtr replaceTableExpressionAndRemoveJoin(
QueryTreeNodePtr query,
QueryTreeNodePtr original_table_expression,
QueryTreeNodePtr replacement_table_expression,
const ContextPtr & context,
const Names & required_column_names)
{
auto * query_node = query->as<QueryNode>();
auto join_tree_type = query_node->getJoinTree()->getNodeType();
auto modified_query = query_node->cloneAndReplace(original_table_expression, replacement_table_expression);
if (join_tree_type == QueryTreeNodeType::TABLE || join_tree_type == QueryTreeNodeType::TABLE_FUNCTION)
return modified_query;
auto * modified_query_node = modified_query->as<QueryNode>();
modified_query = modified_query->cloneAndReplace(modified_query_node->getJoinTree(), replacement_table_expression);
modified_query_node = modified_query->as<QueryNode>();
query_node = modified_query->as<QueryNode>();
if (query_node->hasPrewhere())
replaceFilterExpression(query_node->getPrewhere(), replacement_table_expression, context);
if (query_node->hasWhere())
replaceFilterExpression(query_node->getWhere(), replacement_table_expression, context);
query_node->getGroupBy().getNodes().clear();
query_node->getHaving() = {};
query_node->getOrderBy().getNodes().clear();
auto & projection = modified_query_node->getProjection().getNodes();
projection.clear();
NamesAndTypes projection_columns;
for (auto const & column_name : required_column_names)
{
QueryTreeNodePtr fake_node = std::make_shared<IdentifierNode>(Identifier{column_name});
QueryAnalysisPass query_analysis_pass(original_table_expression);
query_analysis_pass.run(fake_node, context);
auto * resolved_column = fake_node->as<ColumnNode>();
if (!resolved_column)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Required column '{}' is not resolved", column_name);
auto fake_column = resolved_column->getColumn();
ApplyAliasColumnExpressionsVisitor visitor(replacement_table_expression);
visitor.visit(fake_node);
projection.push_back(fake_node);
projection_columns.push_back(fake_column);
}
query_node->resolveProjectionColumns(std::move(projection_columns));
return modified_query;
}
}
SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_context,
const StorageWithLockAndName & storage_with_lock_and_name,
const StorageSnapshotPtr & storage_snapshot)
const StorageSnapshotPtr & storage_snapshot,
Names required_column_names,
Names & column_names_as_aliases,
Aliases & aliases) const
{
const auto & [database_name, storage, storage_lock, table_name] = storage_with_lock_and_name;
const StorageID current_storage_id = storage->getStorageID();
@ -612,8 +823,7 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & quer
if (query_info.table_expression_modifiers)
replacement_table_expression->setTableExpressionModifiers(*query_info.table_expression_modifiers);
modified_query_info.query_tree = modified_query_info.query_tree->cloneAndReplace(modified_query_info.table_expression,
replacement_table_expression);
modified_query_info.query_tree = replaceTableExpressionAndRemoveJoin(modified_query_info.query_tree, modified_query_info.table_expression, replacement_table_expression, modified_context, required_column_names);
modified_query_info.table_expression = replacement_table_expression;
modified_query_info.planner_context->getOrCreateTableExpressionData(replacement_table_expression);
@ -624,10 +834,65 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & quer
std::unordered_map<std::string, QueryTreeNodePtr> column_name_to_node;
if (!storage_snapshot->tryGetColumn(get_column_options, "_table"))
column_name_to_node.emplace("_table", std::make_shared<ConstantNode>(current_storage_id.table_name));
{
auto table_name_node = std::make_shared<ConstantNode>(current_storage_id.table_name);
table_name_node->setAlias("_table");
column_name_to_node.emplace("_table", table_name_node);
}
if (!storage_snapshot->tryGetColumn(get_column_options, "_database"))
column_name_to_node.emplace("_database", std::make_shared<ConstantNode>(current_storage_id.database_name));
{
auto database_name_node = std::make_shared<ConstantNode>(current_storage_id.database_name);
database_name_node->setAlias("_database");
column_name_to_node.emplace("_database", database_name_node);
}
auto storage_columns = storage_snapshot->metadata->getColumns();
bool with_aliases = /* common_processed_stage == QueryProcessingStage::FetchColumns && */ !storage_columns.getAliases().empty();
if (with_aliases)
{
auto filter_actions_dag = std::make_shared<ActionsDAG>();
for (const auto & column : required_column_names)
{
const auto column_default = storage_columns.getDefault(column);
bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias;
QueryTreeNodePtr column_node;
if (is_alias)
{
QueryTreeNodePtr fake_node = std::make_shared<IdentifierNode>(Identifier{column});
QueryAnalysisPass query_analysis_pass(modified_query_info.table_expression);
query_analysis_pass.run(fake_node, modified_context);
auto * resolved_column = fake_node->as<ColumnNode>();
column_node = fake_node;
ApplyAliasColumnExpressionsVisitor visitor(replacement_table_expression);
visitor.visit(column_node);
if (!resolved_column || !resolved_column->getExpression())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Alias column is not resolved");
column_name_to_node.emplace(column, column_node);
aliases.push_back({ .name = column, .type = resolved_column->getResultType(), .expression = column_node->toAST() });
}
else
{
column_node = std::make_shared<ColumnNode>(NameAndTypePair{column, storage_columns.getColumn(get_column_options, column).type }, modified_query_info.table_expression);
}
PlannerActionsVisitor actions_visitor(modified_query_info.planner_context, false /*use_column_identifier_as_action_node_name*/);
actions_visitor.visit(filter_actions_dag, column_node);
}
column_names_as_aliases = filter_actions_dag->getRequiredColumnsNames();
if (column_names_as_aliases.empty())
column_names_as_aliases.push_back(ExpressionActions::getSmallestColumn(storage_snapshot->metadata->getColumns().getAllPhysical()).name);
}
if (!column_name_to_node.empty())
{
@ -756,7 +1021,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources(
/// Subordinary tables could have different but convertible types, like numeric types of different width.
/// We must return streams with structure equals to structure of Merge table.
convertAndFilterSourceStream(header, storage_snapshot->metadata, aliases, row_policy_data_opt, modified_context, *builder, processed_stage);
convertAndFilterSourceStream(header, modified_query_info, storage_snapshot, aliases, row_policy_data_opt, modified_context, *builder, processed_stage);
}
return builder;
@ -1107,38 +1372,73 @@ void StorageMerge::alter(
void ReadFromMerge::convertAndFilterSourceStream(
const Block & header,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & modified_query_info,
const StorageSnapshotPtr & snapshot,
const Aliases & aliases,
const RowPolicyDataOpt & row_policy_data_opt,
ContextPtr local_context,
ContextMutablePtr local_context,
QueryPipelineBuilder & builder,
QueryProcessingStage::Enum processed_stage)
{
Block before_block_header = builder.getHeader();
auto storage_sample_block = metadata_snapshot->getSampleBlock();
auto storage_sample_block = snapshot->metadata->getSampleBlock();
auto pipe_columns = builder.getHeader().getNamesAndTypesList();
for (const auto & alias : aliases)
if (local_context->getSettingsRef().allow_experimental_analyzer)
{
pipe_columns.emplace_back(NameAndTypePair(alias.name, alias.type));
ASTPtr expr = alias.expression;
auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns);
auto expression_analyzer = ExpressionAnalyzer{alias.expression, syntax_result, local_context};
auto dag = std::make_shared<ActionsDAG>(pipe_columns);
auto actions_dag = expression_analyzer.getActionsDAG(true, false);
auto actions = std::make_shared<ExpressionActions>(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes));
builder.addSimpleTransform([&](const Block & stream_header)
for (const auto & alias : aliases)
{
return std::make_shared<ExpressionTransform>(stream_header, actions);
});
pipe_columns.emplace_back(NameAndTypePair(alias.name, alias.type));
auto actions_dag = std::make_shared<ActionsDAG>(pipe_columns);
QueryTreeNodePtr query_tree = buildQueryTree(alias.expression, local_context);
query_tree->setAlias(alias.name);
QueryAnalysisPass query_analysis_pass(modified_query_info.table_expression);
query_analysis_pass.run(query_tree, local_context);
PlannerActionsVisitor actions_visitor(modified_query_info.planner_context, false /*use_column_identifier_as_action_node_name*/);
const auto & nodes = actions_visitor.visit(actions_dag, query_tree);
if (nodes.size() != 1)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected to have 1 output but got {}", nodes.size());
actions_dag->addOrReplaceInOutputs(actions_dag->addAlias(*nodes.front(), alias.name));
auto actions = std::make_shared<ExpressionActions>(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes));
builder.addSimpleTransform([&](const Block & stream_header)
{
return std::make_shared<ExpressionTransform>(stream_header, actions);
});
}
}
else
{
for (const auto & alias : aliases)
{
pipe_columns.emplace_back(NameAndTypePair(alias.name, alias.type));
ASTPtr expr = alias.expression;
auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns);
auto expression_analyzer = ExpressionAnalyzer{alias.expression, syntax_result, local_context};
auto dag = std::make_shared<ActionsDAG>(pipe_columns);
auto actions_dag = expression_analyzer.getActionsDAG(true, false);
auto actions = std::make_shared<ExpressionActions>(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes));
builder.addSimpleTransform([&](const Block & stream_header)
{
return std::make_shared<ExpressionTransform>(stream_header, actions);
});
}
}
ActionsDAG::MatchColumnsMode convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Name;
if (local_context->getSettingsRef().allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns)
if (local_context->getSettingsRef().allow_experimental_analyzer
&& (processed_stage != QueryProcessingStage::FetchColumns || dynamic_cast<const StorageDistributed *>(&snapshot->storage) != nullptr))
convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position;
if (row_policy_data_opt)

View File

@ -189,6 +189,13 @@ private:
using Aliases = std::vector<AliasData>;
SelectQueryInfo getModifiedQueryInfo(const ContextPtr & modified_context,
const StorageWithLockAndName & storage_with_lock_and_name,
const StorageSnapshotPtr & storage_snapshot,
Names required_column_names,
Names & column_names_as_aliases,
Aliases & aliases) const;
/// An object of this helper class is created
/// when processing a Merge table data source (subordinary table)
/// that has row policies
@ -261,17 +268,13 @@ private:
ContextMutablePtr modified_context,
bool concat_streams = false) const;
static SelectQueryInfo getModifiedQueryInfo(const SelectQueryInfo & query_info,
const ContextPtr & modified_context,
const StorageWithLockAndName & storage_with_lock_and_name,
const StorageSnapshotPtr & storage_snapshot);
static void convertAndFilterSourceStream(
const Block & header,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & modified_query_info,
const StorageSnapshotPtr & snapshot,
const Aliases & aliases,
const RowPolicyDataOpt & row_policy_data_opt,
ContextPtr context,
ContextMutablePtr context,
QueryPipelineBuilder & builder,
QueryProcessingStage::Enum processed_stage);

View File

@ -4,10 +4,8 @@
01062_pm_all_join_with_block_continuation
01083_expressions_in_engine_arguments
01155_rename_move_materialized_view
01214_test_storage_merge_aliases_with_where
01244_optimize_distributed_group_by_sharding_key
01268_shard_avgweighted
01560_merge_distributed_join
01584_distributed_buffer_cannot_find_column
01624_soft_constraints
01656_test_query_log_factories_info
@ -16,13 +14,11 @@
01747_join_view_filter_dictionary
01761_cast_to_enum_nullable
01925_join_materialized_columns
01925_test_storage_merge_aliases
01952_optimize_distributed_group_by_sharding_key
02174_cte_scalar_cache_mv
02354_annoy
02428_parameterized_view
02493_inconsistent_hex_and_binary_number
02575_merge_prewhere_different_default_kind
02725_agg_projection_resprect_PK
02763_row_policy_storage_merge_alias
02818_parameterized_view_with_cte_multiple_usage