From 26c9042ea0f0529f464435cbeef111f3e6d396a5 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 12 Jun 2023 17:06:52 +0000 Subject: [PATCH 01/18] Analyzer: support aliases in StorageMerge --- src/Analyzer/IQueryTreePass.h | 2 +- ...egateFunctionsArithmericOperationsPass.cpp | 2 +- ...gregateFunctionsArithmericOperationsPass.h | 2 +- src/Analyzer/Passes/ArrayExistsToHasPass.cpp | 2 +- src/Analyzer/Passes/ArrayExistsToHasPass.h | 2 +- src/Analyzer/Passes/AutoFinalOnQueryPass.cpp | 2 +- src/Analyzer/Passes/AutoFinalOnQueryPass.h | 2 +- .../Passes/ComparisonTupleEliminationPass.cpp | 2 +- .../Passes/ComparisonTupleEliminationPass.h | 2 +- .../Passes/ConvertOrLikeChainPass.cpp | 2 +- src/Analyzer/Passes/ConvertOrLikeChainPass.h | 2 +- src/Analyzer/Passes/ConvertQueryToCNFPass.cpp | 2 +- src/Analyzer/Passes/ConvertQueryToCNFPass.h | 2 +- src/Analyzer/Passes/CountDistinctPass.cpp | 2 +- src/Analyzer/Passes/CountDistinctPass.h | 2 +- src/Analyzer/Passes/CrossToInnerJoinPass.cpp | 2 +- src/Analyzer/Passes/CrossToInnerJoinPass.h | 2 +- .../Passes/FunctionToSubcolumnsPass.cpp | 2 +- .../Passes/FunctionToSubcolumnsPass.h | 2 +- src/Analyzer/Passes/FuseFunctionsPass.cpp | 2 +- src/Analyzer/Passes/FuseFunctionsPass.h | 2 +- .../Passes/GroupingFunctionsResolvePass.cpp | 2 +- .../Passes/GroupingFunctionsResolvePass.h | 2 +- src/Analyzer/Passes/IfChainToMultiIfPass.cpp | 2 +- src/Analyzer/Passes/IfChainToMultiIfPass.h | 2 +- .../Passes/IfConstantConditionPass.cpp | 2 +- src/Analyzer/Passes/IfConstantConditionPass.h | 2 +- .../Passes/IfTransformStringsToEnumPass.cpp | 2 +- .../Passes/IfTransformStringsToEnumPass.h | 2 +- .../Passes/LogicalExpressionOptimizerPass.cpp | 2 +- .../Passes/LogicalExpressionOptimizerPass.h | 2 +- src/Analyzer/Passes/MultiIfToIfPass.cpp | 2 +- src/Analyzer/Passes/MultiIfToIfPass.h | 2 +- .../Passes/NormalizeCountVariantsPass.cpp | 2 +- .../Passes/NormalizeCountVariantsPass.h | 2 +- .../OptimizeGroupByFunctionKeysPass.cpp | 2 +- .../Passes/OptimizeGroupByFunctionKeysPass.h | 2 +- ...ptimizeRedundantFunctionsInOrderByPass.cpp | 2 +- .../OptimizeRedundantFunctionsInOrderByPass.h | 2 +- ...OrderByLimitByDuplicateEliminationPass.cpp | 2 +- .../OrderByLimitByDuplicateEliminationPass.h | 2 +- .../Passes/OrderByTupleEliminationPass.cpp | 2 +- .../Passes/OrderByTupleEliminationPass.h | 2 +- src/Analyzer/Passes/QueryAnalysisPass.cpp | 15 ++- src/Analyzer/Passes/QueryAnalysisPass.h | 2 +- .../RewriteAggregateFunctionWithIfPass.cpp | 2 +- .../RewriteAggregateFunctionWithIfPass.h | 2 +- .../Passes/ShardNumColumnToFunctionPass.cpp | 2 +- .../Passes/ShardNumColumnToFunctionPass.h | 2 +- src/Analyzer/Passes/SumIfToCountIfPass.cpp | 2 +- src/Analyzer/Passes/SumIfToCountIfPass.h | 2 +- .../UniqInjectiveFunctionsEliminationPass.cpp | 2 +- .../UniqInjectiveFunctionsEliminationPass.h | 2 +- src/Planner/PlannerActionsVisitor.cpp | 4 +- src/Storages/StorageDistributed.cpp | 4 +- src/Storages/StorageMerge.cpp | 121 +++++++++++++++--- src/Storages/StorageMerge.h | 9 +- 57 files changed, 177 insertions(+), 80 deletions(-) diff --git a/src/Analyzer/IQueryTreePass.h b/src/Analyzer/IQueryTreePass.h index 4293934c32d..d4499c3271c 100644 --- a/src/Analyzer/IQueryTreePass.h +++ b/src/Analyzer/IQueryTreePass.h @@ -31,7 +31,7 @@ public: virtual String getDescription() = 0; /// Run pass over query tree - virtual void run(QueryTreeNodePtr query_tree_node, ContextPtr context) = 0; + virtual void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) = 0; }; diff --git a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp index 1476a66c892..2a69292ff78 100644 --- a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp +++ b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp @@ -201,7 +201,7 @@ private: } -void AggregateFunctionsArithmericOperationsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void AggregateFunctionsArithmericOperationsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { AggregateFunctionsArithmericOperationsVisitor visitor(std::move(context)); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.h b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.h index a89d2f87ad9..d510b62f9be 100644 --- a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.h +++ b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.h @@ -17,7 +17,7 @@ public: String getDescription() override { return "Extract arithmeric operations from aggregate functions."; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/ArrayExistsToHasPass.cpp b/src/Analyzer/Passes/ArrayExistsToHasPass.cpp index c0f958588f1..63d417cd570 100644 --- a/src/Analyzer/Passes/ArrayExistsToHasPass.cpp +++ b/src/Analyzer/Passes/ArrayExistsToHasPass.cpp @@ -92,7 +92,7 @@ public: } -void RewriteArrayExistsToHasPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void RewriteArrayExistsToHasPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { RewriteArrayExistsToHasVisitor visitor(context); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/ArrayExistsToHasPass.h b/src/Analyzer/Passes/ArrayExistsToHasPass.h index 8f4623116e3..4795b61c625 100644 --- a/src/Analyzer/Passes/ArrayExistsToHasPass.h +++ b/src/Analyzer/Passes/ArrayExistsToHasPass.h @@ -20,7 +20,7 @@ public: String getDescription() override { return "Rewrite arrayExists(func, arr) functions to has(arr, elem) when logically equivalent"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; } diff --git a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp index 15326ca1dc8..ee9e1023949 100644 --- a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp +++ b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp @@ -67,7 +67,7 @@ private: } -void AutoFinalOnQueryPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void AutoFinalOnQueryPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { auto visitor = AutoFinalOnQueryPassVisitor(std::move(context)); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/AutoFinalOnQueryPass.h b/src/Analyzer/Passes/AutoFinalOnQueryPass.h index 3489597108c..d595b98d349 100644 --- a/src/Analyzer/Passes/AutoFinalOnQueryPass.h +++ b/src/Analyzer/Passes/AutoFinalOnQueryPass.h @@ -25,7 +25,7 @@ public: return "Automatically applies final modifier to table expressions in queries if it is supported and if user level final setting is set"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; } diff --git a/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp b/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp index 4e0562a2fe8..57920065513 100644 --- a/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp +++ b/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp @@ -201,7 +201,7 @@ private: } -void ComparisonTupleEliminationPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void ComparisonTupleEliminationPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { ComparisonTupleEliminationPassVisitor visitor(std::move(context)); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/ComparisonTupleEliminationPass.h b/src/Analyzer/Passes/ComparisonTupleEliminationPass.h index 954a9d6a2f0..7f4245e2d95 100644 --- a/src/Analyzer/Passes/ComparisonTupleEliminationPass.h +++ b/src/Analyzer/Passes/ComparisonTupleEliminationPass.h @@ -17,7 +17,7 @@ public: String getDescription() override { return "Rewrite tuples comparison into equivalent comparison of tuples arguments"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/ConvertOrLikeChainPass.cpp b/src/Analyzer/Passes/ConvertOrLikeChainPass.cpp index 7d7362fb742..0d2ddd20374 100644 --- a/src/Analyzer/Passes/ConvertOrLikeChainPass.cpp +++ b/src/Analyzer/Passes/ConvertOrLikeChainPass.cpp @@ -132,7 +132,7 @@ private: } -void ConvertOrLikeChainPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void ConvertOrLikeChainPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { auto or_function_resolver = FunctionFactory::instance().get("or", context); auto match_function_resolver = FunctionFactory::instance().get("multiMatchAny", context); diff --git a/src/Analyzer/Passes/ConvertOrLikeChainPass.h b/src/Analyzer/Passes/ConvertOrLikeChainPass.h index 0f734bfa73d..90bccaa0e8d 100644 --- a/src/Analyzer/Passes/ConvertOrLikeChainPass.h +++ b/src/Analyzer/Passes/ConvertOrLikeChainPass.h @@ -14,7 +14,7 @@ public: String getDescription() override { return "Replaces all the 'or's with {i}like to multiMatchAny"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; } diff --git a/src/Analyzer/Passes/ConvertQueryToCNFPass.cpp b/src/Analyzer/Passes/ConvertQueryToCNFPass.cpp index 4d32c96b845..ecba2e28749 100644 --- a/src/Analyzer/Passes/ConvertQueryToCNFPass.cpp +++ b/src/Analyzer/Passes/ConvertQueryToCNFPass.cpp @@ -720,7 +720,7 @@ public: } -void ConvertLogicalExpressionToCNFPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void ConvertLogicalExpressionToCNFPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { const auto & settings = context->getSettingsRef(); if (!settings.convert_query_to_cnf) diff --git a/src/Analyzer/Passes/ConvertQueryToCNFPass.h b/src/Analyzer/Passes/ConvertQueryToCNFPass.h index 5ed874db006..60943c04d78 100644 --- a/src/Analyzer/Passes/ConvertQueryToCNFPass.h +++ b/src/Analyzer/Passes/ConvertQueryToCNFPass.h @@ -12,7 +12,7 @@ public: String getDescription() override { return "Convert logical expression to CNF and apply optimizations using constraints"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; } diff --git a/src/Analyzer/Passes/CountDistinctPass.cpp b/src/Analyzer/Passes/CountDistinctPass.cpp index 945295f5cbc..eb2859020be 100644 --- a/src/Analyzer/Passes/CountDistinctPass.cpp +++ b/src/Analyzer/Passes/CountDistinctPass.cpp @@ -84,7 +84,7 @@ public: } -void CountDistinctPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void CountDistinctPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { CountDistinctVisitor visitor(std::move(context)); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/CountDistinctPass.h b/src/Analyzer/Passes/CountDistinctPass.h index cac5033c98f..33728b0228c 100644 --- a/src/Analyzer/Passes/CountDistinctPass.h +++ b/src/Analyzer/Passes/CountDistinctPass.h @@ -20,7 +20,7 @@ public: return "Optimize single countDistinct into count over subquery"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp index d4877d23f28..3283c163890 100644 --- a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp +++ b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp @@ -264,7 +264,7 @@ private: } -void CrossToInnerJoinPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void CrossToInnerJoinPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { CrossToInnerJoinVisitor visitor(std::move(context)); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/CrossToInnerJoinPass.h b/src/Analyzer/Passes/CrossToInnerJoinPass.h index 127d26dc41d..b0437c562ac 100644 --- a/src/Analyzer/Passes/CrossToInnerJoinPass.h +++ b/src/Analyzer/Passes/CrossToInnerJoinPass.h @@ -22,7 +22,7 @@ public: return "Replace CROSS JOIN with INNER JOIN"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; } diff --git a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp index 696483862e0..1b04136e6a4 100644 --- a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp +++ b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp @@ -202,7 +202,7 @@ private: } -void FunctionToSubcolumnsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void FunctionToSubcolumnsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { FunctionToSubcolumnsVisitor visitor(context); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/FunctionToSubcolumnsPass.h b/src/Analyzer/Passes/FunctionToSubcolumnsPass.h index 0e1d2583e7b..d4edcc5b922 100644 --- a/src/Analyzer/Passes/FunctionToSubcolumnsPass.h +++ b/src/Analyzer/Passes/FunctionToSubcolumnsPass.h @@ -24,7 +24,7 @@ public: String getDescription() override { return "Rewrite function to subcolumns, for example tupleElement(column, subcolumn) into column.subcolumn"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/FuseFunctionsPass.cpp b/src/Analyzer/Passes/FuseFunctionsPass.cpp index 14082697955..ef87528964c 100644 --- a/src/Analyzer/Passes/FuseFunctionsPass.cpp +++ b/src/Analyzer/Passes/FuseFunctionsPass.cpp @@ -254,7 +254,7 @@ void tryFuseQuantiles(QueryTreeNodePtr query_tree_node, ContextPtr context) } -void FuseFunctionsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void FuseFunctionsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { tryFuseSumCountAvg(query_tree_node, context); tryFuseQuantiles(query_tree_node, context); diff --git a/src/Analyzer/Passes/FuseFunctionsPass.h b/src/Analyzer/Passes/FuseFunctionsPass.h index a92b77b1115..2fd85da4747 100644 --- a/src/Analyzer/Passes/FuseFunctionsPass.h +++ b/src/Analyzer/Passes/FuseFunctionsPass.h @@ -20,7 +20,7 @@ public: String getDescription() override { return "Replaces several calls of aggregate functions of the same family into one call"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; } diff --git a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp index 0cf5310a3ad..774014e5ffd 100644 --- a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp +++ b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp @@ -248,7 +248,7 @@ private: } -void GroupingFunctionsResolvePass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void GroupingFunctionsResolvePass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { GroupingFunctionsResolveVisitor visitor(std::move(context)); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/GroupingFunctionsResolvePass.h b/src/Analyzer/Passes/GroupingFunctionsResolvePass.h index 070c8dd9389..cd932f76977 100644 --- a/src/Analyzer/Passes/GroupingFunctionsResolvePass.h +++ b/src/Analyzer/Passes/GroupingFunctionsResolvePass.h @@ -24,7 +24,7 @@ public: String getDescription() override { return "Resolve GROUPING functions based on GROUP BY modifiers"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp index 1f97e012331..91a5709f142 100644 --- a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp +++ b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp @@ -73,7 +73,7 @@ private: } -void IfChainToMultiIfPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void IfChainToMultiIfPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { auto multi_if_function_ptr = FunctionFactory::instance().get("multiIf", context); IfChainToMultiIfPassVisitor visitor(std::move(multi_if_function_ptr), std::move(context)); diff --git a/src/Analyzer/Passes/IfChainToMultiIfPass.h b/src/Analyzer/Passes/IfChainToMultiIfPass.h index 43f3fb8831d..9e7335d93e4 100644 --- a/src/Analyzer/Passes/IfChainToMultiIfPass.h +++ b/src/Analyzer/Passes/IfChainToMultiIfPass.h @@ -18,7 +18,7 @@ public: String getDescription() override { return "Optimize if chain to multiIf"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/IfConstantConditionPass.cpp b/src/Analyzer/Passes/IfConstantConditionPass.cpp index 6f9cfe482f1..35c6718f018 100644 --- a/src/Analyzer/Passes/IfConstantConditionPass.cpp +++ b/src/Analyzer/Passes/IfConstantConditionPass.cpp @@ -49,7 +49,7 @@ public: } -void IfConstantConditionPass::run(QueryTreeNodePtr query_tree_node, ContextPtr) +void IfConstantConditionPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr) { IfConstantConditionVisitor visitor; visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/IfConstantConditionPass.h b/src/Analyzer/Passes/IfConstantConditionPass.h index 7817e67aa5e..7548fc702bc 100644 --- a/src/Analyzer/Passes/IfConstantConditionPass.h +++ b/src/Analyzer/Passes/IfConstantConditionPass.h @@ -21,7 +21,7 @@ public: String getDescription() override { return "Optimize if, multiIf for constant condition."; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp b/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp index 562aff4cf05..32e3c3cda51 100644 --- a/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp +++ b/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp @@ -205,7 +205,7 @@ public: } -void IfTransformStringsToEnumPass::run(QueryTreeNodePtr query, ContextPtr context) +void IfTransformStringsToEnumPass::run(QueryTreeNodePtr & query, ContextPtr context) { ConvertStringsToEnumVisitor visitor(std::move(context)); visitor.visit(query); diff --git a/src/Analyzer/Passes/IfTransformStringsToEnumPass.h b/src/Analyzer/Passes/IfTransformStringsToEnumPass.h index a4a014967e0..522087aafae 100644 --- a/src/Analyzer/Passes/IfTransformStringsToEnumPass.h +++ b/src/Analyzer/Passes/IfTransformStringsToEnumPass.h @@ -33,7 +33,7 @@ public: String getDescription() override { return "Replaces string-type arguments in If and Transform to enum"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; } diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 13f8025f5ea..7e0b6b2f828 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -233,7 +233,7 @@ private: } }; -void LogicalExpressionOptimizerPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void LogicalExpressionOptimizerPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { LogicalExpressionOptimizerVisitor visitor(std::move(context)); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h index 05c10ddc685..51d9968b48c 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h @@ -76,7 +76,7 @@ public: String getDescription() override { return "Transform equality chain to a single IN function or a constant if possible"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; } diff --git a/src/Analyzer/Passes/MultiIfToIfPass.cpp b/src/Analyzer/Passes/MultiIfToIfPass.cpp index 4672351bcfb..5012aa7fa78 100644 --- a/src/Analyzer/Passes/MultiIfToIfPass.cpp +++ b/src/Analyzer/Passes/MultiIfToIfPass.cpp @@ -43,7 +43,7 @@ private: } -void MultiIfToIfPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void MultiIfToIfPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { auto if_function_ptr = FunctionFactory::instance().get("if", context); MultiIfToIfVisitor visitor(std::move(if_function_ptr), std::move(context)); diff --git a/src/Analyzer/Passes/MultiIfToIfPass.h b/src/Analyzer/Passes/MultiIfToIfPass.h index 2213f3713ed..e3c03913aaa 100644 --- a/src/Analyzer/Passes/MultiIfToIfPass.h +++ b/src/Analyzer/Passes/MultiIfToIfPass.h @@ -17,7 +17,7 @@ public: String getDescription() override { return "Optimize multiIf with single condition to if."; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp b/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp index d36be98751c..20b308c3af6 100644 --- a/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp +++ b/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp @@ -64,7 +64,7 @@ private: } -void NormalizeCountVariantsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void NormalizeCountVariantsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { NormalizeCountVariantsVisitor visitor(context); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/NormalizeCountVariantsPass.h b/src/Analyzer/Passes/NormalizeCountVariantsPass.h index 78a114f4a85..6cf9f34619a 100644 --- a/src/Analyzer/Passes/NormalizeCountVariantsPass.h +++ b/src/Analyzer/Passes/NormalizeCountVariantsPass.h @@ -20,7 +20,7 @@ public: String getDescription() override { return "Optimize count(literal), sum(1) into count()."; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp index 5ed52f1210b..7c851d5fc35 100644 --- a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp @@ -130,7 +130,7 @@ private: } }; -void OptimizeGroupByFunctionKeysPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void OptimizeGroupByFunctionKeysPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { OptimizeGroupByFunctionKeysVisitor visitor(std::move(context)); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.h b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.h index 632960c45bb..fd5eadcb796 100644 --- a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.h +++ b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.h @@ -16,7 +16,7 @@ public: String getDescription() override { return "Eliminates functions of other keys in GROUP BY section."; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; } diff --git a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp index c6d312d0ecf..b6cc50caffe 100644 --- a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp +++ b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp @@ -124,7 +124,7 @@ private: } -void OptimizeRedundantFunctionsInOrderByPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void OptimizeRedundantFunctionsInOrderByPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { OptimizeRedundantFunctionsInOrderByVisitor visitor(std::move(context)); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h index 609a6360d27..4a63c78022b 100644 --- a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h +++ b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h @@ -17,7 +17,7 @@ public: String getDescription() override { return "If ORDER BY has argument x followed by f(x) transforms it to ORDER BY x."; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; } diff --git a/src/Analyzer/Passes/OrderByLimitByDuplicateEliminationPass.cpp b/src/Analyzer/Passes/OrderByLimitByDuplicateEliminationPass.cpp index 3632c41028b..26ca5984b49 100644 --- a/src/Analyzer/Passes/OrderByLimitByDuplicateEliminationPass.cpp +++ b/src/Analyzer/Passes/OrderByLimitByDuplicateEliminationPass.cpp @@ -70,7 +70,7 @@ private: } -void OrderByLimitByDuplicateEliminationPass::run(QueryTreeNodePtr query_tree_node, ContextPtr) +void OrderByLimitByDuplicateEliminationPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr) { OrderByLimitByDuplicateEliminationVisitor visitor; visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/OrderByLimitByDuplicateEliminationPass.h b/src/Analyzer/Passes/OrderByLimitByDuplicateEliminationPass.h index 11a025af5b9..de5e1898a4c 100644 --- a/src/Analyzer/Passes/OrderByLimitByDuplicateEliminationPass.h +++ b/src/Analyzer/Passes/OrderByLimitByDuplicateEliminationPass.h @@ -20,7 +20,7 @@ public: String getDescription() override { return "Remove duplicate columns from ORDER BY, LIMIT BY."; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/OrderByTupleEliminationPass.cpp b/src/Analyzer/Passes/OrderByTupleEliminationPass.cpp index f70ec27ba5d..7c106082124 100644 --- a/src/Analyzer/Passes/OrderByTupleEliminationPass.cpp +++ b/src/Analyzer/Passes/OrderByTupleEliminationPass.cpp @@ -50,7 +50,7 @@ public: } -void OrderByTupleEliminationPass::run(QueryTreeNodePtr query_tree_node, ContextPtr) +void OrderByTupleEliminationPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr) { OrderByTupleEliminationVisitor visitor; visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/OrderByTupleEliminationPass.h b/src/Analyzer/Passes/OrderByTupleEliminationPass.h index 5665561e227..45c8a756795 100644 --- a/src/Analyzer/Passes/OrderByTupleEliminationPass.h +++ b/src/Analyzer/Passes/OrderByTupleEliminationPass.h @@ -17,7 +17,7 @@ public: String getDescription() override { return "Remove tuple from ORDER BY."; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index c454ad9f84f..1a76bc762a4 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -77,6 +77,8 @@ #include #include #include +#include +#include namespace ProfileEvents { @@ -1056,7 +1058,7 @@ private: class QueryAnalyzer { public: - void resolve(QueryTreeNodePtr node, const QueryTreeNodePtr & table_expression, ContextPtr context) + void resolve(QueryTreeNodePtr & node, const QueryTreeNodePtr & table_expression, ContextPtr context) { IdentifierResolveScope scope(node, nullptr /*parent_scope*/); @@ -1097,6 +1099,7 @@ public: { if (table_expression) { + LOG_DEBUG(&Poco::Logger::get("resolve"), "Table expression: {}", table_expression->dumpTree()); scope.expression_join_tree_node = table_expression; validateTableExpressionModifiers(scope.expression_join_tree_node, scope); initializeTableExpressionData(scope.expression_join_tree_node, scope); @@ -1106,6 +1109,7 @@ public: resolveExpressionNodeList(node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); else resolveExpressionNode(node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + LOG_DEBUG(&Poco::Logger::get("resolve"), "Result: {}", node->dumpTree()); break; } @@ -2677,6 +2681,7 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromAliases(const Identifier */ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableColumns(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope) { + LOG_DEBUG(&Poco::Logger::get("tryResolveIdentifierFromTableColumns"), "{} {}", scope.column_name_to_column_node.size(), !identifier_lookup.isExpressionLookup()); if (scope.column_name_to_column_node.empty() || !identifier_lookup.isExpressionLookup()) return {}; @@ -2836,11 +2841,14 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id QueryTreeNodePtr result_expression; bool match_full_identifier = false; + LOG_DEBUG(&Poco::Logger::get("resolve_identifier_from_storage_or_throw"), "Looking for id: {}", identifier_without_column_qualifier.getFullName()); + auto it = table_expression_data.column_name_to_column_node.find(identifier_without_column_qualifier.getFullName()); if (it != table_expression_data.column_name_to_column_node.end()) { match_full_identifier = true; result_expression = it->second; + LOG_DEBUG(&Poco::Logger::get("resolve_identifier_from_storage_or_throw"), "Found: {}", result_expression->dumpTree()); } else { @@ -5389,6 +5397,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id auto unresolved_identifier = identifier_node.getIdentifier(); auto resolve_identifier_expression_result = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::EXPRESSION}, scope); auto resolved_identifier_node = resolve_identifier_expression_result.resolved_identifier; + LOG_DEBUG(&Poco::Logger::get("resolveExpressionNode"), "Resolved: {}", resolved_identifier_node ? resolved_identifier_node->dumpTree() : "Not resolved"); if (resolved_identifier_node && result_projection_names.empty() && (resolve_identifier_expression_result.isResolvedFromJoinTree() || resolve_identifier_expression_result.isResolvedFromExpressionArguments())) @@ -5470,6 +5479,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id } node = std::move(resolved_identifier_node); + LOG_DEBUG(&Poco::Logger::get("resolveExpressionNode"), "Result node: {}", node ? node->dumpTree() : "Not resolved"); if (node->getNodeType() == QueryTreeNodeType::LIST) { @@ -6173,6 +6183,7 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table table_expression_data.should_qualify_columns = false; } + LOG_DEBUG(&Poco::Logger::get("Analyzer"), "Table data: {}", table_expression_data.dump()); scope.table_expression_node_to_data.emplace(table_expression_node, std::move(table_expression_data)); } @@ -7152,7 +7163,7 @@ QueryAnalysisPass::QueryAnalysisPass(QueryTreeNodePtr table_expression_) : table_expression(std::move(table_expression_)) {} -void QueryAnalysisPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void QueryAnalysisPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { QueryAnalyzer analyzer; analyzer.resolve(query_tree_node, table_expression, context); diff --git a/src/Analyzer/Passes/QueryAnalysisPass.h b/src/Analyzer/Passes/QueryAnalysisPass.h index fa8778ebf76..5d335d3e712 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.h +++ b/src/Analyzer/Passes/QueryAnalysisPass.h @@ -89,7 +89,7 @@ public: return "Resolve type for each query expression. Replace identifiers, matchers with query expressions. Perform constant folding. Evaluate scalar subqueries."; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; private: QueryTreeNodePtr table_expression; diff --git a/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp b/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp index de264948d4c..2fe5a89578b 100644 --- a/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp +++ b/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp @@ -108,7 +108,7 @@ private: } -void RewriteAggregateFunctionWithIfPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void RewriteAggregateFunctionWithIfPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { RewriteAggregateFunctionWithIfVisitor visitor(context); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.h b/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.h index be8ad3ac34d..0a2fc1ba423 100644 --- a/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.h +++ b/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.h @@ -20,7 +20,7 @@ public: return "Rewrite aggregate functions with if expression as argument when logically equivalent"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/ShardNumColumnToFunctionPass.cpp b/src/Analyzer/Passes/ShardNumColumnToFunctionPass.cpp index b28816e8ff3..c273aecc9b5 100644 --- a/src/Analyzer/Passes/ShardNumColumnToFunctionPass.cpp +++ b/src/Analyzer/Passes/ShardNumColumnToFunctionPass.cpp @@ -58,7 +58,7 @@ public: } -void ShardNumColumnToFunctionPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void ShardNumColumnToFunctionPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { ShardNumColumnToFunctionVisitor visitor(context); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/ShardNumColumnToFunctionPass.h b/src/Analyzer/Passes/ShardNumColumnToFunctionPass.h index 71a038bcf39..248f4e29bbe 100644 --- a/src/Analyzer/Passes/ShardNumColumnToFunctionPass.h +++ b/src/Analyzer/Passes/ShardNumColumnToFunctionPass.h @@ -17,7 +17,7 @@ public: String getDescription() override { return "Rewrite _shard_num column into shardNum() function"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/SumIfToCountIfPass.cpp b/src/Analyzer/Passes/SumIfToCountIfPass.cpp index d55af278152..04d6c134d10 100644 --- a/src/Analyzer/Passes/SumIfToCountIfPass.cpp +++ b/src/Analyzer/Passes/SumIfToCountIfPass.cpp @@ -180,7 +180,7 @@ private: } -void SumIfToCountIfPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void SumIfToCountIfPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { SumIfToCountIfVisitor visitor(context); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/SumIfToCountIfPass.h b/src/Analyzer/Passes/SumIfToCountIfPass.h index f3ba47f1c2c..439d80c6306 100644 --- a/src/Analyzer/Passes/SumIfToCountIfPass.h +++ b/src/Analyzer/Passes/SumIfToCountIfPass.h @@ -23,7 +23,7 @@ public: String getDescription() override { return "Rewrite sum(if) and sumIf into countIf"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp index 5c4484457e8..e256934010d 100644 --- a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp +++ b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp @@ -87,7 +87,7 @@ public: } -void UniqInjectiveFunctionsEliminationPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void UniqInjectiveFunctionsEliminationPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { UniqInjectiveFunctionsEliminationVisitor visitor(std::move(context)); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.h b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.h index a0f07dfb7b5..c143fe2c39c 100644 --- a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.h +++ b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.h @@ -17,7 +17,7 @@ public: String getDescription() override { return "Remove injective functions from uniq functions arguments."; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index c64d82299ca..e9fa72f925d 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -494,8 +494,8 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi return visitFunction(node); throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Expected column, constant, function. Actual {}", - node->formatASTForErrorMessage()); + "Expected column, constant, function. Actual {} with type: {}", + node->formatASTForErrorMessage(), node_type); } PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitColumn(const QueryTreeNodePtr & node) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index b91ad0b963a..9f9f0fda9e2 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -30,6 +30,7 @@ #include #include #include +#include "Analyzer/IQueryTreeNode.h" #include #include @@ -937,7 +938,8 @@ QueryTreeNodePtr buildQueryTreeDistributed(SelectQueryInfo & query_info, table_function_node->setTableExpressionModifiers(*table_expression_modifiers); QueryAnalysisPass query_analysis_pass; - query_analysis_pass.run(table_function_node, query_context); + QueryTreeNodePtr node = table_function_node; + query_analysis_pass.run(node, query_context); replacement_table_expression = std::move(table_function_node); } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index b0ed242d14d..a49155ac2d9 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -27,9 +27,18 @@ #include #include #include +#include "Common/logger_useful.h" #include #include +#include "Analyzer/ColumnNode.h" +#include "Analyzer/IQueryTreeNode.h" +#include "Analyzer/Identifier.h" +#include "Analyzer/IdentifierNode.h" +#include "Analyzer/Passes/QueryAnalysisPass.h" +#include "Analyzer/QueryTreeBuilder.h" +#include "Core/NamesAndTypes.h" #include "DataTypes/IDataType.h" +#include "Planner/PlannerActionsVisitor.h" #include #include #include @@ -42,6 +51,7 @@ #include #include #include +#include namespace @@ -464,8 +474,8 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); auto nested_storage_snaphsot = storage->getStorageSnapshot(storage_metadata_snapshot, context); - auto modified_query_info = getModifiedQueryInfo(query_info, context, table, nested_storage_snaphsot); Names column_names_as_aliases; + auto modified_query_info = getModifiedQueryInfo(context, table, nested_storage_snaphsot, column_names_as_aliases); if (!context->getSettingsRef().allow_experimental_analyzer) { @@ -553,10 +563,10 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu pipeline.addResources(std::move(resources)); } -SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & query_info, - const ContextPtr & modified_context, +SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_context, const StorageWithLockAndName & storage_with_lock_and_name, - const StorageSnapshotPtr & storage_snapshot) + const StorageSnapshotPtr & storage_snapshot, + Names & column_names_as_aliases) const { const auto & [database_name, storage, storage_lock, table_name] = storage_with_lock_and_name; const StorageID current_storage_id = storage->getStorageID(); @@ -586,6 +596,47 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & quer if (!storage_snapshot->tryGetColumn(get_column_options, "_database")) column_name_to_node.emplace("_database", std::make_shared(current_storage_id.database_name)); + auto storage_columns = storage_snapshot->metadata->getColumns(); + + bool with_aliases = /* common_processed_stage == QueryProcessingStage::FetchColumns && */ !storage_columns.getAliases().empty(); + if (with_aliases) + { + auto filter_actions_dag = std::make_shared(); + for (const auto & column : column_names) + { + const auto column_default = storage_columns.getDefault(column); + bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias; + + QueryTreeNodePtr column_node; + + if (is_alias) + { + column_node = buildQueryTree(column_default->expression, modified_context); + + LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "QT before: {}\n{}", column_node->dumpTree(), modified_query_info.table_expression->dumpTree()); + + column_node->setAlias(column); + + QueryAnalysisPass query_analysis_pass(modified_query_info.table_expression); + query_analysis_pass.run(column_node, modified_context); + + LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "QT after: {}", column_node->dumpTree()); + + column_name_to_node.emplace(column, column_node); + } + else + { + column_node = std::make_shared(NameAndTypePair{column, storage_columns.getColumn(get_column_options, column).type }, modified_query_info.table_expression); + } + + + PlannerActionsVisitor actions_visitor(modified_query_info.planner_context, false /*use_column_identifier_as_action_node_name*/); + actions_visitor.visit(filter_actions_dag, column_node); + } + column_names_as_aliases = filter_actions_dag->getRequiredColumnsNames(); + LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "Required names: {}", toString(column_names_as_aliases)); + } + if (!column_name_to_node.empty()) { replaceColumns(modified_query_info.query_tree, @@ -594,6 +645,7 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & quer } modified_query_info.query = queryNodeToSelectQuery(modified_query_info.query_tree); + LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "Modified query: {}", modified_query_info.query->formatForLogging()); } else { @@ -640,6 +692,8 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( modified_select.setFinal(); } + LOG_DEBUG(&Poco::Logger::get("createSources"), "real_column_names: {}", toString(real_column_names)); + bool allow_experimental_analyzer = modified_context->getSettingsRef().allow_experimental_analyzer; auto storage_stage = storage->getQueryProcessingStage(modified_context, @@ -783,7 +837,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. - convertingSourceStream(header, storage_snapshot->metadata, aliases, modified_context, *builder, processed_stage); + convertingSourceStream(header, modified_query_info, storage_snapshot->metadata, aliases, modified_context, *builder, processed_stage); } return builder; @@ -957,9 +1011,10 @@ void StorageMerge::alter( void ReadFromMerge::convertingSourceStream( const Block & header, + SelectQueryInfo & modified_query_info, const StorageMetadataPtr & metadata_snapshot, const Aliases & aliases, - ContextPtr local_context, + ContextMutablePtr local_context, QueryPipelineBuilder & builder, const QueryProcessingStage::Enum & processed_stage) { @@ -968,21 +1023,49 @@ void ReadFromMerge::convertingSourceStream( auto storage_sample_block = metadata_snapshot->getSampleBlock(); auto pipe_columns = builder.getHeader().getNamesAndTypesList(); - for (const auto & alias : aliases) + if (local_context->getSettingsRef().allow_experimental_analyzer) { - pipe_columns.emplace_back(NameAndTypePair(alias.name, alias.type)); - ASTPtr expr = alias.expression; - auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns); - auto expression_analyzer = ExpressionAnalyzer{alias.expression, syntax_result, local_context}; - - auto dag = std::make_shared(pipe_columns); - auto actions_dag = expression_analyzer.getActionsDAG(true, false); - auto actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); - - builder.addSimpleTransform([&](const Block & stream_header) + for (const auto & alias : aliases) { - return std::make_shared(stream_header, actions); - }); + pipe_columns.emplace_back(NameAndTypePair(alias.name, alias.type)); + + auto actions_dag = std::make_shared(); + + QueryTreeNodePtr query_tree = buildQueryTree(alias.expression, local_context); + query_tree->setAlias(alias.name); + + QueryAnalysisPass query_analysis_pass(modified_query_info.table_expression); + query_analysis_pass.run(query_tree, local_context); + + PlannerActionsVisitor actions_visitor(modified_query_info.planner_context, false /*use_column_identifier_as_action_node_name*/); + actions_visitor.visit(actions_dag, query_tree); + + auto actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + + builder.addSimpleTransform([&](const Block & stream_header) + { + return std::make_shared(stream_header, actions); + }); + } + } + else + { + for (const auto & alias : aliases) + { + pipe_columns.emplace_back(NameAndTypePair(alias.name, alias.type)); + ASTPtr expr = alias.expression; + auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns); + auto expression_analyzer = ExpressionAnalyzer{alias.expression, syntax_result, local_context}; + + auto dag = std::make_shared(pipe_columns); + auto actions_dag = expression_analyzer.getActionsDAG(true, false); + auto actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + + builder.addSimpleTransform([&](const Block & stream_header) + { + return std::make_shared(stream_header, actions); + }); + } } ActionsDAG::MatchColumnsMode convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Name; diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index babf0dd92e8..739d6831f6f 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -177,10 +177,10 @@ private: using Aliases = std::vector; - static SelectQueryInfo getModifiedQueryInfo(const SelectQueryInfo & query_info, - const ContextPtr & modified_context, + SelectQueryInfo getModifiedQueryInfo(const ContextPtr & modified_context, const StorageWithLockAndName & storage_with_lock_and_name, - const StorageSnapshotPtr & storage_snapshot); + const StorageSnapshotPtr & storage_snapshot, + Names & column_names_as_aliases) const; QueryPipelineBuilderPtr createSources( const StorageSnapshotPtr & storage_snapshot, @@ -197,9 +197,10 @@ private: static void convertingSourceStream( const Block & header, + SelectQueryInfo & modified_query_info, const StorageMetadataPtr & metadata_snapshot, const Aliases & aliases, - ContextPtr context, + ContextMutablePtr context, QueryPipelineBuilder & builder, const QueryProcessingStage::Enum & processed_stage); }; From fc9ee3eb4e1e4c4b145bc39bc7ce507cf05b9d1d Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 13 Jun 2023 15:01:31 +0000 Subject: [PATCH 02/18] Correctly build the ActionsDAG --- src/Storages/StorageMerge.cpp | 28 +++++++++++++++++++++------- src/Storages/StorageMerge.h | 3 ++- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index a49155ac2d9..d036eaa9f25 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -475,7 +475,7 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu auto nested_storage_snaphsot = storage->getStorageSnapshot(storage_metadata_snapshot, context); Names column_names_as_aliases; - auto modified_query_info = getModifiedQueryInfo(context, table, nested_storage_snaphsot, column_names_as_aliases); + auto modified_query_info = getModifiedQueryInfo(context, table, nested_storage_snaphsot, column_names_as_aliases, aliases); if (!context->getSettingsRef().allow_experimental_analyzer) { @@ -566,7 +566,8 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_context, const StorageWithLockAndName & storage_with_lock_and_name, const StorageSnapshotPtr & storage_snapshot, - Names & column_names_as_aliases) const + Names & column_names_as_aliases, + Aliases & aliases) const { const auto & [database_name, storage, storage_lock, table_name] = storage_with_lock_and_name; const StorageID current_storage_id = storage->getStorageID(); @@ -611,18 +612,23 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ if (is_alias) { - column_node = buildQueryTree(column_default->expression, modified_context); + // column_node = buildQueryTree(column_default->expression, modified_context); + column_node = std::make_shared(Identifier{column}); LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "QT before: {}\n{}", column_node->dumpTree(), modified_query_info.table_expression->dumpTree()); - column_node->setAlias(column); - QueryAnalysisPass query_analysis_pass(modified_query_info.table_expression); query_analysis_pass.run(column_node, modified_context); LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "QT after: {}", column_node->dumpTree()); + auto * resolved_column = column_node->as(); + if (!resolved_column || !resolved_column->getExpression()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Alias column is not resolved"); + + column_node = resolved_column->getExpression(); column_name_to_node.emplace(column, column_node); + aliases.push_back({ .name = column, .type = resolved_column->getResultType(), .expression = column_node->toAST() }); } else { @@ -634,6 +640,9 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ actions_visitor.visit(filter_actions_dag, column_node); } column_names_as_aliases = filter_actions_dag->getRequiredColumnsNames(); + if (column_names_as_aliases.empty()) + column_names_as_aliases.push_back(ExpressionActions::getSmallestColumn(storage_snapshot->metadata->getColumns().getAllPhysical()).name); + LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "Required names: {}", toString(column_names_as_aliases)); } @@ -1029,7 +1038,7 @@ void ReadFromMerge::convertingSourceStream( { pipe_columns.emplace_back(NameAndTypePair(alias.name, alias.type)); - auto actions_dag = std::make_shared(); + auto actions_dag = std::make_shared(pipe_columns); QueryTreeNodePtr query_tree = buildQueryTree(alias.expression, local_context); query_tree->setAlias(alias.name); @@ -1038,7 +1047,12 @@ void ReadFromMerge::convertingSourceStream( query_analysis_pass.run(query_tree, local_context); PlannerActionsVisitor actions_visitor(modified_query_info.planner_context, false /*use_column_identifier_as_action_node_name*/); - actions_visitor.visit(actions_dag, query_tree); + const auto & nodes = actions_visitor.visit(actions_dag, query_tree); + + if (nodes.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected to have 1 output but got {}", nodes.size()); + + actions_dag->addOrReplaceInOutputs(actions_dag->addAlias(*nodes.front(), alias.name)); auto actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 739d6831f6f..987869e5de3 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -180,7 +180,8 @@ private: SelectQueryInfo getModifiedQueryInfo(const ContextPtr & modified_context, const StorageWithLockAndName & storage_with_lock_and_name, const StorageSnapshotPtr & storage_snapshot, - Names & column_names_as_aliases) const; + Names & column_names_as_aliases, + Aliases & aliases) const; QueryPipelineBuilderPtr createSources( const StorageSnapshotPtr & storage_snapshot, From 55b81a5a5e7ad73a3e53aee0d0b83731ff8e76ed Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 13 Jun 2023 23:13:18 +0000 Subject: [PATCH 03/18] Fix style --- src/Storages/StorageMerge.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index d036eaa9f25..e2a27d4e20e 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -80,6 +80,7 @@ namespace DB namespace ErrorCodes { + extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; extern const int NOT_IMPLEMENTED; extern const int ILLEGAL_PREWHERE; From 6489922dc19a0fda86bdcc8e08c108812dc4aebf Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 16 Jun 2023 18:49:59 +0000 Subject: [PATCH 04/18] Fix for column aliases that use other aliases --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 9 ------ src/Storages/StorageMerge.cpp | 38 ++++++++++++++++++++--- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 1a76bc762a4..309f067c4c0 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1099,7 +1099,6 @@ public: { if (table_expression) { - LOG_DEBUG(&Poco::Logger::get("resolve"), "Table expression: {}", table_expression->dumpTree()); scope.expression_join_tree_node = table_expression; validateTableExpressionModifiers(scope.expression_join_tree_node, scope); initializeTableExpressionData(scope.expression_join_tree_node, scope); @@ -1109,7 +1108,6 @@ public: resolveExpressionNodeList(node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); else resolveExpressionNode(node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); - LOG_DEBUG(&Poco::Logger::get("resolve"), "Result: {}", node->dumpTree()); break; } @@ -2681,7 +2679,6 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromAliases(const Identifier */ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableColumns(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope) { - LOG_DEBUG(&Poco::Logger::get("tryResolveIdentifierFromTableColumns"), "{} {}", scope.column_name_to_column_node.size(), !identifier_lookup.isExpressionLookup()); if (scope.column_name_to_column_node.empty() || !identifier_lookup.isExpressionLookup()) return {}; @@ -2841,14 +2838,11 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id QueryTreeNodePtr result_expression; bool match_full_identifier = false; - LOG_DEBUG(&Poco::Logger::get("resolve_identifier_from_storage_or_throw"), "Looking for id: {}", identifier_without_column_qualifier.getFullName()); - auto it = table_expression_data.column_name_to_column_node.find(identifier_without_column_qualifier.getFullName()); if (it != table_expression_data.column_name_to_column_node.end()) { match_full_identifier = true; result_expression = it->second; - LOG_DEBUG(&Poco::Logger::get("resolve_identifier_from_storage_or_throw"), "Found: {}", result_expression->dumpTree()); } else { @@ -5397,7 +5391,6 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id auto unresolved_identifier = identifier_node.getIdentifier(); auto resolve_identifier_expression_result = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::EXPRESSION}, scope); auto resolved_identifier_node = resolve_identifier_expression_result.resolved_identifier; - LOG_DEBUG(&Poco::Logger::get("resolveExpressionNode"), "Resolved: {}", resolved_identifier_node ? resolved_identifier_node->dumpTree() : "Not resolved"); if (resolved_identifier_node && result_projection_names.empty() && (resolve_identifier_expression_result.isResolvedFromJoinTree() || resolve_identifier_expression_result.isResolvedFromExpressionArguments())) @@ -5479,7 +5472,6 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id } node = std::move(resolved_identifier_node); - LOG_DEBUG(&Poco::Logger::get("resolveExpressionNode"), "Result node: {}", node ? node->dumpTree() : "Not resolved"); if (node->getNodeType() == QueryTreeNodeType::LIST) { @@ -6183,7 +6175,6 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table table_expression_data.should_qualify_columns = false; } - LOG_DEBUG(&Poco::Logger::get("Analyzer"), "Table data: {}", table_expression_data.dump()); scope.table_expression_node_to_data.emplace(table_expression_node, std::move(table_expression_data)); } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index e2a27d4e20e..13548a84826 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -34,6 +34,7 @@ #include "Analyzer/IQueryTreeNode.h" #include "Analyzer/Identifier.h" #include "Analyzer/IdentifierNode.h" +#include "Analyzer/InDepthQueryTreeVisitor.h" #include "Analyzer/Passes/QueryAnalysisPass.h" #include "Analyzer/QueryTreeBuilder.h" #include "Core/NamesAndTypes.h" @@ -564,6 +565,26 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu pipeline.addResources(std::move(resources)); } +namespace +{ + +class ApplyAliasColumnExpressionsVisitor : public InDepthQueryTreeVisitor +{ +public: + ApplyAliasColumnExpressionsVisitor() = default; + + void visitImpl(QueryTreeNodePtr & node) + { + if (auto * column = node->as(); + column != nullptr && column->hasExpression()) + { + node = column->getExpressionOrThrow(); + } + } +}; + +} + SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_context, const StorageWithLockAndName & storage_with_lock_and_name, const StorageSnapshotPtr & storage_snapshot, @@ -611,23 +632,28 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ QueryTreeNodePtr column_node; + if (is_alias) { // column_node = buildQueryTree(column_default->expression, modified_context); - column_node = std::make_shared(Identifier{column}); + QueryTreeNodePtr fake_node = std::make_shared(Identifier{column}); - LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "QT before: {}\n{}", column_node->dumpTree(), modified_query_info.table_expression->dumpTree()); + LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "QT before: {}\n{}", fake_node->dumpTree(), modified_query_info.table_expression->dumpTree()); QueryAnalysisPass query_analysis_pass(modified_query_info.table_expression); - query_analysis_pass.run(column_node, modified_context); + query_analysis_pass.run(fake_node, modified_context); + + auto * resolved_column = fake_node->as(); + + column_node = fake_node; + ApplyAliasColumnExpressionsVisitor visitor; + visitor.visit(column_node); LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "QT after: {}", column_node->dumpTree()); - auto * resolved_column = column_node->as(); if (!resolved_column || !resolved_column->getExpression()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Alias column is not resolved"); - column_node = resolved_column->getExpression(); column_name_to_node.emplace(column, column_node); aliases.push_back({ .name = column, .type = resolved_column->getResultType(), .expression = column_node->toAST() }); } @@ -1095,6 +1121,8 @@ void ReadFromMerge::convertingSourceStream( std::move(convert_actions_dag), ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + LOG_DEBUG(&Poco::Logger::get("convertingSourceStream"), "The header: {}", builder.getHeader().dumpStructure()); + builder.addSimpleTransform([&](const Block & stream_header) { return std::make_shared(stream_header, actions); From f9e67fe0427ee2d698d2b946a8286e228d47b0ec Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 19 Jun 2023 15:10:29 +0000 Subject: [PATCH 05/18] Update broken_tests.txt --- tests/broken_tests.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/broken_tests.txt b/tests/broken_tests.txt index d49b4f391e5..1635c8740cc 100644 --- a/tests/broken_tests.txt +++ b/tests/broken_tests.txt @@ -24,7 +24,6 @@ 01173_transaction_control_queries 01211_optimize_skip_unused_shards_type_mismatch 01213_optimize_skip_unused_shards_DISTINCT -01214_test_storage_merge_aliases_with_where 01231_distributed_aggregation_memory_efficient_mix_levels 01244_optimize_distributed_group_by_sharding_key 01247_optimize_distributed_group_by_sharding_key_dist_on_dist @@ -68,7 +67,6 @@ 01890_materialized_distributed_join 01901_in_literal_shard_prune 01925_join_materialized_columns -01925_test_storage_merge_aliases 01930_optimize_skip_unused_shards_rewrite_in 01947_mv_subquery 01951_distributed_push_down_limit From dcdadd5f639def096bd330f987609d0c5740ca83 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 19 Jun 2023 15:18:04 +0000 Subject: [PATCH 06/18] Update broken_tests.txt --- tests/broken_tests.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/broken_tests.txt b/tests/broken_tests.txt index 1635c8740cc..8b11c5f5413 100644 --- a/tests/broken_tests.txt +++ b/tests/broken_tests.txt @@ -99,7 +99,6 @@ 02494_optimize_group_by_function_keys_and_alias_columns 02521_aggregation_by_partitions 02554_fix_grouping_sets_predicate_push_down -02575_merge_prewhere_different_default_kind 02713_array_low_cardinality_string 02707_skip_index_with_in 02241_join_rocksdb_bs From 20c752fb787a05f9180f791401afe56bf372acfc Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 19 Jun 2023 15:44:01 +0000 Subject: [PATCH 07/18] Fix generated query --- src/Storages/StorageMerge.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 13548a84826..22308c1d901 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -614,7 +614,11 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ std::unordered_map column_name_to_node; if (!storage_snapshot->tryGetColumn(get_column_options, "_table")) - column_name_to_node.emplace("_table", std::make_shared(current_storage_id.table_name)); + { + auto table_name_node = std::make_shared(current_storage_id.table_name); + table_name_node->setAlias("_table"); + column_name_to_node.emplace("_table", table_name_node); + } if (!storage_snapshot->tryGetColumn(get_column_options, "_database")) column_name_to_node.emplace("_database", std::make_shared(current_storage_id.database_name)); From 118b84703bb0f08aa622b956b1207d9092f5f2d7 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 21 Jun 2023 01:51:34 +0200 Subject: [PATCH 08/18] WIP on StorageMerge and distributed JOIN --- src/Analyzer/ColumnNode.h | 5 ++ src/Storages/StorageMerge.cpp | 86 ++++++++++++++++++++++++++++++++--- src/Storages/StorageMerge.h | 2 +- 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/src/Analyzer/ColumnNode.h b/src/Analyzer/ColumnNode.h index b320df788c5..46e7c8eb500 100644 --- a/src/Analyzer/ColumnNode.h +++ b/src/Analyzer/ColumnNode.h @@ -108,6 +108,11 @@ public: */ QueryTreeNodePtr getColumnSourceOrNull() const; + void setColumnSource(const QueryTreeNodePtr & source) + { + getSourceWeakPointer() = source; + } + QueryTreeNodeType getNodeType() const override { return QueryTreeNodeType::COLUMN; diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 22308c1d901..85ec21b4765 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -51,6 +52,7 @@ #include #include #include +#include #include #include @@ -583,6 +585,76 @@ public: } }; +bool hasUnknownColumn(const QueryTreeNodePtr & node, + QueryTreeNodePtr original_table_expression, + QueryTreeNodePtr replacement_table_expression) +{ + QueryTreeNodes stack = { node }; + while (!stack.empty()) + { + auto current = stack.back(); + stack.pop_back(); + + switch (current->getNodeType()) + { + case QueryTreeNodeType::CONSTANT: + break; + case QueryTreeNodeType::COLUMN: + { + auto * column_node = current->as(); + auto source = column_node->getColumnSourceOrNull(); + if (source != original_table_expression) + return true; + else + column_node->setColumnSource(replacement_table_expression); + break; + } + default: + { + for (const auto & child : node->getChildren()) + { + if (child) + stack.push_back(child); + } + } + } + } + return false; +} + +QueryTreeNodePtr removeJoin( + QueryTreeNodePtr query, + QueryTreeNodePtr original_table_expression, + QueryTreeNodePtr replacement_table_expression) +{ + auto * query_node = query->as(); + auto modified_query = query_node->cloneAndReplace(query_node->getJoinTree(), replacement_table_expression); + + query_node = modified_query->as(); + query_node->getGroupBy().getNodes().clear(); + query_node->getHaving() = {}; + query_node->getOrderBy().getNodes().clear(); + + auto & projection = query_node->getProjection().getNodes(); + auto projection_columns = query_node->getProjectionColumns(); + for (size_t i = 0; i < projection.size();) + { + if (hasUnknownColumn(projection[i], original_table_expression, replacement_table_expression)) + { + projection.erase(projection.begin() + i); + projection_columns.erase(projection_columns.begin() + i); + continue; + } + ++i; + } + + query_node->resolveProjectionColumns(std::move(projection_columns)); + + LOG_DEBUG(&Poco::Logger::get("removeJoin"), "Query without JOIN:\n{}", modified_query->dumpTree()); + + return modified_query; +} + } SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_context, @@ -602,8 +674,9 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ if (query_info.table_expression_modifiers) replacement_table_expression->setTableExpressionModifiers(*query_info.table_expression_modifiers); - modified_query_info.query_tree = modified_query_info.query_tree->cloneAndReplace(modified_query_info.table_expression, - replacement_table_expression); + modified_query_info.query_tree = removeJoin(modified_query_info.query_tree, modified_query_info.table_expression, replacement_table_expression); + // modified_query_info.query_tree = modified_query_info.query_tree->cloneAndReplace(modified_query_info.table_expression, + // replacement_table_expression); modified_query_info.table_expression = replacement_table_expression; modified_query_info.planner_context->getOrCreateTableExpressionData(replacement_table_expression); @@ -877,7 +950,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. - convertingSourceStream(header, modified_query_info, storage_snapshot->metadata, aliases, modified_context, *builder, processed_stage); + convertingSourceStream(header, modified_query_info, storage_snapshot, aliases, modified_context, *builder, processed_stage); } return builder; @@ -1052,7 +1125,7 @@ void StorageMerge::alter( void ReadFromMerge::convertingSourceStream( const Block & header, SelectQueryInfo & modified_query_info, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & snapshot, const Aliases & aliases, ContextMutablePtr local_context, QueryPipelineBuilder & builder, @@ -1060,7 +1133,7 @@ void ReadFromMerge::convertingSourceStream( { Block before_block_header = builder.getHeader(); - auto storage_sample_block = metadata_snapshot->getSampleBlock(); + auto storage_sample_block = snapshot->metadata->getSampleBlock(); auto pipe_columns = builder.getHeader().getNamesAndTypesList(); if (local_context->getSettingsRef().allow_experimental_analyzer) @@ -1115,7 +1188,8 @@ void ReadFromMerge::convertingSourceStream( ActionsDAG::MatchColumnsMode convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Name; - if (local_context->getSettingsRef().allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns) + if (local_context->getSettingsRef().allow_experimental_analyzer + && (processed_stage != QueryProcessingStage::FetchColumns || dynamic_cast(&snapshot->storage) != nullptr)) convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position; auto convert_actions_dag = ActionsDAG::makeConvertingActions(builder.getHeader().getColumnsWithTypeAndName(), diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 987869e5de3..de9480292f9 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -199,7 +199,7 @@ private: static void convertingSourceStream( const Block & header, SelectQueryInfo & modified_query_info, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & snapshot, const Aliases & aliases, ContextMutablePtr context, QueryPipelineBuilder & builder, From 88fe30254a280286ac2bd2b6bcdc71865ec2aed2 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 21 Jun 2023 17:55:14 +0000 Subject: [PATCH 09/18] Small fixup --- src/Storages/StorageMerge.cpp | 12 +++++++++--- tests/broken_tests.txt | 1 - 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 85ec21b4765..d1ac3f57ae1 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -631,6 +631,10 @@ QueryTreeNodePtr removeJoin( auto modified_query = query_node->cloneAndReplace(query_node->getJoinTree(), replacement_table_expression); query_node = modified_query->as(); + + //TODO: change the predicates to make it valid and execute it on shards. + query_node->getPrewhere() = {}; + query_node->getWhere() = {}; query_node->getGroupBy().getNodes().clear(); query_node->getHaving() = {}; query_node->getOrderBy().getNodes().clear(); @@ -675,8 +679,6 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ replacement_table_expression->setTableExpressionModifiers(*query_info.table_expression_modifiers); modified_query_info.query_tree = removeJoin(modified_query_info.query_tree, modified_query_info.table_expression, replacement_table_expression); - // modified_query_info.query_tree = modified_query_info.query_tree->cloneAndReplace(modified_query_info.table_expression, - // replacement_table_expression); modified_query_info.table_expression = replacement_table_expression; modified_query_info.planner_context->getOrCreateTableExpressionData(replacement_table_expression); @@ -694,7 +696,11 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ } if (!storage_snapshot->tryGetColumn(get_column_options, "_database")) - column_name_to_node.emplace("_database", std::make_shared(current_storage_id.database_name)); + { + auto database_name_node = std::make_shared(current_storage_id.database_name); + database_name_node->setAlias("_database"); + column_name_to_node.emplace("_database", database_name_node); + } auto storage_columns = storage_snapshot->metadata->getColumns(); diff --git a/tests/broken_tests.txt b/tests/broken_tests.txt index e6b5fb4f631..f6e21a29eed 100644 --- a/tests/broken_tests.txt +++ b/tests/broken_tests.txt @@ -38,7 +38,6 @@ 01527_dist_sharding_key_dictGet_reload 01528_allow_nondeterministic_optimize_skip_unused_shards 01540_verbatim_partition_pruning -01560_merge_distributed_join 01563_distributed_query_finish 01576_alias_column_rewrite 01583_const_column_in_set_index From 47fafdc32c320464bbd65468208bbc8e5b7ac62f Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 21 Jun 2023 18:06:24 +0000 Subject: [PATCH 10/18] Code cleanup --- src/Storages/StorageDistributed.cpp | 1 - src/Storages/StorageMerge.cpp | 35 ++++++++--------------------- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 9f9f0fda9e2..b948ca946c3 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -30,7 +30,6 @@ #include #include #include -#include "Analyzer/IQueryTreeNode.h" #include #include diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index d1ac3f57ae1..1a0376edbf5 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -28,19 +28,17 @@ #include #include #include -#include "Common/logger_useful.h" #include #include -#include "Analyzer/ColumnNode.h" -#include "Analyzer/IQueryTreeNode.h" -#include "Analyzer/Identifier.h" -#include "Analyzer/IdentifierNode.h" -#include "Analyzer/InDepthQueryTreeVisitor.h" -#include "Analyzer/Passes/QueryAnalysisPass.h" -#include "Analyzer/QueryTreeBuilder.h" -#include "Core/NamesAndTypes.h" -#include "DataTypes/IDataType.h" -#include "Planner/PlannerActionsVisitor.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -52,7 +50,6 @@ #include #include #include -#include #include #include @@ -654,8 +651,6 @@ QueryTreeNodePtr removeJoin( query_node->resolveProjectionColumns(std::move(projection_columns)); - LOG_DEBUG(&Poco::Logger::get("removeJoin"), "Query without JOIN:\n{}", modified_query->dumpTree()); - return modified_query; } @@ -718,11 +713,8 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ if (is_alias) { - // column_node = buildQueryTree(column_default->expression, modified_context); QueryTreeNodePtr fake_node = std::make_shared(Identifier{column}); - LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "QT before: {}\n{}", fake_node->dumpTree(), modified_query_info.table_expression->dumpTree()); - QueryAnalysisPass query_analysis_pass(modified_query_info.table_expression); query_analysis_pass.run(fake_node, modified_context); @@ -732,8 +724,6 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ ApplyAliasColumnExpressionsVisitor visitor; visitor.visit(column_node); - LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "QT after: {}", column_node->dumpTree()); - if (!resolved_column || !resolved_column->getExpression()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Alias column is not resolved"); @@ -752,8 +742,6 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ column_names_as_aliases = filter_actions_dag->getRequiredColumnsNames(); if (column_names_as_aliases.empty()) column_names_as_aliases.push_back(ExpressionActions::getSmallestColumn(storage_snapshot->metadata->getColumns().getAllPhysical()).name); - - LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "Required names: {}", toString(column_names_as_aliases)); } if (!column_name_to_node.empty()) @@ -764,7 +752,6 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ } modified_query_info.query = queryNodeToSelectQuery(modified_query_info.query_tree); - LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "Modified query: {}", modified_query_info.query->formatForLogging()); } else { @@ -811,8 +798,6 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( modified_select.setFinal(); } - LOG_DEBUG(&Poco::Logger::get("createSources"), "real_column_names: {}", toString(real_column_names)); - bool allow_experimental_analyzer = modified_context->getSettingsRef().allow_experimental_analyzer; auto storage_stage = storage->getQueryProcessingStage(modified_context, @@ -1205,8 +1190,6 @@ void ReadFromMerge::convertingSourceStream( std::move(convert_actions_dag), ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); - LOG_DEBUG(&Poco::Logger::get("convertingSourceStream"), "The header: {}", builder.getHeader().dumpStructure()); - builder.addSimpleTransform([&](const Block & stream_header) { return std::make_shared(stream_header, actions); From 97a1ea01badaba10235ab0b01777f324b2f8365e Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 22 Jun 2023 15:10:53 +0000 Subject: [PATCH 11/18] Fix removeJoin --- src/Storages/StorageMerge.cpp | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 1a0376edbf5..fd7c0aae479 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -625,7 +626,8 @@ QueryTreeNodePtr removeJoin( QueryTreeNodePtr replacement_table_expression) { auto * query_node = query->as(); - auto modified_query = query_node->cloneAndReplace(query_node->getJoinTree(), replacement_table_expression); + auto join_tree = query_node->getJoinTree(); + auto modified_query = query_node->cloneAndReplace(join_tree, replacement_table_expression); query_node = modified_query->as(); @@ -636,20 +638,23 @@ QueryTreeNodePtr removeJoin( query_node->getHaving() = {}; query_node->getOrderBy().getNodes().clear(); - auto & projection = query_node->getProjection().getNodes(); - auto projection_columns = query_node->getProjectionColumns(); - for (size_t i = 0; i < projection.size();) + if (join_tree->as() == nullptr && join_tree->as() == nullptr) { - if (hasUnknownColumn(projection[i], original_table_expression, replacement_table_expression)) + auto & projection = query_node->getProjection().getNodes(); + auto projection_columns = query_node->getProjectionColumns(); + for (size_t i = 0; i < projection.size();) { - projection.erase(projection.begin() + i); - projection_columns.erase(projection_columns.begin() + i); - continue; + if (hasUnknownColumn(projection[i], original_table_expression, replacement_table_expression)) + { + projection.erase(projection.begin() + i); + projection_columns.erase(projection_columns.begin() + i); + continue; + } + ++i; } - ++i; - } - query_node->resolveProjectionColumns(std::move(projection_columns)); + query_node->resolveProjectionColumns(std::move(projection_columns)); + } return modified_query; } From 629d4b921e5cf2d709d2ca7a55658d95407e2ff7 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 9 Jan 2024 15:38:04 +0000 Subject: [PATCH 12/18] Fix style --- src/Analyzer/Passes/IfConstantConditionPass.cpp | 2 +- src/Storages/StorageMerge.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Analyzer/Passes/IfConstantConditionPass.cpp b/src/Analyzer/Passes/IfConstantConditionPass.cpp index f3b8b712dbf..6b24eb1d539 100644 --- a/src/Analyzer/Passes/IfConstantConditionPass.cpp +++ b/src/Analyzer/Passes/IfConstantConditionPass.cpp @@ -57,7 +57,7 @@ public: } -void IfConstantConditionPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) +void IfConstantConditionPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { IfConstantConditionVisitor visitor(std::move(context)); visitor.visit(query_tree_node); diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 15ca6e65482..ffbf98e85c7 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -92,7 +92,6 @@ namespace ErrorCodes extern const int SAMPLING_NOT_SUPPORTED; extern const int ALTER_OF_COLUMN_IS_FORBIDDEN; extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; - extern const int LOGICAL_ERROR; } StorageMerge::DatabaseNameOrRegexp::DatabaseNameOrRegexp( From 0d21004218c8fad0be30493629fd93927f5d71b0 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 31 Jan 2024 13:50:15 +0000 Subject: [PATCH 13/18] WIP on StorageMerge --- .../OptimizeGroupByInjectiveFunctionsPass.cpp | 2 +- .../OptimizeGroupByInjectiveFunctionsPass.h | 2 +- .../RewriteSumFunctionWithSumAndCountPass.cpp | 2 +- .../RewriteSumFunctionWithSumAndCountPass.h | 2 +- src/Storages/StorageMerge.cpp | 40 ++++++++++++------- src/Storages/StorageMerge.h | 1 + 6 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp index 864752cdbeb..ad649834fb0 100644 --- a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp @@ -115,7 +115,7 @@ private: } -void OptimizeGroupByInjectiveFunctionsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void OptimizeGroupByInjectiveFunctionsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { OptimizeGroupByInjectiveFunctionsVisitor visitor(std::move(context)); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.h b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.h index 22390451824..b3ba9033b92 100644 --- a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.h +++ b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.h @@ -14,7 +14,7 @@ public: String getDescription() override { return "Replaces injective functions by it's arguments in GROUP BY section."; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; } diff --git a/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp b/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp index 24ddb7522c8..3c93bf9e1bf 100644 --- a/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp +++ b/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp @@ -120,7 +120,7 @@ private: } -void RewriteSumFunctionWithSumAndCountPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +void RewriteSumFunctionWithSumAndCountPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { RewriteSumFunctionWithSumAndCountVisitor visitor(std::move(context)); visitor.visit(query_tree_node); diff --git a/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.h b/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.h index e878a2c0e7a..4615532e3d9 100644 --- a/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.h +++ b/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.h @@ -20,7 +20,7 @@ public: String getDescription() override { return "Rewrite sum(column +/- literal) into sum(column) and literal * count(column)"; } - void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 45bd20bd859..cda47596a4f 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -54,6 +54,7 @@ #include #include #include +#include "Common/logger_useful.h" #include #include #include @@ -388,7 +389,7 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu Names column_names_as_aliases; Aliases aliases; - auto modified_query_info = getModifiedQueryInfo(context, table, nested_storage_snaphsot, column_names_as_aliases, aliases); + auto modified_query_info = getModifiedQueryInfo(context, table, nested_storage_snaphsot, column_names, column_names_as_aliases, aliases); auto source_pipeline = createSources( child_plan.plan, @@ -524,8 +525,6 @@ std::vector ReadFromMerge::createChildrenPlans(SelectQ Names column_names_as_aliases; Names real_column_names = column_names; - auto modified_query_info = getModifiedQueryInfo(context, table, nested_storage_snaphsot, column_names_as_aliases, aliases); - const auto & database_name = std::get<0>(table); const auto & table_name = std::get<3>(table); auto row_policy_filter_ptr = context->getRowPolicyFilter( @@ -538,6 +537,8 @@ std::vector ReadFromMerge::createChildrenPlans(SelectQ row_policy_data_opt->extendNames(real_column_names); } + auto modified_query_info = getModifiedQueryInfo(context, table, nested_storage_snaphsot, real_column_names, column_names_as_aliases, aliases); + if (!context->getSettingsRef().allow_experimental_analyzer) { auto storage_columns = storage_metadata_snapshot->getColumns(); @@ -628,15 +629,14 @@ public: } }; -bool hasUnknownColumn(const QueryTreeNodePtr & node, - QueryTreeNodePtr original_table_expression, - QueryTreeNodePtr replacement_table_expression) +bool hasUnknownColumn(const QueryTreeNodePtr & node, QueryTreeNodePtr replacement_table_expression) { QueryTreeNodes stack = { node }; while (!stack.empty()) { auto current = stack.back(); stack.pop_back(); + LOG_DEBUG(&Poco::Logger::get("hasUnknownColumn"), "Expression: {}", current->formatASTForErrorMessage()); switch (current->getNodeType()) { @@ -646,15 +646,13 @@ bool hasUnknownColumn(const QueryTreeNodePtr & node, { auto * column_node = current->as(); auto source = column_node->getColumnSourceOrNull(); - if (source != original_table_expression) + if (source != replacement_table_expression) return true; - else - column_node->setColumnSource(replacement_table_expression); break; } default: { - for (const auto & child : node->getChildren()) + for (const auto & child : current->getChildren()) { if (child) stack.push_back(child); @@ -670,9 +668,16 @@ QueryTreeNodePtr removeJoin( QueryTreeNodePtr original_table_expression, QueryTreeNodePtr replacement_table_expression) { + LOG_DEBUG(&Poco::Logger::get("removeJoin"), "Entered the function"); + auto * query_node = query->as(); auto join_tree = query_node->getJoinTree(); - auto modified_query = query_node->cloneAndReplace(join_tree, replacement_table_expression); + auto modified_query = query_node->cloneAndReplace(original_table_expression, replacement_table_expression); + + auto * modified_query_node = modified_query->as(); + + modified_query = modified_query->cloneAndReplace(modified_query_node->getJoinTree(), replacement_table_expression); + modified_query_node = modified_query->as(); query_node = modified_query->as(); @@ -685,11 +690,12 @@ QueryTreeNodePtr removeJoin( if (join_tree->as() == nullptr && join_tree->as() == nullptr) { - auto & projection = query_node->getProjection().getNodes(); - auto projection_columns = query_node->getProjectionColumns(); + auto & projection = modified_query_node->getProjection().getNodes(); + auto projection_columns = modified_query_node->getProjectionColumns(); for (size_t i = 0; i < projection.size();) { - if (hasUnknownColumn(projection[i], original_table_expression, replacement_table_expression)) + LOG_DEBUG(&Poco::Logger::get("removeJoin"), "Processing: {}", i); + if (hasUnknownColumn(projection[i], replacement_table_expression)) { projection.erase(projection.begin() + i); projection_columns.erase(projection_columns.begin() + i); @@ -701,6 +707,8 @@ QueryTreeNodePtr removeJoin( query_node->resolveProjectionColumns(std::move(projection_columns)); } + LOG_DEBUG(&Poco::Logger::get("removeJoin"), "Result:\n{}", modified_query->dumpTree()); + return modified_query; } @@ -709,9 +717,11 @@ QueryTreeNodePtr removeJoin( SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_context, const StorageWithLockAndName & storage_with_lock_and_name, const StorageSnapshotPtr & storage_snapshot, + Names real_column_names, Names & column_names_as_aliases, Aliases & aliases) const { + LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "Procesing query"); const auto & [database_name, storage, storage_lock, table_name] = storage_with_lock_and_name; const StorageID current_storage_id = storage->getStorageID(); @@ -753,7 +763,7 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ if (with_aliases) { auto filter_actions_dag = std::make_shared(); - for (const auto & column : column_names) + for (const auto & column : real_column_names) { const auto column_default = storage_columns.getDefault(column); bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias; diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 5acc06ab8de..c500d1358a3 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -192,6 +192,7 @@ private: SelectQueryInfo getModifiedQueryInfo(const ContextPtr & modified_context, const StorageWithLockAndName & storage_with_lock_and_name, const StorageSnapshotPtr & storage_snapshot, + Names real_column_names, Names & column_names_as_aliases, Aliases & aliases) const; From 572d47acc72f3cc2188a4423a36b31e8e960277a Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 31 Jan 2024 14:43:06 +0000 Subject: [PATCH 14/18] Remove logging --- src/Storages/StorageMerge.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index cda47596a4f..17a6ade4059 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -54,7 +54,6 @@ #include #include #include -#include "Common/logger_useful.h" #include #include #include @@ -636,7 +635,6 @@ bool hasUnknownColumn(const QueryTreeNodePtr & node, QueryTreeNodePtr replacemen { auto current = stack.back(); stack.pop_back(); - LOG_DEBUG(&Poco::Logger::get("hasUnknownColumn"), "Expression: {}", current->formatASTForErrorMessage()); switch (current->getNodeType()) { @@ -668,8 +666,6 @@ QueryTreeNodePtr removeJoin( QueryTreeNodePtr original_table_expression, QueryTreeNodePtr replacement_table_expression) { - LOG_DEBUG(&Poco::Logger::get("removeJoin"), "Entered the function"); - auto * query_node = query->as(); auto join_tree = query_node->getJoinTree(); auto modified_query = query_node->cloneAndReplace(original_table_expression, replacement_table_expression); @@ -694,7 +690,6 @@ QueryTreeNodePtr removeJoin( auto projection_columns = modified_query_node->getProjectionColumns(); for (size_t i = 0; i < projection.size();) { - LOG_DEBUG(&Poco::Logger::get("removeJoin"), "Processing: {}", i); if (hasUnknownColumn(projection[i], replacement_table_expression)) { projection.erase(projection.begin() + i); @@ -707,8 +702,6 @@ QueryTreeNodePtr removeJoin( query_node->resolveProjectionColumns(std::move(projection_columns)); } - LOG_DEBUG(&Poco::Logger::get("removeJoin"), "Result:\n{}", modified_query->dumpTree()); - return modified_query; } @@ -721,7 +714,6 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ Names & column_names_as_aliases, Aliases & aliases) const { - LOG_DEBUG(&Poco::Logger::get("getModifiedQueryInfo"), "Procesing query"); const auto & [database_name, storage, storage_lock, table_name] = storage_with_lock_and_name; const StorageID current_storage_id = storage->getStorageID(); From 05a7d22c24f030a5bc9aa75fc22e4ca103e37468 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 2 Feb 2024 13:45:04 +0000 Subject: [PATCH 15/18] WIP on calculate aliases only once --- src/Storages/StorageMerge.cpp | 111 ++++++++++++++++++++-------------- src/Storages/StorageMerge.h | 2 +- 2 files changed, 67 insertions(+), 46 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 17a6ade4059..5ef6b5117f6 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -58,6 +58,7 @@ #include #include #include +#include "Core/NamesAndTypes.h" namespace { @@ -388,7 +389,12 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu Names column_names_as_aliases; Aliases aliases; - auto modified_query_info = getModifiedQueryInfo(context, table, nested_storage_snaphsot, column_names, column_names_as_aliases, aliases); + + Names real_column_names = column_names; + if (child_plan.row_policy_data_opt) + child_plan.row_policy_data_opt->extendNames(real_column_names); + + auto modified_query_info = getModifiedQueryInfo(context, table, nested_storage_snaphsot, real_column_names, column_names_as_aliases, aliases); auto source_pipeline = createSources( child_plan.plan, @@ -624,47 +630,50 @@ public: column != nullptr && column->hasExpression()) { node = column->getExpressionOrThrow(); + node->setAlias(column->getColumnName()); } } }; -bool hasUnknownColumn(const QueryTreeNodePtr & node, QueryTreeNodePtr replacement_table_expression) -{ - QueryTreeNodes stack = { node }; - while (!stack.empty()) - { - auto current = stack.back(); - stack.pop_back(); +// bool hasUnknownColumn(const QueryTreeNodePtr & node, QueryTreeNodePtr replacement_table_expression) +// { +// QueryTreeNodes stack = { node }; +// while (!stack.empty()) +// { +// auto current = stack.back(); +// stack.pop_back(); - switch (current->getNodeType()) - { - case QueryTreeNodeType::CONSTANT: - break; - case QueryTreeNodeType::COLUMN: - { - auto * column_node = current->as(); - auto source = column_node->getColumnSourceOrNull(); - if (source != replacement_table_expression) - return true; - break; - } - default: - { - for (const auto & child : current->getChildren()) - { - if (child) - stack.push_back(child); - } - } - } - } - return false; -} +// switch (current->getNodeType()) +// { +// case QueryTreeNodeType::CONSTANT: +// break; +// case QueryTreeNodeType::COLUMN: +// { +// auto * column_node = current->as(); +// auto source = column_node->getColumnSourceOrNull(); +// if (source != replacement_table_expression) +// return true; +// break; +// } +// default: +// { +// for (const auto & child : current->getChildren()) +// { +// if (child) +// stack.push_back(child); +// } +// } +// } +// } +// return false; +// } QueryTreeNodePtr removeJoin( QueryTreeNodePtr query, QueryTreeNodePtr original_table_expression, - QueryTreeNodePtr replacement_table_expression) + QueryTreeNodePtr replacement_table_expression, + const ContextPtr & context, + const Names & required_column_names) { auto * query_node = query->as(); auto join_tree = query_node->getJoinTree(); @@ -687,21 +696,33 @@ QueryTreeNodePtr removeJoin( if (join_tree->as() == nullptr && join_tree->as() == nullptr) { auto & projection = modified_query_node->getProjection().getNodes(); - auto projection_columns = modified_query_node->getProjectionColumns(); - for (size_t i = 0; i < projection.size();) + projection.clear(); + NamesAndTypes projection_columns; + + for (auto const & column_name : required_column_names) { - if (hasUnknownColumn(projection[i], replacement_table_expression)) - { - projection.erase(projection.begin() + i); - projection_columns.erase(projection_columns.begin() + i); - continue; - } - ++i; + QueryTreeNodePtr fake_node = std::make_shared(Identifier{column_name}); + + QueryAnalysisPass query_analysis_pass(original_table_expression); + query_analysis_pass.run(fake_node, context); + + auto * resolved_column = fake_node->as(); + if (!resolved_column) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Required column '{}' is not resolved", column_name); + auto fake_column = resolved_column->getColumn(); + + ApplyAliasColumnExpressionsVisitor visitor; + visitor.visit(fake_node); + + projection.push_back(fake_node); + projection_columns.push_back(fake_column); } query_node->resolveProjectionColumns(std::move(projection_columns)); } + LOG_DEBUG(&Poco::Logger::get("removeJoin"), "Result:\n{}", modified_query->dumpTree()); + return modified_query; } @@ -710,7 +731,7 @@ QueryTreeNodePtr removeJoin( SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_context, const StorageWithLockAndName & storage_with_lock_and_name, const StorageSnapshotPtr & storage_snapshot, - Names real_column_names, + Names required_column_names, Names & column_names_as_aliases, Aliases & aliases) const { @@ -725,7 +746,7 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ if (query_info.table_expression_modifiers) replacement_table_expression->setTableExpressionModifiers(*query_info.table_expression_modifiers); - modified_query_info.query_tree = removeJoin(modified_query_info.query_tree, modified_query_info.table_expression, replacement_table_expression); + modified_query_info.query_tree = removeJoin(modified_query_info.query_tree, modified_query_info.table_expression, replacement_table_expression, modified_context, required_column_names); modified_query_info.table_expression = replacement_table_expression; modified_query_info.planner_context->getOrCreateTableExpressionData(replacement_table_expression); @@ -755,7 +776,7 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ if (with_aliases) { auto filter_actions_dag = std::make_shared(); - for (const auto & column : real_column_names) + for (const auto & column : required_column_names) { const auto column_default = storage_columns.getDefault(column); bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias; diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index c500d1358a3..f5b6c3a7ca9 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -192,7 +192,7 @@ private: SelectQueryInfo getModifiedQueryInfo(const ContextPtr & modified_context, const StorageWithLockAndName & storage_with_lock_and_name, const StorageSnapshotPtr & storage_snapshot, - Names real_column_names, + Names required_column_names, Names & column_names_as_aliases, Aliases & aliases) const; From 1abcf26df69bd75efb7f54960fc11486fe3a37a4 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 6 Feb 2024 15:39:52 +0000 Subject: [PATCH 16/18] Fix filter expressions --- src/Storages/StorageMerge.cpp | 187 +++++++++++++++++++++++----------- 1 file changed, 129 insertions(+), 58 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 5ef6b5117f6..df5b0cd715d 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1,7 +1,9 @@ #include #include +#include #include #include +#include #include #include #include @@ -59,6 +61,7 @@ #include #include #include "Core/NamesAndTypes.h" +#include namespace { @@ -635,40 +638,106 @@ public: } }; -// bool hasUnknownColumn(const QueryTreeNodePtr & node, QueryTreeNodePtr replacement_table_expression) -// { -// QueryTreeNodes stack = { node }; -// while (!stack.empty()) -// { -// auto current = stack.back(); -// stack.pop_back(); +bool hasUnknownColumn(const QueryTreeNodePtr & node, QueryTreeNodePtr replacement_table_expression) +{ + QueryTreeNodes stack = { node }; + while (!stack.empty()) + { + auto current = stack.back(); + stack.pop_back(); -// switch (current->getNodeType()) -// { -// case QueryTreeNodeType::CONSTANT: -// break; -// case QueryTreeNodeType::COLUMN: -// { -// auto * column_node = current->as(); -// auto source = column_node->getColumnSourceOrNull(); -// if (source != replacement_table_expression) -// return true; -// break; -// } -// default: -// { -// for (const auto & child : current->getChildren()) -// { -// if (child) -// stack.push_back(child); -// } -// } -// } -// } -// return false; -// } + switch (current->getNodeType()) + { + case QueryTreeNodeType::CONSTANT: + break; + case QueryTreeNodeType::COLUMN: + { + auto * column_node = current->as(); + auto source = column_node->getColumnSourceOrNull(); + if (source != replacement_table_expression) + return true; + break; + } + default: + { + for (const auto & child : current->getChildren()) + { + if (child) + stack.push_back(child); + } + } + } + } + return false; +} -QueryTreeNodePtr removeJoin( +void replaceFilterExpression( + QueryTreeNodePtr & expression, + const QueryTreeNodePtr & replacement_table_expression, + const ContextPtr & context) +{ + auto * function = expression->as(); + if (!function) + return; + + if (function->getFunctionName() != "and") + { + if (hasUnknownColumn(expression, replacement_table_expression)) + expression = nullptr; + return; + } + + QueryTreeNodes conjunctions; + QueryTreeNodes processing{ expression }; + + while (!processing.empty()) + { + auto node = std::move(processing.back()); + processing.pop_back(); + + if (auto * function_node = node->as()) + { + if (function_node->getFunctionName() == "and") + std::copy( + function_node->getArguments().begin(), + function_node->getArguments().end(), + std::back_inserter(processing) + ); + else + conjunctions.push_back(node); + } + else + { + conjunctions.push_back(node); + } + } + + std::swap(processing, conjunctions); + + for (const auto & node : processing) + { + if (!hasUnknownColumn(node, replacement_table_expression)) + conjunctions.push_back(node); + } + + if (conjunctions.empty()) + { + expression = {}; + return; + } + if (conjunctions.size() == 1) + { + expression = conjunctions[0]; + return; + } + + function->getArguments().getNodes() = std::move(conjunctions); + + const auto function_impl = FunctionFactory::instance().get("and", context); + function->resolveAsFunction(function_impl->build(function->getArgumentColumns())); +} + +QueryTreeNodePtr replaceTableExpressionAndRemoveJoin( QueryTreeNodePtr query, QueryTreeNodePtr original_table_expression, QueryTreeNodePtr replacement_table_expression, @@ -676,9 +745,12 @@ QueryTreeNodePtr removeJoin( const Names & required_column_names) { auto * query_node = query->as(); - auto join_tree = query_node->getJoinTree(); + auto join_tree_type = query_node->getJoinTree()->getNodeType(); auto modified_query = query_node->cloneAndReplace(original_table_expression, replacement_table_expression); + if (join_tree_type == QueryTreeNodeType::TABLE || join_tree_type == QueryTreeNodeType::TABLE_FUNCTION) + return modified_query; + auto * modified_query_node = modified_query->as(); modified_query = modified_query->cloneAndReplace(modified_query_node->getJoinTree(), replacement_table_expression); @@ -686,41 +758,40 @@ QueryTreeNodePtr removeJoin( query_node = modified_query->as(); - //TODO: change the predicates to make it valid and execute it on shards. - query_node->getPrewhere() = {}; - query_node->getWhere() = {}; + if (query_node->hasPrewhere()) + replaceFilterExpression(query_node->getPrewhere(), replacement_table_expression, context); + if (query_node->hasWhere()) + replaceFilterExpression(query_node->getWhere(), replacement_table_expression, context); + query_node->getGroupBy().getNodes().clear(); query_node->getHaving() = {}; query_node->getOrderBy().getNodes().clear(); - if (join_tree->as() == nullptr && join_tree->as() == nullptr) + auto & projection = modified_query_node->getProjection().getNodes(); + projection.clear(); + NamesAndTypes projection_columns; + + for (auto const & column_name : required_column_names) { - auto & projection = modified_query_node->getProjection().getNodes(); - projection.clear(); - NamesAndTypes projection_columns; + QueryTreeNodePtr fake_node = std::make_shared(Identifier{column_name}); - for (auto const & column_name : required_column_names) - { - QueryTreeNodePtr fake_node = std::make_shared(Identifier{column_name}); + QueryAnalysisPass query_analysis_pass(original_table_expression); + query_analysis_pass.run(fake_node, context); - QueryAnalysisPass query_analysis_pass(original_table_expression); - query_analysis_pass.run(fake_node, context); + auto * resolved_column = fake_node->as(); + if (!resolved_column) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Required column '{}' is not resolved", column_name); + auto fake_column = resolved_column->getColumn(); - auto * resolved_column = fake_node->as(); - if (!resolved_column) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Required column '{}' is not resolved", column_name); - auto fake_column = resolved_column->getColumn(); + ApplyAliasColumnExpressionsVisitor visitor; + visitor.visit(fake_node); - ApplyAliasColumnExpressionsVisitor visitor; - visitor.visit(fake_node); - - projection.push_back(fake_node); - projection_columns.push_back(fake_column); - } - - query_node->resolveProjectionColumns(std::move(projection_columns)); + projection.push_back(fake_node); + projection_columns.push_back(fake_column); } + query_node->resolveProjectionColumns(std::move(projection_columns)); + LOG_DEBUG(&Poco::Logger::get("removeJoin"), "Result:\n{}", modified_query->dumpTree()); return modified_query; @@ -746,7 +817,7 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ if (query_info.table_expression_modifiers) replacement_table_expression->setTableExpressionModifiers(*query_info.table_expression_modifiers); - modified_query_info.query_tree = removeJoin(modified_query_info.query_tree, modified_query_info.table_expression, replacement_table_expression, modified_context, required_column_names); + modified_query_info.query_tree = replaceTableExpressionAndRemoveJoin(modified_query_info.query_tree, modified_query_info.table_expression, replacement_table_expression, modified_context, required_column_names); modified_query_info.table_expression = replacement_table_expression; modified_query_info.planner_context->getOrCreateTableExpressionData(replacement_table_expression); From ff4729ccc4adbedeb3f70d5221820e9f54febaab Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 7 Feb 2024 16:40:23 +0100 Subject: [PATCH 17/18] Improve replaceTableExpressionAndRemoveJoin function --- src/Storages/StorageMerge.cpp | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index df5b0cd715d..301ff3f37d5 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -60,7 +60,7 @@ #include #include #include -#include "Core/NamesAndTypes.h" +#include #include namespace @@ -625,17 +625,25 @@ namespace class ApplyAliasColumnExpressionsVisitor : public InDepthQueryTreeVisitor { public: - ApplyAliasColumnExpressionsVisitor() = default; + explicit ApplyAliasColumnExpressionsVisitor(QueryTreeNodePtr replacement_table_expression_) + : replacement_table_expression(replacement_table_expression_) + {} void visitImpl(QueryTreeNodePtr & node) { - if (auto * column = node->as(); - column != nullptr && column->hasExpression()) + if (auto * column = node->as(); column != nullptr) { - node = column->getExpressionOrThrow(); - node->setAlias(column->getColumnName()); + if (column->hasExpression()) + { + node = column->getExpressionOrThrow(); + node->setAlias(column->getColumnName()); + } + else + column->setColumnSource(replacement_table_expression); } } +private: + QueryTreeNodePtr replacement_table_expression; }; bool hasUnknownColumn(const QueryTreeNodePtr & node, QueryTreeNodePtr replacement_table_expression) @@ -783,7 +791,7 @@ QueryTreeNodePtr replaceTableExpressionAndRemoveJoin( throw Exception(ErrorCodes::LOGICAL_ERROR, "Required column '{}' is not resolved", column_name); auto fake_column = resolved_column->getColumn(); - ApplyAliasColumnExpressionsVisitor visitor; + ApplyAliasColumnExpressionsVisitor visitor(replacement_table_expression); visitor.visit(fake_node); projection.push_back(fake_node); @@ -865,7 +873,7 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ auto * resolved_column = fake_node->as(); column_node = fake_node; - ApplyAliasColumnExpressionsVisitor visitor; + ApplyAliasColumnExpressionsVisitor visitor(replacement_table_expression); visitor.visit(column_node); if (!resolved_column || !resolved_column->getExpression()) From 813020a1a6f4462dea1edec3cc42e000206b6327 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 7 Feb 2024 16:44:09 +0100 Subject: [PATCH 18/18] remove redundant logging --- src/Storages/StorageMerge.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 301ff3f37d5..d3b8f30b1c5 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -800,8 +800,6 @@ QueryTreeNodePtr replaceTableExpressionAndRemoveJoin( query_node->resolveProjectionColumns(std::move(projection_columns)); - LOG_DEBUG(&Poco::Logger::get("removeJoin"), "Result:\n{}", modified_query->dumpTree()); - return modified_query; }