From c975f8156af1b2c109e12c771ac0e1a226ec26f6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 26 Nov 2020 19:16:44 +0300 Subject: [PATCH 1/9] Add ActionsDAG::merge --- src/Interpreters/ActionsDAG.cpp | 91 +++++++++++++++++++++++++++++++++ src/Interpreters/ActionsDAG.h | 14 +++++ 2 files changed, 105 insertions(+) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 3ea48af7daa..00ba4d6d21c 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -670,6 +670,97 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions( return actions_dag; } +ActionsDAGPtr ActionsDAG::merge(ActionsDAG && lhs, ActionsDAG && rhs) +{ + /// lhs: x (1), x (2), y ==> x (2), z, x (3) + /// rhs: x (1), x (2), x (3) ==> x (3), x (2), x (1) + /// merge: x (1), x (2), x (3), y =(lhs)=> x (3), y, x (2), z, x (4) =(rhs)=> y, z, x (4), x (2), x (3) + + /// Will store merged result in lhs. + + std::unordered_set removed_lhs_result; + std::unordered_map inputs_map; + + /// Update inputs list. + { + std::unordered_map> lhs_result; + for (auto & node : lhs.index) + lhs_result[node->result_name].push_back(node); + + for (auto & node : rhs.inputs) + { + auto it = lhs_result.find(node->result_name); + if (it == lhs_result.end() || it->second.empty()) + { + if (lhs.settings.project_input) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot find column {} in ActionsDAG result", node->result_name); + + lhs.inputs.push_back(node); + } + else + { + inputs_map[node] = it->second.front(); + removed_lhs_result.emplace(it->second.front()); + it->second.pop_front(); + } + } + } + + /// Update index. + if (rhs.settings.project_input) + { + lhs.index.swap(rhs.index); + lhs.settings.project_input = true; + } + else + { + /// Remove rhs inputs from index. + for (auto it = lhs.index.begin(); it != lhs.index.end();) + { + auto cur = it; + ++it; + + if (removed_lhs_result.count(*cur)) + lhs.index.remove(cur); + } + + for (auto * node : rhs.index) + lhs.index.insert(node); + } + + /// Replace inputs from rhs to nodes from lhs result. + for (auto & node : rhs.nodes) + { + for (auto & child : node.children) + { + if (child->type == ActionType::INPUT) + { + auto it = inputs_map.find(child); + if (it != inputs_map.end()) + child = it->second; + } + } + } + + lhs.nodes.splice(lhs.nodes.end(), std::move(rhs.nodes)); + +#if USE_EMBEDDED_COMPILER + if (lhs.compilation_cache == nullptr) + lhs.compilation_cache = rhs.compilation_cache; +#endif + + lhs.settings.max_temporary_columns = std::max(lhs.settings.max_temporary_columns, rhs.settings.max_temporary_columns); + lhs.settings.max_temporary_non_const_columns = std::max(lhs.settings.max_temporary_non_const_columns, rhs.settings.max_temporary_non_const_columns); + lhs.settings.min_count_to_compile_expression = std::max(lhs.settings.min_count_to_compile_expression, rhs.settings.min_count_to_compile_expression); + lhs.settings.projected_output = rhs.settings.projected_output; + + /// Drop unused inputs and, probably, some actions. + lhs.removeUnusedActions(); + + return std::make_shared(std::move(lhs)); +} + ActionsDAGPtr ActionsDAG::splitActionsBeforeArrayJoin(const NameSet & array_joined_columns) { /// Split DAG into two parts. diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index ca54dab7231..704415c91d2 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -143,6 +143,15 @@ public: map.erase(it); } + void remove(std::list::iterator it) + { + auto map_it = map.find((*it)->result_name); + if (map_it != map.end() && map_it->second == it) + map.erase(map_it); + + list.erase(it); + } + void swap(Index & other) { list.swap(other.list); @@ -176,6 +185,7 @@ private: public: ActionsDAG() = default; + ActionsDAG(ActionsDAG &&) = default; ActionsDAG(const ActionsDAG &) = delete; ActionsDAG & operator=(const ActionsDAG &) = delete; explicit ActionsDAG(const NamesAndTypesList & inputs_); @@ -248,6 +258,10 @@ public: MatchColumnsMode mode, bool ignore_constant_values = false); /// Do not check that constants are same. Use value from result_header. + /// Create ActionsDAG which represents expression equivalent to applying lhs and rhs actions consequently. + /// Is used to replace `(lhs -> rhs)` expression chain to single `merge(lhs, rhs)` expression. + static ActionsDAGPtr merge(ActionsDAG && lhs, ActionsDAG && rhs); + private: Node & addNode(Node node, bool can_replace = false); Node & getNode(const std::string & name); From c42cc8f1fff80b8303aa311573f54531115ce010 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 26 Nov 2020 22:09:42 +0300 Subject: [PATCH 2/9] Merge Expressions in QueryPlan. --- src/Processors/QueryPlan/QueryPlan.cpp | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index 206a46123dc..bd999d124ab 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -484,6 +484,29 @@ static void tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Node * : filter_step->updateInputStream(array_join_step->getOutputStream(), true); } +static bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Node * child_node) +{ + auto & parent = parent_node->step; + auto & child = child_node->step; + auto * parent_expr = typeid_cast(parent.get()); + auto * child_expr = typeid_cast(child.get()); + + if (parent_expr && child_expr) + { + auto merged = ActionsDAG::merge(std::move(*child_expr->getExpression()), + std::move(*parent_expr->getExpression())); + + auto expr = std::make_unique(child_expr->getInputStreams().front(), merged); + expr->setStepDescription(parent_expr->getStepDescription() + " + " + child_expr->getStepDescription()); + + parent_node->step = std::move(expr); + parent_node->children.swap(child_node->children); + return true; + } + + return false; +} + void QueryPlan::optimize() { struct Frame @@ -503,7 +526,11 @@ void QueryPlan::optimize() { /// First entrance, try push down. if (frame.node->children.size() == 1) + { tryPushDownLimit(frame.node->step, frame.node->children.front()); + + while (tryMergeExpressions(frame.node, frame.node->children.front())); + } } if (frame.next_child < frame.node->children.size()) From 6ed4916d5db5b7823cf11d01c64babf6409565da Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 26 Nov 2020 22:48:21 +0300 Subject: [PATCH 3/9] Fix ActionsDAG::merge --- src/Interpreters/ActionsDAG.cpp | 37 +++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 00ba4d6d21c..874a06c7ef5 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -707,6 +707,30 @@ ActionsDAGPtr ActionsDAG::merge(ActionsDAG && lhs, ActionsDAG && rhs) } } + /// Replace inputs from rhs to nodes from lhs result. + for (auto & node : rhs.nodes) + { + for (auto & child : node.children) + { + if (child->type == ActionType::INPUT) + { + auto it = inputs_map.find(child); + if (it != inputs_map.end()) + child = it->second; + } + } + } + + for (auto & node : rhs.index) + { + if (node->type == ActionType::INPUT) + { + auto it = inputs_map.find(node); + if (it != inputs_map.end()) + node = it->second; + } + } + /// Update index. if (rhs.settings.project_input) { @@ -729,19 +753,6 @@ ActionsDAGPtr ActionsDAG::merge(ActionsDAG && lhs, ActionsDAG && rhs) lhs.index.insert(node); } - /// Replace inputs from rhs to nodes from lhs result. - for (auto & node : rhs.nodes) - { - for (auto & child : node.children) - { - if (child->type == ActionType::INPUT) - { - auto it = inputs_map.find(child); - if (it != inputs_map.end()) - child = it->second; - } - } - } lhs.nodes.splice(lhs.nodes.end(), std::move(rhs.nodes)); From 600f249790630ffc6948983033f71babdaba0374 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 27 Nov 2020 12:11:34 +0300 Subject: [PATCH 4/9] Do not remove arrayJoin from actions --- src/Interpreters/ActionsDAG.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 874a06c7ef5..d3832e86bc5 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -349,6 +349,16 @@ void ActionsDAG::removeUnusedActions() stack.push(node); } + /// We cannot remove arrayJoin because it changes the number of rows. + for (auto & node : nodes) + { + if (node.type == ActionType::ARRAY_JOIN && visited_nodes.count(&node) == 0) + { + visited_nodes.insert(&node); + stack.push(&node); + } + } + while (!stack.empty()) { auto * node = stack.top(); From a2c450d7e2b78634c764b5718cc7bbf89c7897c4 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 27 Nov 2020 12:50:56 +0300 Subject: [PATCH 5/9] Do not merge functions with arrayJoin and stateful functions. --- src/Interpreters/ActionsDAG.cpp | 9 +++++++++ src/Interpreters/ActionsDAG.h | 1 + src/Processors/QueryPlan/QueryPlan.cpp | 12 ++++++++++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index d3832e86bc5..781e1493e78 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -562,6 +562,15 @@ bool ActionsDAG::hasArrayJoin() const return false; } +bool ActionsDAG::hasStatefulFunctions() const +{ + for (const auto & node : nodes) + if (node.type == ActionType::FUNCTION && node.function_base->isStateful()) + return true; + + return false; +} + bool ActionsDAG::empty() const { for (const auto & node : nodes) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 704415c91d2..f13fd0fe442 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -232,6 +232,7 @@ public: ActionsDAGPtr splitActionsBeforeArrayJoin(const NameSet & array_joined_columns); bool hasArrayJoin() const; + bool hasStatefulFunctions() const; bool empty() const; /// If actions only contain inputs. const ActionsSettings & getSettings() const { return settings; } diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index bd999d124ab..db6194ce0a0 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -493,8 +493,16 @@ static bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Node * if (parent_expr && child_expr) { - auto merged = ActionsDAG::merge(std::move(*child_expr->getExpression()), - std::move(*parent_expr->getExpression())); + const auto & child_actions = child_expr->getExpression(); + const auto & parent_actions = parent_expr->getExpression(); + + /// We cannot combine actions with arrayJoin and stateful function because we not always can reorder them. + /// Example: select rowNumberInBlock() from (select arrayJoin([1, 2])) + /// Such a query will return two zeroes if we combine actions together. + if (child_actions->hasArrayJoin() && parent_actions->hasStatefulFunctions()) + return false; + + auto merged = ActionsDAG::merge(std::move(*child_actions), std::move(*parent_actions)); auto expr = std::make_unique(child_expr->getInputStreams().front(), merged); expr->setStepDescription(parent_expr->getStepDescription() + " + " + child_expr->getStepDescription()); From c206b2506bcbedc90b81ee8d7d9fc8da245b3e41 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 27 Nov 2020 13:10:09 +0300 Subject: [PATCH 6/9] Update tests. --- .../01508_explain_header.reference | 9 ++-- ...1_mergetree_read_in_order_spread.reference | 54 +++++++++---------- 2 files changed, 29 insertions(+), 34 deletions(-) diff --git a/tests/queries/0_stateless/01508_explain_header.reference b/tests/queries/0_stateless/01508_explain_header.reference index ba6c5f03dce..8cab5be1687 100644 --- a/tests/queries/0_stateless/01508_explain_header.reference +++ b/tests/queries/0_stateless/01508_explain_header.reference @@ -1,9 +1,6 @@ -Expression (Projection) +Expression (Projection + Before ORDER BY and SELECT) Header: x UInt8 - Expression (Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) Header: dummy UInt8 - 1 UInt8 - SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Header: dummy UInt8 - ReadFromStorage (SystemOne) - Header: dummy UInt8 diff --git a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference index adbb03ae018..fb07aec9dd5 100644 --- a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference +++ b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference @@ -1,30 +1,28 @@ (Expression) ExpressionTransform - (Expression) - ExpressionTransform - (Aggregating) - FinalizingSimpleTransform - AggregatingSortedTransform 3 → 1 - AggregatingInOrderTransform × 3 - (Expression) - ExpressionTransform × 3 - (SettingQuotaAndLimits) - (Expression) - ExpressionTransform × 3 - (Union) - (MergingSorted) - (Expression) - ExpressionTransform - (ReadFromStorage) - MergeTree 0 → 1 - (MergingSorted) - MergingSortedTransform 2 → 1 - (Expression) - ExpressionTransform × 2 - (ReadFromStorage) - MergeTree × 2 0 → 1 - (MergingSorted) - (Expression) - ExpressionTransform - (ReadFromStorage) - MergeTree 0 → 1 + (Aggregating) + FinalizingSimpleTransform + AggregatingSortedTransform 3 → 1 + AggregatingInOrderTransform × 3 + (Expression) + ExpressionTransform × 3 + (SettingQuotaAndLimits) + (Expression) + ExpressionTransform × 3 + (Union) + (MergingSorted) + (Expression) + ExpressionTransform + (ReadFromStorage) + MergeTree 0 → 1 + (MergingSorted) + MergingSortedTransform 2 → 1 + (Expression) + ExpressionTransform × 2 + (ReadFromStorage) + MergeTree × 2 0 → 1 + (MergingSorted) + (Expression) + ExpressionTransform + (ReadFromStorage) + MergeTree 0 → 1 From f3bea2210cd4a1f0153562436975ed64d131159b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 27 Nov 2020 13:55:45 +0300 Subject: [PATCH 7/9] Update tests. --- ..._explain_select_with_union_query.reference | 498 ++++++++---------- 1 file changed, 213 insertions(+), 285 deletions(-) diff --git a/tests/queries/0_stateless/01556_explain_select_with_union_query.reference b/tests/queries/0_stateless/01556_explain_select_with_union_query.reference index 2d09a1f8625..442754ec344 100644 --- a/tests/queries/0_stateless/01556_explain_select_with_union_query.reference +++ b/tests/queries/0_stateless/01556_explain_select_with_union_query.reference @@ -1,324 +1,252 @@ Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) +Union + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) +Distinct + Union + Expression (Projection + Before ORDER BY and SELECT) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) + Expression (Projection + Before ORDER BY and SELECT) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) +Distinct + Union + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) +Distinct + Union + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) +Distinct + Union + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromStorage (SystemOne) Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) -Distinct - Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) -Distinct - Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) -Distinct - Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) -Distinct - Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) -Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) -Distinct - Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) + Expression (Projection + Before ORDER BY and SELECT) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) + Expression (Projection + Before ORDER BY and SELECT) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromStorage (SystemOne) Distinct Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) -Distinct - Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) -Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) + Expression (Projection + Before ORDER BY and SELECT) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) + Expression (Projection + Before ORDER BY and SELECT) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) +Distinct + Union + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) +Distinct + Union + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromStorage (SystemOne) Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) +Union + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) +Union + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) +Distinct + Union + Expression (Projection + Before ORDER BY and SELECT) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) + Expression (Projection + Before ORDER BY and SELECT) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) +Distinct + Union + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) +Distinct + Union + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromStorage (SystemOne) Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) -Distinct - Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) -Distinct - Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) -Distinct - Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) -Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - SettingQuotaAndLimits (Set limits and quota after reading from storage) - ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) + Expression (Projection + Before ORDER BY and SELECT) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) From d9773942a2c1557574bde35394aabcc7a5a295c9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 1 Dec 2020 14:19:03 +0300 Subject: [PATCH 8/9] Rewiev fixes. --- src/Interpreters/ActionsDAG.cpp | 75 +++++++++++++++++---------------- src/Interpreters/ActionsDAG.h | 6 ++- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 781e1493e78..75915bdad1a 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -689,45 +689,48 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions( return actions_dag; } -ActionsDAGPtr ActionsDAG::merge(ActionsDAG && lhs, ActionsDAG && rhs) +ActionsDAGPtr ActionsDAG::merge(ActionsDAG && first, ActionsDAG && second) { - /// lhs: x (1), x (2), y ==> x (2), z, x (3) - /// rhs: x (1), x (2), x (3) ==> x (3), x (2), x (1) - /// merge: x (1), x (2), x (3), y =(lhs)=> x (3), y, x (2), z, x (4) =(rhs)=> y, z, x (4), x (2), x (3) + /// first: x (1), x (2), y ==> x (2), z, x (3) + /// second: x (1), x (2), x (3) ==> x (3), x (2), x (1) + /// merge: x (1), x (2), x (3), y =(first)=> x (3), y, x (2), z, x (4) =(second)=> y, z, x (4), x (2), x (3) - /// Will store merged result in lhs. + /// Will store merged result in `first`. - std::unordered_set removed_lhs_result; + /// This map contains nodes which should be removed from `first` index, cause they are used as inputs for `second`. + std::unordered_set removed_first_result; + /// Map inputs of `second` to nodes of `first`. std::unordered_map inputs_map; /// Update inputs list. { - std::unordered_map> lhs_result; - for (auto & node : lhs.index) - lhs_result[node->result_name].push_back(node); + /// Index may have multiple columns with same name. They also may be used by `second`. Order is important. + std::unordered_map> first_result; + for (auto & node : first.index) + first_result[node->result_name].push_back(node); - for (auto & node : rhs.inputs) + for (auto & node : second.inputs) { - auto it = lhs_result.find(node->result_name); - if (it == lhs_result.end() || it->second.empty()) + auto it = first_result.find(node->result_name); + if (it == first_result.end() || it->second.empty()) { - if (lhs.settings.project_input) + if (first.settings.project_input) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find column {} in ActionsDAG result", node->result_name); - lhs.inputs.push_back(node); + first.inputs.push_back(node); } else { inputs_map[node] = it->second.front(); - removed_lhs_result.emplace(it->second.front()); + removed_first_result.emplace(it->second.front()); it->second.pop_front(); } } } - /// Replace inputs from rhs to nodes from lhs result. - for (auto & node : rhs.nodes) + /// Replace inputs from `second` to nodes from `first` result. + for (auto & node : second.nodes) { for (auto & child : node.children) { @@ -740,7 +743,7 @@ ActionsDAGPtr ActionsDAG::merge(ActionsDAG && lhs, ActionsDAG && rhs) } } - for (auto & node : rhs.index) + for (auto & node : second.index) { if (node->type == ActionType::INPUT) { @@ -751,44 +754,44 @@ ActionsDAGPtr ActionsDAG::merge(ActionsDAG && lhs, ActionsDAG && rhs) } /// Update index. - if (rhs.settings.project_input) + if (second.settings.project_input) { - lhs.index.swap(rhs.index); - lhs.settings.project_input = true; + first.index.swap(second.index); + first.settings.project_input = true; } else { - /// Remove rhs inputs from index. - for (auto it = lhs.index.begin(); it != lhs.index.end();) + /// Remove `second` inputs from index. + for (auto it = first.index.begin(); it != first.index.end();) { auto cur = it; ++it; - if (removed_lhs_result.count(*cur)) - lhs.index.remove(cur); + if (removed_first_result.count(*cur)) + first.index.remove(cur); } - for (auto * node : rhs.index) - lhs.index.insert(node); + for (auto * node : second.index) + first.index.insert(node); } - lhs.nodes.splice(lhs.nodes.end(), std::move(rhs.nodes)); + first.nodes.splice(first.nodes.end(), std::move(second.nodes)); #if USE_EMBEDDED_COMPILER - if (lhs.compilation_cache == nullptr) - lhs.compilation_cache = rhs.compilation_cache; + if (first.compilation_cache == nullptr) + first.compilation_cache = second.compilation_cache; #endif - lhs.settings.max_temporary_columns = std::max(lhs.settings.max_temporary_columns, rhs.settings.max_temporary_columns); - lhs.settings.max_temporary_non_const_columns = std::max(lhs.settings.max_temporary_non_const_columns, rhs.settings.max_temporary_non_const_columns); - lhs.settings.min_count_to_compile_expression = std::max(lhs.settings.min_count_to_compile_expression, rhs.settings.min_count_to_compile_expression); - lhs.settings.projected_output = rhs.settings.projected_output; + first.settings.max_temporary_columns = std::max(first.settings.max_temporary_columns, second.settings.max_temporary_columns); + first.settings.max_temporary_non_const_columns = std::max(first.settings.max_temporary_non_const_columns, second.settings.max_temporary_non_const_columns); + first.settings.min_count_to_compile_expression = std::max(first.settings.min_count_to_compile_expression, second.settings.min_count_to_compile_expression); + first.settings.projected_output = second.settings.projected_output; /// Drop unused inputs and, probably, some actions. - lhs.removeUnusedActions(); + first.removeUnusedActions(); - return std::make_shared(std::move(lhs)); + return std::make_shared(std::move(first)); } ActionsDAGPtr ActionsDAG::splitActionsBeforeArrayJoin(const NameSet & array_joined_columns) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index f13fd0fe442..a0ac210e752 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -260,8 +260,10 @@ public: bool ignore_constant_values = false); /// Do not check that constants are same. Use value from result_header. /// Create ActionsDAG which represents expression equivalent to applying lhs and rhs actions consequently. - /// Is used to replace `(lhs -> rhs)` expression chain to single `merge(lhs, rhs)` expression. - static ActionsDAGPtr merge(ActionsDAG && lhs, ActionsDAG && rhs); + /// Is used to replace `(first -> second)` expression chain to single `merge(first, second)` expression. + /// If first.settings.project_input is set, then outputs of `first` must include inputs of `second`. + /// Otherwise, any two actions may be combined. + static ActionsDAGPtr merge(ActionsDAG && first, ActionsDAG && second); private: Node & addNode(Node node, bool can_replace = false); From e57ad9cb7e061bbbf35f2caf90a59ea387caed06 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 4 Dec 2020 11:41:36 +0300 Subject: [PATCH 9/9] Update QueryPlan.cpp --- src/Processors/QueryPlan/QueryPlan.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index db6194ce0a0..e88090b4819 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -488,6 +488,7 @@ static bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Node * { auto & parent = parent_node->step; auto & child = child_node->step; + /// TODO: FilterStep auto * parent_expr = typeid_cast(parent.get()); auto * child_expr = typeid_cast(child.get());