From e9f893d9b0c8906b00252c6fdc97ae54069f57f4 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 27 Jan 2021 16:57:26 +0300 Subject: [PATCH 1/8] Move query plan optimixations into separate files. --- src/CMakeLists.txt | 1 + .../QueryPlan/Optimizations/Optimizations.h | 24 ++ .../Optimizations/liftUpArrayJoin.cpp | 79 +++++ .../QueryPlan/Optimizations/limitPushDown.cpp | 107 +++++++ .../Optimizations/mergeExpressions.cpp | 60 ++++ .../QueryPlan/Optimizations/splitFilter.cpp | 50 ++++ src/Processors/QueryPlan/QueryPlan.cpp | 278 +----------------- src/Processors/ya.make | 4 + 8 files changed, 333 insertions(+), 270 deletions(-) create mode 100644 src/Processors/QueryPlan/Optimizations/Optimizations.h create mode 100644 src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp create mode 100644 src/Processors/QueryPlan/Optimizations/limitPushDown.cpp create mode 100644 src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp create mode 100644 src/Processors/QueryPlan/Optimizations/splitFilter.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4207d371c09..dba9385fe27 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -190,6 +190,7 @@ add_object_library(clickhouse_processors_sources Processors/Sources) add_object_library(clickhouse_processors_merges Processors/Merges) add_object_library(clickhouse_processors_merges_algorithms Processors/Merges/Algorithms) add_object_library(clickhouse_processors_queryplan Processors/QueryPlan) +add_object_library(clickhouse_processors_queryplan_optimizations Processors/QueryPlan/Optimizations) set (DBMS_COMMON_LIBRARIES) # libgcc_s does not provide an implementation of an atomics library. Instead, diff --git a/src/Processors/QueryPlan/Optimizations/Optimizations.h b/src/Processors/QueryPlan/Optimizations/Optimizations.h new file mode 100644 index 00000000000..93b4be90a98 --- /dev/null +++ b/src/Processors/QueryPlan/Optimizations/Optimizations.h @@ -0,0 +1,24 @@ +#include + +namespace DB +{ + +namespace QueryPlanOptimizations +{ + +/// Move LimitStep down if possible. +void tryPushDownLimit(QueryPlanStepPtr & parent, QueryPlan::Node * child_node); + +/// Split FilterStep into chain `ExpressionStep -> FilterStep`, where FilterStep contains minimal number of nodes. +bool trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes); + +/// Replace chain `ExpressionStep -> ExpressionStep` to single ExpressionStep +/// Replace chain `FilterStep -> ExpressionStep` to single FilterStep +bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Node * child_node); + +/// Move ARRAY JOIN up if possible. +void tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Node * child_node, QueryPlan::Nodes & nodes); + +} + +} diff --git a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp new file mode 100644 index 00000000000..474124d970d --- /dev/null +++ b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp @@ -0,0 +1,79 @@ +#include +#include +#include +#include +#include +#include + +namespace DB::QueryPlanOptimizations +{ + +void tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Node * child_node, QueryPlan::Nodes & nodes) +{ + auto & parent = parent_node->step; + auto & child = child_node->step; + auto * expression_step = typeid_cast(parent.get()); + auto * filter_step = typeid_cast(parent.get()); + auto * array_join_step = typeid_cast(child.get()); + + if (!(expression_step || filter_step) || !array_join_step) + return; + + const auto & array_join = array_join_step->arrayJoin(); + const auto & expression = expression_step ? expression_step->getExpression() + : filter_step->getExpression(); + + auto split_actions = expression->splitActionsBeforeArrayJoin(array_join->columns); + + /// No actions can be moved before ARRAY JOIN. + if (split_actions.first->empty()) + return; + + auto description = parent->getStepDescription(); + + /// All actions was moved before ARRAY JOIN. Swap Expression and ArrayJoin. + if (split_actions.second->empty()) + { + auto expected_header = parent->getOutputStream().header; + + /// Expression/Filter -> ArrayJoin + std::swap(parent, child); + /// ArrayJoin -> Expression/Filter + + if (expression_step) + child = std::make_unique(child_node->children.at(0)->step->getOutputStream(), + std::move(split_actions.first)); + else + child = std::make_unique(child_node->children.at(0)->step->getOutputStream(), + std::move(split_actions.first), + filter_step->getFilterColumnName(), + filter_step->removesFilterColumn()); + + child->setStepDescription(std::move(description)); + + array_join_step->updateInputStream(child->getOutputStream(), expected_header); + return; + } + + /// Add new expression step before ARRAY JOIN. + /// Expression/Filter -> ArrayJoin -> Something + auto & node = nodes.emplace_back(); + node.children.swap(child_node->children); + child_node->children.emplace_back(&node); + /// Expression/Filter -> ArrayJoin -> node -> Something + + node.step = std::make_unique(node.children.at(0)->step->getOutputStream(), + std::move(split_actions.first)); + node.step->setStepDescription(description); + array_join_step->updateInputStream(node.step->getOutputStream(), {}); + + if (expression_step) + parent = std::make_unique(array_join_step->getOutputStream(), split_actions.second); + else + parent = std::make_unique(array_join_step->getOutputStream(), split_actions.second, + filter_step->getFilterColumnName(), filter_step->removesFilterColumn()); + + parent->setStepDescription(description + " [split]"); +} + +} diff --git a/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp b/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp new file mode 100644 index 00000000000..8d2f5f50fc8 --- /dev/null +++ b/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp @@ -0,0 +1,107 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::QueryPlanOptimizations +{ + +/// If plan looks like Limit -> Sorting, update limit for Sorting +static bool tryUpdateLimitForSortingSteps(QueryPlan::Node * node, size_t limit) +{ + if (limit == 0) + return false; + + QueryPlanStepPtr & step = node->step; + QueryPlan::Node * child = nullptr; + bool updated = false; + + if (auto * merging_sorted = typeid_cast(step.get())) + { + /// TODO: remove LimitStep here. + merging_sorted->updateLimit(limit); + updated = true; + child = node->children.front(); + } + else if (auto * finish_sorting = typeid_cast(step.get())) + { + /// TODO: remove LimitStep here. + finish_sorting->updateLimit(limit); + updated = true; + } + else if (auto * merge_sorting = typeid_cast(step.get())) + { + merge_sorting->updateLimit(limit); + updated = true; + child = node->children.front(); + } + else if (auto * partial_sorting = typeid_cast(step.get())) + { + partial_sorting->updateLimit(limit); + updated = true; + } + + /// We often have chain PartialSorting -> MergeSorting -> MergingSorted + /// Try update limit for them also if possible. + if (child) + tryUpdateLimitForSortingSteps(child, limit); + + return updated; +} + +void tryPushDownLimit(QueryPlanStepPtr & parent, QueryPlan::Node * child_node) +{ + auto & child = child_node->step; + auto * limit = typeid_cast(parent.get()); + + if (!limit) + return; + + /// Skip LIMIT WITH TIES by now. + if (limit->withTies()) + return; + + const auto * transforming = dynamic_cast(child.get()); + + /// Skip everything which is not transform. + if (!transforming) + return; + + /// Special cases for sorting steps. + if (tryUpdateLimitForSortingSteps(child_node, limit->getLimitForSorting())) + return; + + /// Special case for TotalsHaving. Totals may be incorrect if we push down limit. + if (typeid_cast(child.get())) + return; + + /// Now we should decide if pushing down limit possible for this step. + + const auto & transform_traits = transforming->getTransformTraits(); + const auto & data_stream_traits = transforming->getDataStreamTraits(); + + /// Cannot push down if child changes the number of rows. + if (!transform_traits.preserves_number_of_rows) + return; + + /// Cannot push down if data was sorted exactly by child stream. + if (!child->getOutputStream().sort_description.empty() && !data_stream_traits.preserves_sorting) + return; + + /// Now we push down limit only if it doesn't change any stream properties. + /// TODO: some of them may be changed and, probably, not important for following streams. We may add such info. + if (!limit->getOutputStream().hasEqualPropertiesWith(transforming->getOutputStream())) + return; + + /// Input stream for Limit have changed. + limit->updateInputStream(transforming->getInputStreams().front()); + + parent.swap(child); +} + +} diff --git a/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp b/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp new file mode 100644 index 00000000000..3bbfe0e1efb --- /dev/null +++ b/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp @@ -0,0 +1,60 @@ +#include +#include +#include +#include + +namespace DB::QueryPlanOptimizations +{ + +bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Node * child_node) +{ + auto & parent = parent_node->step; + auto & child = child_node->step; + + auto * parent_expr = typeid_cast(parent.get()); + auto * parent_filter = typeid_cast(parent.get()); + auto * child_expr = typeid_cast(child.get()); + + if (parent_expr && child_expr) + { + const auto & child_actions = child_expr->getExpression(); + const auto & parent_actions = parent_expr->getExpression(); + + /// We cannot combine actions with arrayJoin and stateful function because we not always can reorder them. + /// Example: select rowNumberInBlock() from (select arrayJoin([1, 2])) + /// Such a query will return two zeroes if we combine actions together. + if (child_actions->hasArrayJoin() && parent_actions->hasStatefulFunctions()) + return false; + + auto merged = ActionsDAG::merge(std::move(*child_actions), std::move(*parent_actions)); + + auto expr = std::make_unique(child_expr->getInputStreams().front(), merged); + expr->setStepDescription("(" + parent_expr->getStepDescription() + " + " + child_expr->getStepDescription() + ")"); + + parent_node->step = std::move(expr); + parent_node->children.swap(child_node->children); + return true; + } + else if (parent_filter && child_expr) + { + const auto & child_actions = child_expr->getExpression(); + const auto & parent_actions = parent_filter->getExpression(); + + if (child_actions->hasArrayJoin() && parent_actions->hasStatefulFunctions()) + return false; + + auto merged = ActionsDAG::merge(std::move(*child_actions), std::move(*parent_actions)); + + auto filter = std::make_unique(child_expr->getInputStreams().front(), merged, + parent_filter->getFilterColumnName(), parent_filter->removesFilterColumn()); + filter->setStepDescription("(" + parent_filter->getStepDescription() + " + " + child_expr->getStepDescription() + ")"); + + parent_node->step = std::move(filter); + parent_node->children.swap(child_node->children); + return true; + } + + return false; +} + +} diff --git a/src/Processors/QueryPlan/Optimizations/splitFilter.cpp b/src/Processors/QueryPlan/Optimizations/splitFilter.cpp new file mode 100644 index 00000000000..fd82bd69a9e --- /dev/null +++ b/src/Processors/QueryPlan/Optimizations/splitFilter.cpp @@ -0,0 +1,50 @@ +#include +#include +#include +#include + +namespace DB::QueryPlanOptimizations +{ + +/// Split FilterStep into chain `ExpressionStep -> FilterStep`, where FilterStep contains minimal number of nodes. +bool trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes) +{ + auto * filter_step = typeid_cast(node->step.get()); + if (!filter_step) + return false; + + const auto & expr = filter_step->getExpression(); + + /// Do not split if there are function like runningDifference. + if (expr->hasStatefulFunctions()) + return false; + + auto split = expr->splitActionsForFilter(filter_step->getFilterColumnName()); + + if (split.second->empty()) + return false; + + if (filter_step->removesFilterColumn()) + split.second->removeUnusedInput(filter_step->getFilterColumnName()); + + auto description = filter_step->getStepDescription(); + + auto & filter_node = nodes.emplace_back(); + node->children.swap(filter_node.children); + node->children.push_back(&filter_node); + + filter_node.step = std::make_unique( + filter_node.children.at(0)->step->getOutputStream(), + std::move(split.first), + filter_step->getFilterColumnName(), + filter_step->removesFilterColumn()); + + node->step = std::make_unique(filter_node.step->getOutputStream(), std::move(split.second)); + + filter_node.step->setStepDescription("(" + description + ")[split]"); + node->step->setStepDescription(description); + + return true; +} + +} diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index 6fba991a56c..49249e99864 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -6,12 +6,7 @@ #include #include #include -#include -#include "MergingSortedStep.h" -#include "FinishSortingStep.h" -#include "MergeSortingStep.h" -#include "PartialSortingStep.h" -#include "TotalsHavingStep.h" +#include #include "ExpressionStep.h" #include "ArrayJoinStep.h" #include "FilterStep.h" @@ -341,263 +336,6 @@ void QueryPlan::explainPipeline(WriteBuffer & buffer, const ExplainPipelineOptio } } -/// If plan looks like Limit -> Sorting, update limit for Sorting -bool tryUpdateLimitForSortingSteps(QueryPlan::Node * node, size_t limit) -{ - if (limit == 0) - return false; - - QueryPlanStepPtr & step = node->step; - QueryPlan::Node * child = nullptr; - bool updated = false; - - if (auto * merging_sorted = typeid_cast(step.get())) - { - /// TODO: remove LimitStep here. - merging_sorted->updateLimit(limit); - updated = true; - child = node->children.front(); - } - else if (auto * finish_sorting = typeid_cast(step.get())) - { - /// TODO: remove LimitStep here. - finish_sorting->updateLimit(limit); - updated = true; - } - else if (auto * merge_sorting = typeid_cast(step.get())) - { - merge_sorting->updateLimit(limit); - updated = true; - child = node->children.front(); - } - else if (auto * partial_sorting = typeid_cast(step.get())) - { - partial_sorting->updateLimit(limit); - updated = true; - } - - /// We often have chain PartialSorting -> MergeSorting -> MergingSorted - /// Try update limit for them also if possible. - if (child) - tryUpdateLimitForSortingSteps(child, limit); - - return updated; -} - -/// Move LimitStep down if possible. -static void tryPushDownLimit(QueryPlanStepPtr & parent, QueryPlan::Node * child_node) -{ - auto & child = child_node->step; - auto * limit = typeid_cast(parent.get()); - - if (!limit) - return; - - /// Skip LIMIT WITH TIES by now. - if (limit->withTies()) - return; - - const auto * transforming = dynamic_cast(child.get()); - - /// Skip everything which is not transform. - if (!transforming) - return; - - /// Special cases for sorting steps. - if (tryUpdateLimitForSortingSteps(child_node, limit->getLimitForSorting())) - return; - - /// Special case for TotalsHaving. Totals may be incorrect if we push down limit. - if (typeid_cast(child.get())) - return; - - /// Now we should decide if pushing down limit possible for this step. - - const auto & transform_traits = transforming->getTransformTraits(); - const auto & data_stream_traits = transforming->getDataStreamTraits(); - - /// Cannot push down if child changes the number of rows. - if (!transform_traits.preserves_number_of_rows) - return; - - /// Cannot push down if data was sorted exactly by child stream. - if (!child->getOutputStream().sort_description.empty() && !data_stream_traits.preserves_sorting) - return; - - /// Now we push down limit only if it doesn't change any stream properties. - /// TODO: some of them may be changed and, probably, not important for following streams. We may add such info. - if (!limit->getOutputStream().hasEqualPropertiesWith(transforming->getOutputStream())) - return; - - /// Input stream for Limit have changed. - limit->updateInputStream(transforming->getInputStreams().front()); - - parent.swap(child); -} - -/// Move ARRAY JOIN up if possible. -static void tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Node * child_node, QueryPlan::Nodes & nodes) -{ - auto & parent = parent_node->step; - auto & child = child_node->step; - auto * expression_step = typeid_cast(parent.get()); - auto * filter_step = typeid_cast(parent.get()); - auto * array_join_step = typeid_cast(child.get()); - - if (!(expression_step || filter_step) || !array_join_step) - return; - - const auto & array_join = array_join_step->arrayJoin(); - const auto & expression = expression_step ? expression_step->getExpression() - : filter_step->getExpression(); - - auto split_actions = expression->splitActionsBeforeArrayJoin(array_join->columns); - - /// No actions can be moved before ARRAY JOIN. - if (split_actions.first->empty()) - return; - - auto description = parent->getStepDescription(); - - /// All actions was moved before ARRAY JOIN. Swap Expression and ArrayJoin. - if (split_actions.second->empty()) - { - auto expected_header = parent->getOutputStream().header; - - /// Expression/Filter -> ArrayJoin - std::swap(parent, child); - /// ArrayJoin -> Expression/Filter - - if (expression_step) - child = std::make_unique(child_node->children.at(0)->step->getOutputStream(), - std::move(split_actions.first)); - else - child = std::make_unique(child_node->children.at(0)->step->getOutputStream(), - std::move(split_actions.first), - filter_step->getFilterColumnName(), - filter_step->removesFilterColumn()); - - child->setStepDescription(std::move(description)); - - array_join_step->updateInputStream(child->getOutputStream(), expected_header); - return; - } - - /// Add new expression step before ARRAY JOIN. - /// Expression/Filter -> ArrayJoin -> Something - auto & node = nodes.emplace_back(); - node.children.swap(child_node->children); - child_node->children.emplace_back(&node); - /// Expression/Filter -> ArrayJoin -> node -> Something - - node.step = std::make_unique(node.children.at(0)->step->getOutputStream(), - std::move(split_actions.first)); - node.step->setStepDescription(description); - array_join_step->updateInputStream(node.step->getOutputStream(), {}); - - if (expression_step) - parent = std::make_unique(array_join_step->getOutputStream(), split_actions.second); - else - parent = std::make_unique(array_join_step->getOutputStream(), split_actions.second, - filter_step->getFilterColumnName(), filter_step->removesFilterColumn()); - - parent->setStepDescription(description + " [split]"); -} - -/// Replace chain `ExpressionStep -> ExpressionStep` to single ExpressionStep -/// Replace chain `FilterStep -> ExpressionStep` to single FilterStep -static bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Node * child_node) -{ - auto & parent = parent_node->step; - auto & child = child_node->step; - - auto * parent_expr = typeid_cast(parent.get()); - auto * parent_filter = typeid_cast(parent.get()); - auto * child_expr = typeid_cast(child.get()); - - if (parent_expr && child_expr) - { - const auto & child_actions = child_expr->getExpression(); - const auto & parent_actions = parent_expr->getExpression(); - - /// We cannot combine actions with arrayJoin and stateful function because we not always can reorder them. - /// Example: select rowNumberInBlock() from (select arrayJoin([1, 2])) - /// Such a query will return two zeroes if we combine actions together. - if (child_actions->hasArrayJoin() && parent_actions->hasStatefulFunctions()) - return false; - - auto merged = ActionsDAG::merge(std::move(*child_actions), std::move(*parent_actions)); - - auto expr = std::make_unique(child_expr->getInputStreams().front(), merged); - expr->setStepDescription("(" + parent_expr->getStepDescription() + " + " + child_expr->getStepDescription() + ")"); - - parent_node->step = std::move(expr); - parent_node->children.swap(child_node->children); - return true; - } - else if (parent_filter && child_expr) - { - const auto & child_actions = child_expr->getExpression(); - const auto & parent_actions = parent_filter->getExpression(); - - if (child_actions->hasArrayJoin() && parent_actions->hasStatefulFunctions()) - return false; - - auto merged = ActionsDAG::merge(std::move(*child_actions), std::move(*parent_actions)); - - auto filter = std::make_unique(child_expr->getInputStreams().front(), merged, - parent_filter->getFilterColumnName(), parent_filter->removesFilterColumn()); - filter->setStepDescription("(" + parent_filter->getStepDescription() + " + " + child_expr->getStepDescription() + ")"); - - parent_node->step = std::move(filter); - parent_node->children.swap(child_node->children); - return true; - } - - return false; -} - -/// Split FilterStep into chain `ExpressionStep -> FilterStep`, where FilterStep contains minimal number of nodes. -static bool trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes) -{ - auto * filter_step = typeid_cast(node->step.get()); - if (!filter_step) - return false; - - const auto & expr = filter_step->getExpression(); - - /// Do not split if there are function like runningDifference. - if (expr->hasStatefulFunctions()) - return false; - - auto split = expr->splitActionsForFilter(filter_step->getFilterColumnName()); - - if (split.second->empty()) - return false; - - if (filter_step->removesFilterColumn()) - split.second->removeUnusedInput(filter_step->getFilterColumnName()); - - auto description = filter_step->getStepDescription(); - - auto & filter_node = nodes.emplace_back(); - node->children.swap(filter_node.children); - node->children.push_back(&filter_node); - - filter_node.step = std::make_unique( - filter_node.children.at(0)->step->getOutputStream(), - std::move(split.first), - filter_step->getFilterColumnName(), - filter_step->removesFilterColumn()); - - node->step = std::make_unique(filter_node.step->getOutputStream(), std::move(split.second)); - - filter_node.step->setStepDescription("(" + description + ")[split]"); - node->step->setStepDescription(description); - - return true; -} - void QueryPlan::optimize() { /* Stack contains info for every nodes in the path from tree root to the current node. @@ -623,14 +361,14 @@ void QueryPlan::optimize() { if (frame.node->children.size() == 1) { - tryPushDownLimit(frame.node->step, frame.node->children.front()); + QueryPlanOptimizations::tryPushDownLimit(frame.node->step, frame.node->children.front()); - while (tryMergeExpressions(frame.node, frame.node->children.front())); + while (QueryPlanOptimizations::tryMergeExpressions(frame.node, frame.node->children.front())); if (frame.node->children.size() == 1) - tryLiftUpArrayJoin(frame.node, frame.node->children.front(), nodes); + QueryPlanOptimizations::tryLiftUpArrayJoin(frame.node, frame.node->children.front(), nodes); - trySplitFilter(frame.node, nodes); + QueryPlanOptimizations::trySplitFilter(frame.node, nodes); } } @@ -643,11 +381,11 @@ void QueryPlan::optimize() { if (frame.node->children.size() == 1) { - while (tryMergeExpressions(frame.node, frame.node->children.front())); + while (QueryPlanOptimizations::tryMergeExpressions(frame.node, frame.node->children.front())); - trySplitFilter(frame.node, nodes); + QueryPlanOptimizations::trySplitFilter(frame.node, nodes); - tryLiftUpArrayJoin(frame.node, frame.node->children.front(), nodes); + QueryPlanOptimizations::tryLiftUpArrayJoin(frame.node, frame.node->children.front(), nodes); } stack.pop(); diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 2eb27be8899..f05a9f0bdba 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -116,6 +116,10 @@ SRCS( QueryPlan/MergingFinal.cpp QueryPlan/MergingSortedStep.cpp QueryPlan/OffsetStep.cpp + QueryPlan/Optimizations/liftUpArrayJoin.cpp + QueryPlan/Optimizations/limitPushDown.cpp + QueryPlan/Optimizations/mergeExpressions.cpp + QueryPlan/Optimizations/splitFilter.cpp QueryPlan/PartialSortingStep.cpp QueryPlan/QueryPlan.cpp QueryPlan/ReadFromPreparedSource.cpp From cf05c17cfa341e1162be515bebf59e10e37d9455 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 27 Jan 2021 20:36:53 +0300 Subject: [PATCH 2/8] Rewrite QueryPlan tree optimizations traverse. --- .../QueryPlan/Optimizations/Optimizations.h | 43 +++++++- .../Optimizations/liftUpArrayJoin.cpp | 13 ++- .../QueryPlan/Optimizations/limitPushDown.cpp | 24 +++-- .../Optimizations/mergeExpressions.cpp | 7 +- .../QueryPlan/Optimizations/optimizeTree.cpp | 102 ++++++++++++++++++ src/Processors/QueryPlan/QueryPlan.cpp | 57 +--------- 6 files changed, 172 insertions(+), 74 deletions(-) create mode 100644 src/Processors/QueryPlan/Optimizations/optimizeTree.cpp diff --git a/src/Processors/QueryPlan/Optimizations/Optimizations.h b/src/Processors/QueryPlan/Optimizations/Optimizations.h index 93b4be90a98..37b32d6a095 100644 --- a/src/Processors/QueryPlan/Optimizations/Optimizations.h +++ b/src/Processors/QueryPlan/Optimizations/Optimizations.h @@ -1,4 +1,5 @@ #include +#include namespace DB { @@ -6,18 +7,52 @@ namespace DB namespace QueryPlanOptimizations { +/// This is the main function which optimizes the whole QueryPlan tree. +void optimizeTree(QueryPlan::Node & root, QueryPlan::Nodes & nodes); + +/// Optimization is a function applied to QueryPlan::Node. +/// It can read and update subtree of specified node. +/// It return true if some change of thee happened. +/// New nodes should be added to QueryPlan::Nodes list. +/// It is not needed to remove old nodes from the list. +/// +/// Optimization must guarantee that: +/// * the structure of tree is correct +/// * no more then `read_depth` layers of subtree was read +/// * no more then `update_depth` layers of subtree was updated +struct Optimization +{ + using Function = bool (*)(QueryPlan::Node *, QueryPlan::Nodes &); + const Function run = nullptr; + const size_t read_depth; + const size_t update_depth; +}; + +/// Move ARRAY JOIN up if possible. +bool tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes); + /// Move LimitStep down if possible. -void tryPushDownLimit(QueryPlanStepPtr & parent, QueryPlan::Node * child_node); +bool tryPushDownLimit(QueryPlan::Node * parent_node, QueryPlan::Nodes &); /// Split FilterStep into chain `ExpressionStep -> FilterStep`, where FilterStep contains minimal number of nodes. bool trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes); /// Replace chain `ExpressionStep -> ExpressionStep` to single ExpressionStep /// Replace chain `FilterStep -> ExpressionStep` to single FilterStep -bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Node * child_node); +bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &); -/// Move ARRAY JOIN up if possible. -void tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Node * child_node, QueryPlan::Nodes & nodes); +inline const auto & getOptimizations() +{ + static const std::array optimizations = + {{ + {tryLiftUpArrayJoin, 2, 2}, + {tryPushDownLimit, 2, 2}, + {trySplitFilter, 1, 2}, + {tryMergeExpressions, 2, 1}, + }}; + + return optimizations; +} } diff --git a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp index 474124d970d..026ceccebfb 100644 --- a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp @@ -8,8 +8,13 @@ namespace DB::QueryPlanOptimizations { -void tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Node * child_node, QueryPlan::Nodes & nodes) +bool tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes) { + if (parent_node->children.size() != 1) + return false; + + QueryPlan::Node * child_node = parent_node->children.front(); + auto & parent = parent_node->step; auto & child = child_node->step; auto * expression_step = typeid_cast(parent.get()); @@ -17,7 +22,7 @@ void tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Node * child_n auto * array_join_step = typeid_cast(child.get()); if (!(expression_step || filter_step) || !array_join_step) - return; + return false; const auto & array_join = array_join_step->arrayJoin(); const auto & expression = expression_step ? expression_step->getExpression() @@ -27,7 +32,7 @@ void tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Node * child_n /// No actions can be moved before ARRAY JOIN. if (split_actions.first->empty()) - return; + return false; auto description = parent->getStepDescription(); @@ -52,7 +57,7 @@ void tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Node * child_n child->setStepDescription(std::move(description)); array_join_step->updateInputStream(child->getOutputStream(), expected_header); - return; + return false; } /// Add new expression step before ARRAY JOIN. diff --git a/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp b/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp index 8d2f5f50fc8..fd6c2e19f6b 100644 --- a/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp @@ -54,31 +54,37 @@ static bool tryUpdateLimitForSortingSteps(QueryPlan::Node * node, size_t limit) return updated; } -void tryPushDownLimit(QueryPlanStepPtr & parent, QueryPlan::Node * child_node) +bool tryPushDownLimit(QueryPlan::Node * parent_node, QueryPlan::Nodes &) { + if (parent_node->children.size() != 1) + return false; + + QueryPlan::Node * child_node = parent_node->children.front(); + + auto & parent = parent_node->step; auto & child = child_node->step; auto * limit = typeid_cast(parent.get()); if (!limit) - return; + return false; /// Skip LIMIT WITH TIES by now. if (limit->withTies()) - return; + return false; const auto * transforming = dynamic_cast(child.get()); /// Skip everything which is not transform. if (!transforming) - return; + return false; /// Special cases for sorting steps. if (tryUpdateLimitForSortingSteps(child_node, limit->getLimitForSorting())) - return; + return false; /// Special case for TotalsHaving. Totals may be incorrect if we push down limit. if (typeid_cast(child.get())) - return; + return false; /// Now we should decide if pushing down limit possible for this step. @@ -87,16 +93,16 @@ void tryPushDownLimit(QueryPlanStepPtr & parent, QueryPlan::Node * child_node) /// Cannot push down if child changes the number of rows. if (!transform_traits.preserves_number_of_rows) - return; + return false; /// Cannot push down if data was sorted exactly by child stream. if (!child->getOutputStream().sort_description.empty() && !data_stream_traits.preserves_sorting) - return; + return false; /// Now we push down limit only if it doesn't change any stream properties. /// TODO: some of them may be changed and, probably, not important for following streams. We may add such info. if (!limit->getOutputStream().hasEqualPropertiesWith(transforming->getOutputStream())) - return; + return false; /// Input stream for Limit have changed. limit->updateInputStream(transforming->getInputStreams().front()); diff --git a/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp b/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp index 3bbfe0e1efb..4b4bf540cc5 100644 --- a/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp +++ b/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp @@ -6,8 +6,13 @@ namespace DB::QueryPlanOptimizations { -bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Node * child_node) +bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &) { + if (parent_node->children.size() != 1) + return false; + + QueryPlan::Node * child_node = parent_node->children.front(); + auto & parent = parent_node->step; auto & child = child_node->step; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp new file mode 100644 index 00000000000..8ac5a4482f1 --- /dev/null +++ b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp @@ -0,0 +1,102 @@ +#include +#include + +namespace DB::QueryPlanOptimizations +{ + +void optimizeTree(QueryPlan::Node & root, QueryPlan::Nodes & nodes) +{ + const auto & optimizations = getOptimizations(); + + struct Frame + { + QueryPlan::Node * node; + Frame * parent = nullptr; + + /// Will update only depth_limit layers of tree (if no other optimizations happen). + size_t depth_limit = 0; + + size_t next_child = 0; + + size_t read_depth_limit = 0; + }; + + std::stack stack; + stack.push(Frame{.node = &root}); + + while (!stack.empty()) + { + auto & frame = stack.top(); + + if (frame.depth_limit != 1) + { + /// Traverse all children first. + if (frame.next_child < frame.node->children.size()) + { + stack.push(Frame + { + .node = frame.node->children[frame.next_child], + .parent = &frame, + .depth_limit = frame.depth_limit ? (frame.depth_limit - 1) : 0, + }); + + ++frame.next_child; + continue; + } + } + + if (frame.depth_limit == 0 || frame.read_depth_limit) + { + size_t max_update_depth = 0; + + /// Apply all optimizations. + for (const auto & optimization : optimizations) + { + /// Just in case, skip optimization if it is not initialized. + if (!optimization.run) + continue; + + /// Skip optimization if read_depth_limit is applied. + if (frame.read_depth_limit && optimization.read_depth <= frame.read_depth_limit) + continue; + + /// Try to apply optimization. + if (optimization.run(frame.node, nodes)) + max_update_depth = std::max(max_update_depth, optimization.update_depth); + } + + /// Nothing was applied. + if (max_update_depth == 0) + { + stack.pop(); + continue; + } + + /// Traverse `max_update_depth` layers of tree again. + frame.depth_limit = max_update_depth; + frame.next_child = 0; + + /// Also go to parents and tell them to apply some optimizations again. + Frame * cur_frame = &frame; + for (size_t cur_depth = 0; cur_frame && cur_frame->depth_limit; ++cur_depth) + { + /// If cur_frame is traversed first time, all optimizations will apply anyway. + if (cur_frame->depth_limit == 0) + break; + + /// Stop if limit is applied and stricter then current. + if (cur_frame->read_depth_limit && cur_frame->read_depth_limit <= cur_depth) + break; + + cur_frame->read_depth_limit = cur_depth; + cur_frame = cur_frame->parent; + } + + continue; + } + + stack.pop(); + } +} + +} diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index 49249e99864..755944fdf9f 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -7,9 +7,6 @@ #include #include #include -#include "ExpressionStep.h" -#include "ArrayJoinStep.h" -#include "FilterStep.h" namespace DB { @@ -338,59 +335,7 @@ void QueryPlan::explainPipeline(WriteBuffer & buffer, const ExplainPipelineOptio void QueryPlan::optimize() { - /* Stack contains info for every nodes in the path from tree root to the current node. - * Every optimization changes only current node and it's children. - * Optimization may change QueryPlanStep, but not QueryPlan::Node (only add a new one). - * So, QueryPlan::Node::children will be always valid. - */ - - struct Frame - { - Node * node; - size_t next_child = 0; - }; - - std::stack stack; - stack.push(Frame{.node = root}); - - while (!stack.empty()) - { - auto & frame = stack.top(); - - if (frame.next_child == 0) - { - if (frame.node->children.size() == 1) - { - QueryPlanOptimizations::tryPushDownLimit(frame.node->step, frame.node->children.front()); - - while (QueryPlanOptimizations::tryMergeExpressions(frame.node, frame.node->children.front())); - - if (frame.node->children.size() == 1) - QueryPlanOptimizations::tryLiftUpArrayJoin(frame.node, frame.node->children.front(), nodes); - - QueryPlanOptimizations::trySplitFilter(frame.node, nodes); - } - } - - if (frame.next_child < frame.node->children.size()) - { - stack.push(Frame{frame.node->children[frame.next_child]}); - ++frame.next_child; - } - else - { - if (frame.node->children.size() == 1) - { - while (QueryPlanOptimizations::tryMergeExpressions(frame.node, frame.node->children.front())); - - QueryPlanOptimizations::trySplitFilter(frame.node, nodes); - - QueryPlanOptimizations::tryLiftUpArrayJoin(frame.node, frame.node->children.front(), nodes); - } - - stack.pop(); - } - } + QueryPlanOptimizations::optimizeTree(*root, nodes); } } From b244499d6916923761b07fbe46edc700511985ee Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 27 Jan 2021 22:24:59 +0300 Subject: [PATCH 3/8] Fix build. --- src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp | 1 + src/Processors/QueryPlan/Optimizations/limitPushDown.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp index 026ceccebfb..a45bf76202d 100644 --- a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp @@ -79,6 +79,7 @@ bool tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes) filter_step->getFilterColumnName(), filter_step->removesFilterColumn()); parent->setStepDescription(description + " [split]"); + return true; } } diff --git a/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp b/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp index fd6c2e19f6b..0b2fcdfb209 100644 --- a/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp @@ -108,6 +108,7 @@ bool tryPushDownLimit(QueryPlan::Node * parent_node, QueryPlan::Nodes &) limit->updateInputStream(transforming->getInputStreams().front()); parent.swap(child); + return true; } } From 3b7373e9928ba808632b6f9aa1b894a49e07bb84 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 27 Jan 2021 22:53:07 +0300 Subject: [PATCH 4/8] More comments. --- .../QueryPlan/Optimizations/optimizeTree.cpp | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp index 8ac5a4482f1..962452b24af 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp @@ -13,11 +13,15 @@ void optimizeTree(QueryPlan::Node & root, QueryPlan::Nodes & nodes) QueryPlan::Node * node; Frame * parent = nullptr; - /// Will update only depth_limit layers of tree (if no other optimizations happen). - size_t depth_limit = 0; + /// If not zero, traverse only traverse_depth_limit layers of tree (if no other optimizations happen). + /// Otherwise, traverse all children. + size_t traverse_depth_limit = 0; + /// Next child to process. size_t next_child = 0; + /// If not zero, optimizations to current node again. + /// Skip optimizations which read less then `read_depth_limit` layers of tree. size_t read_depth_limit = 0; }; @@ -28,7 +32,9 @@ void optimizeTree(QueryPlan::Node & root, QueryPlan::Nodes & nodes) { auto & frame = stack.top(); - if (frame.depth_limit != 1) + /// If traverse_depth_limit == 0, then traverse without limit (first entrance) + /// If traverse_depth_limit > 1, then traverse with (limit - 1) + if (frame.traverse_depth_limit != 1) { /// Traverse all children first. if (frame.next_child < frame.node->children.size()) @@ -37,7 +43,7 @@ void optimizeTree(QueryPlan::Node & root, QueryPlan::Nodes & nodes) { .node = frame.node->children[frame.next_child], .parent = &frame, - .depth_limit = frame.depth_limit ? (frame.depth_limit - 1) : 0, + .traverse_depth_limit = frame.traverse_depth_limit ? (frame.traverse_depth_limit - 1) : 0, }); ++frame.next_child; @@ -45,7 +51,9 @@ void optimizeTree(QueryPlan::Node & root, QueryPlan::Nodes & nodes) } } - if (frame.depth_limit == 0 || frame.read_depth_limit) + /// If frame.traverse_depth_limit == 0, apply optimizations on first entrance. + /// If frame.read_depth_limit, then one of children was updated, and we may need to repeat some optimizations. + if (frame.traverse_depth_limit == 0 || frame.read_depth_limit) { size_t max_update_depth = 0; @@ -73,15 +81,18 @@ void optimizeTree(QueryPlan::Node & root, QueryPlan::Nodes & nodes) } /// Traverse `max_update_depth` layers of tree again. - frame.depth_limit = max_update_depth; + frame.traverse_depth_limit = max_update_depth; frame.next_child = 0; /// Also go to parents and tell them to apply some optimizations again. + /// Check: for our parent we set read_depth_limit = 1, which means it can skip optimizations + /// which use ony 1 layer of tree (not read current node). + /// Note that frame.read_depth_limit will be zeroed. Frame * cur_frame = &frame; - for (size_t cur_depth = 0; cur_frame && cur_frame->depth_limit; ++cur_depth) + for (size_t cur_depth = 0; cur_frame && cur_frame->traverse_depth_limit; ++cur_depth) { /// If cur_frame is traversed first time, all optimizations will apply anyway. - if (cur_frame->depth_limit == 0) + if (cur_frame->traverse_depth_limit == 0) break; /// Stop if limit is applied and stricter then current. From e1765e7f882dca74c226448c5fa6fd0194f4054c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 28 Jan 2021 14:00:24 +0300 Subject: [PATCH 5/8] Add method trivial to ActionsDAG --- src/Interpreters/ActionsDAG.cpp | 4 ++-- src/Interpreters/ActionsDAG.h | 2 +- src/Processors/QueryPlan/Optimizations/Optimizations.h | 2 +- src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp | 4 ++-- src/Processors/QueryPlan/Optimizations/splitFilter.cpp | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 4c3a4cbe0fa..498141821ed 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -609,10 +609,10 @@ bool ActionsDAG::hasStatefulFunctions() const return false; } -bool ActionsDAG::empty() const +bool ActionsDAG::trivial() const { for (const auto & node : nodes) - if (node.type != ActionType::INPUT) + if (node.type == ActionType::FUNCTION || node.type == ActionType::ARRAY_JOIN) return false; return true; diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index c82496b2a8a..b12da30e24f 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -223,7 +223,7 @@ public: bool hasArrayJoin() const; bool hasStatefulFunctions() const; - bool empty() const; /// If actions only contain inputs. + bool trivial() const; /// If actions has no functions or array join. const ActionsSettings & getSettings() const { return settings; } diff --git a/src/Processors/QueryPlan/Optimizations/Optimizations.h b/src/Processors/QueryPlan/Optimizations/Optimizations.h index 37b32d6a095..d6e9d345a23 100644 --- a/src/Processors/QueryPlan/Optimizations/Optimizations.h +++ b/src/Processors/QueryPlan/Optimizations/Optimizations.h @@ -45,7 +45,7 @@ inline const auto & getOptimizations() { static const std::array optimizations = {{ - {tryLiftUpArrayJoin, 2, 2}, + {tryLiftUpArrayJoin, 2, 3}, {tryPushDownLimit, 2, 2}, {trySplitFilter, 1, 2}, {tryMergeExpressions, 2, 1}, diff --git a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp index a45bf76202d..0dc022b14af 100644 --- a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp @@ -31,13 +31,13 @@ bool tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes) auto split_actions = expression->splitActionsBeforeArrayJoin(array_join->columns); /// No actions can be moved before ARRAY JOIN. - if (split_actions.first->empty()) + if (split_actions.first->trivial()) return false; auto description = parent->getStepDescription(); /// All actions was moved before ARRAY JOIN. Swap Expression and ArrayJoin. - if (split_actions.second->empty()) + if (split_actions.second->trivial()) { auto expected_header = parent->getOutputStream().header; diff --git a/src/Processors/QueryPlan/Optimizations/splitFilter.cpp b/src/Processors/QueryPlan/Optimizations/splitFilter.cpp index fd82bd69a9e..09ce500ee54 100644 --- a/src/Processors/QueryPlan/Optimizations/splitFilter.cpp +++ b/src/Processors/QueryPlan/Optimizations/splitFilter.cpp @@ -21,7 +21,7 @@ bool trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes) auto split = expr->splitActionsForFilter(filter_step->getFilterColumnName()); - if (split.second->empty()) + if (split.second->trivial()) return false; if (filter_step->removesFilterColumn()) From 4ef56a41cb71ab0ae07dbdec50711cd8841023db Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 28 Jan 2021 14:15:45 +0300 Subject: [PATCH 6/8] Simplify optimizeTree --- .../QueryPlan/Optimizations/optimizeTree.cpp | 78 +++++-------------- 1 file changed, 20 insertions(+), 58 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp index 962452b24af..da031378e0c 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp @@ -11,18 +11,13 @@ void optimizeTree(QueryPlan::Node & root, QueryPlan::Nodes & nodes) struct Frame { QueryPlan::Node * node; - Frame * parent = nullptr; - /// If not zero, traverse only traverse_depth_limit layers of tree (if no other optimizations happen). + /// If not zero, traverse only depth_limit layers of tree (if no other optimizations happen). /// Otherwise, traverse all children. - size_t traverse_depth_limit = 0; + size_t depth_limit = 0; /// Next child to process. size_t next_child = 0; - - /// If not zero, optimizations to current node again. - /// Skip optimizations which read less then `read_depth_limit` layers of tree. - size_t read_depth_limit = 0; }; std::stack stack; @@ -34,7 +29,7 @@ void optimizeTree(QueryPlan::Node & root, QueryPlan::Nodes & nodes) /// If traverse_depth_limit == 0, then traverse without limit (first entrance) /// If traverse_depth_limit > 1, then traverse with (limit - 1) - if (frame.traverse_depth_limit != 1) + if (frame.depth_limit != 1) { /// Traverse all children first. if (frame.next_child < frame.node->children.size()) @@ -42,8 +37,7 @@ void optimizeTree(QueryPlan::Node & root, QueryPlan::Nodes & nodes) stack.push(Frame { .node = frame.node->children[frame.next_child], - .parent = &frame, - .traverse_depth_limit = frame.traverse_depth_limit ? (frame.traverse_depth_limit - 1) : 0, + .depth_limit = frame.depth_limit ? (frame.depth_limit - 1) : 0, }); ++frame.next_child; @@ -51,61 +45,29 @@ void optimizeTree(QueryPlan::Node & root, QueryPlan::Nodes & nodes) } } - /// If frame.traverse_depth_limit == 0, apply optimizations on first entrance. - /// If frame.read_depth_limit, then one of children was updated, and we may need to repeat some optimizations. - if (frame.traverse_depth_limit == 0 || frame.read_depth_limit) + size_t max_update_depth = 0; + + /// Apply all optimizations. + for (const auto & optimization : optimizations) { - size_t max_update_depth = 0; - - /// Apply all optimizations. - for (const auto & optimization : optimizations) - { - /// Just in case, skip optimization if it is not initialized. - if (!optimization.run) - continue; - - /// Skip optimization if read_depth_limit is applied. - if (frame.read_depth_limit && optimization.read_depth <= frame.read_depth_limit) - continue; - - /// Try to apply optimization. - if (optimization.run(frame.node, nodes)) - max_update_depth = std::max(max_update_depth, optimization.update_depth); - } - - /// Nothing was applied. - if (max_update_depth == 0) - { - stack.pop(); + /// Just in case, skip optimization if it is not initialized. + if (!optimization.run) continue; - } - /// Traverse `max_update_depth` layers of tree again. - frame.traverse_depth_limit = max_update_depth; + /// Try to apply optimization. + if (optimization.run(frame.node, nodes)) + max_update_depth = std::max(max_update_depth, optimization.update_depth); + } + + /// Traverse `max_update_depth` layers of tree again. + if (max_update_depth) + { + frame.depth_limit = max_update_depth; frame.next_child = 0; - - /// Also go to parents and tell them to apply some optimizations again. - /// Check: for our parent we set read_depth_limit = 1, which means it can skip optimizations - /// which use ony 1 layer of tree (not read current node). - /// Note that frame.read_depth_limit will be zeroed. - Frame * cur_frame = &frame; - for (size_t cur_depth = 0; cur_frame && cur_frame->traverse_depth_limit; ++cur_depth) - { - /// If cur_frame is traversed first time, all optimizations will apply anyway. - if (cur_frame->traverse_depth_limit == 0) - break; - - /// Stop if limit is applied and stricter then current. - if (cur_frame->read_depth_limit && cur_frame->read_depth_limit <= cur_depth) - break; - - cur_frame->read_depth_limit = cur_depth; - cur_frame = cur_frame->parent; - } - continue; } + /// Nothing was applied. stack.pop(); } } From 3b50a7475904fbfcc7540d26373145b6533a048d Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 28 Jan 2021 15:32:02 +0300 Subject: [PATCH 7/8] Simplify optimization interface. --- .../QueryPlan/Optimizations/Optimizations.h | 33 +++++++++---------- .../Optimizations/liftUpArrayJoin.cpp | 12 +++---- .../QueryPlan/Optimizations/limitPushDown.cpp | 22 ++++++------- .../Optimizations/mergeExpressions.cpp | 12 +++---- .../QueryPlan/Optimizations/optimizeTree.cpp | 6 ++-- .../QueryPlan/Optimizations/splitFilter.cpp | 10 +++--- 6 files changed, 46 insertions(+), 49 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/Optimizations.h b/src/Processors/QueryPlan/Optimizations/Optimizations.h index d6e9d345a23..454eab9649a 100644 --- a/src/Processors/QueryPlan/Optimizations/Optimizations.h +++ b/src/Processors/QueryPlan/Optimizations/Optimizations.h @@ -1,3 +1,4 @@ +#pragma once #include #include @@ -12,43 +13,39 @@ void optimizeTree(QueryPlan::Node & root, QueryPlan::Nodes & nodes); /// Optimization is a function applied to QueryPlan::Node. /// It can read and update subtree of specified node. -/// It return true if some change of thee happened. +/// It return the number of updated layers of subtree if some change happened. +/// It must guarantee that the structure of tree is correct. +/// /// New nodes should be added to QueryPlan::Nodes list. /// It is not needed to remove old nodes from the list. -/// -/// Optimization must guarantee that: -/// * the structure of tree is correct -/// * no more then `read_depth` layers of subtree was read -/// * no more then `update_depth` layers of subtree was updated struct Optimization { - using Function = bool (*)(QueryPlan::Node *, QueryPlan::Nodes &); - const Function run = nullptr; - const size_t read_depth; - const size_t update_depth; + using Function = size_t (*)(QueryPlan::Node *, QueryPlan::Nodes &); + const Function apply = nullptr; + const char * name; }; /// Move ARRAY JOIN up if possible. -bool tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes); +size_t tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes); /// Move LimitStep down if possible. -bool tryPushDownLimit(QueryPlan::Node * parent_node, QueryPlan::Nodes &); +size_t tryPushDownLimit(QueryPlan::Node * parent_node, QueryPlan::Nodes &); /// Split FilterStep into chain `ExpressionStep -> FilterStep`, where FilterStep contains minimal number of nodes. -bool trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes); +size_t trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes); /// Replace chain `ExpressionStep -> ExpressionStep` to single ExpressionStep /// Replace chain `FilterStep -> ExpressionStep` to single FilterStep -bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &); +size_t tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &); inline const auto & getOptimizations() { static const std::array optimizations = {{ - {tryLiftUpArrayJoin, 2, 3}, - {tryPushDownLimit, 2, 2}, - {trySplitFilter, 1, 2}, - {tryMergeExpressions, 2, 1}, + {tryLiftUpArrayJoin, "liftUpArrayJoin"}, + {tryPushDownLimit, "pushDownLimit"}, + {trySplitFilter, "splitFilter"}, + {tryMergeExpressions, "mergeExpressions"}, }}; return optimizations; diff --git a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp index 0dc022b14af..e20c5f93d6e 100644 --- a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp @@ -8,10 +8,10 @@ namespace DB::QueryPlanOptimizations { -bool tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes) +size_t tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes) { if (parent_node->children.size() != 1) - return false; + return 0; QueryPlan::Node * child_node = parent_node->children.front(); @@ -22,7 +22,7 @@ bool tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes) auto * array_join_step = typeid_cast(child.get()); if (!(expression_step || filter_step) || !array_join_step) - return false; + return 0; const auto & array_join = array_join_step->arrayJoin(); const auto & expression = expression_step ? expression_step->getExpression() @@ -32,7 +32,7 @@ bool tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes) /// No actions can be moved before ARRAY JOIN. if (split_actions.first->trivial()) - return false; + return 0; auto description = parent->getStepDescription(); @@ -57,7 +57,7 @@ bool tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes) child->setStepDescription(std::move(description)); array_join_step->updateInputStream(child->getOutputStream(), expected_header); - return false; + return 2; } /// Add new expression step before ARRAY JOIN. @@ -79,7 +79,7 @@ bool tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes) filter_step->getFilterColumnName(), filter_step->removesFilterColumn()); parent->setStepDescription(description + " [split]"); - return true; + return 3; } } diff --git a/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp b/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp index 0b2fcdfb209..01af6a2bbde 100644 --- a/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp @@ -54,10 +54,10 @@ static bool tryUpdateLimitForSortingSteps(QueryPlan::Node * node, size_t limit) return updated; } -bool tryPushDownLimit(QueryPlan::Node * parent_node, QueryPlan::Nodes &) +size_t tryPushDownLimit(QueryPlan::Node * parent_node, QueryPlan::Nodes &) { if (parent_node->children.size() != 1) - return false; + return 0; QueryPlan::Node * child_node = parent_node->children.front(); @@ -66,25 +66,25 @@ bool tryPushDownLimit(QueryPlan::Node * parent_node, QueryPlan::Nodes &) auto * limit = typeid_cast(parent.get()); if (!limit) - return false; + return 0; /// Skip LIMIT WITH TIES by now. if (limit->withTies()) - return false; + return 0; const auto * transforming = dynamic_cast(child.get()); /// Skip everything which is not transform. if (!transforming) - return false; + return 0; /// Special cases for sorting steps. if (tryUpdateLimitForSortingSteps(child_node, limit->getLimitForSorting())) - return false; + return 0; /// Special case for TotalsHaving. Totals may be incorrect if we push down limit. if (typeid_cast(child.get())) - return false; + return 0; /// Now we should decide if pushing down limit possible for this step. @@ -93,22 +93,22 @@ bool tryPushDownLimit(QueryPlan::Node * parent_node, QueryPlan::Nodes &) /// Cannot push down if child changes the number of rows. if (!transform_traits.preserves_number_of_rows) - return false; + return 0; /// Cannot push down if data was sorted exactly by child stream. if (!child->getOutputStream().sort_description.empty() && !data_stream_traits.preserves_sorting) - return false; + return 0; /// Now we push down limit only if it doesn't change any stream properties. /// TODO: some of them may be changed and, probably, not important for following streams. We may add such info. if (!limit->getOutputStream().hasEqualPropertiesWith(transforming->getOutputStream())) - return false; + return 0; /// Input stream for Limit have changed. limit->updateInputStream(transforming->getInputStreams().front()); parent.swap(child); - return true; + return 2; } } diff --git a/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp b/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp index 4b4bf540cc5..dfd15a2a929 100644 --- a/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp +++ b/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp @@ -6,7 +6,7 @@ namespace DB::QueryPlanOptimizations { -bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &) +size_t tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &) { if (parent_node->children.size() != 1) return false; @@ -29,7 +29,7 @@ bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &) /// Example: select rowNumberInBlock() from (select arrayJoin([1, 2])) /// Such a query will return two zeroes if we combine actions together. if (child_actions->hasArrayJoin() && parent_actions->hasStatefulFunctions()) - return false; + return 0; auto merged = ActionsDAG::merge(std::move(*child_actions), std::move(*parent_actions)); @@ -38,7 +38,7 @@ bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &) parent_node->step = std::move(expr); parent_node->children.swap(child_node->children); - return true; + return 1; } else if (parent_filter && child_expr) { @@ -46,7 +46,7 @@ bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &) const auto & parent_actions = parent_filter->getExpression(); if (child_actions->hasArrayJoin() && parent_actions->hasStatefulFunctions()) - return false; + return 0; auto merged = ActionsDAG::merge(std::move(*child_actions), std::move(*parent_actions)); @@ -56,10 +56,10 @@ bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &) parent_node->step = std::move(filter); parent_node->children.swap(child_node->children); - return true; + return 1; } - return false; + return 0; } } diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp index da031378e0c..e5ccc173ed8 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp @@ -51,12 +51,12 @@ void optimizeTree(QueryPlan::Node & root, QueryPlan::Nodes & nodes) for (const auto & optimization : optimizations) { /// Just in case, skip optimization if it is not initialized. - if (!optimization.run) + if (!optimization.apply) continue; /// Try to apply optimization. - if (optimization.run(frame.node, nodes)) - max_update_depth = std::max(max_update_depth, optimization.update_depth); + auto update_depth = optimization.apply(frame.node, nodes); + max_update_depth = std::max(max_update_depth, update_depth); } /// Traverse `max_update_depth` layers of tree again. diff --git a/src/Processors/QueryPlan/Optimizations/splitFilter.cpp b/src/Processors/QueryPlan/Optimizations/splitFilter.cpp index 09ce500ee54..38ba8f25b24 100644 --- a/src/Processors/QueryPlan/Optimizations/splitFilter.cpp +++ b/src/Processors/QueryPlan/Optimizations/splitFilter.cpp @@ -7,22 +7,22 @@ namespace DB::QueryPlanOptimizations { /// Split FilterStep into chain `ExpressionStep -> FilterStep`, where FilterStep contains minimal number of nodes. -bool trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes) +size_t trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes) { auto * filter_step = typeid_cast(node->step.get()); if (!filter_step) - return false; + return 0; const auto & expr = filter_step->getExpression(); /// Do not split if there are function like runningDifference. if (expr->hasStatefulFunctions()) - return false; + return 0; auto split = expr->splitActionsForFilter(filter_step->getFilterColumnName()); if (split.second->trivial()) - return false; + return 0; if (filter_step->removesFilterColumn()) split.second->removeUnusedInput(filter_step->getFilterColumnName()); @@ -44,7 +44,7 @@ bool trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes) filter_node.step->setStepDescription("(" + description + ")[split]"); node->step->setStepDescription(description); - return true; + return 2; } } From 9f9b7fc3e26da5118a74aaddc7dcae038b7e9d46 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 29 Jan 2021 13:05:21 +0300 Subject: [PATCH 8/8] Update ya.make --- src/Processors/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Processors/ya.make b/src/Processors/ya.make index f05a9f0bdba..1c6f0cf1f7c 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -119,6 +119,7 @@ SRCS( QueryPlan/Optimizations/liftUpArrayJoin.cpp QueryPlan/Optimizations/limitPushDown.cpp QueryPlan/Optimizations/mergeExpressions.cpp + QueryPlan/Optimizations/optimizeTree.cpp QueryPlan/Optimizations/splitFilter.cpp QueryPlan/PartialSortingStep.cpp QueryPlan/QueryPlan.cpp