From b23773380cfd87016954ed8da92b70294b134285 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 22 Aug 2023 15:06:50 +0000 Subject: [PATCH] Fix filter pushdown for full_sorting_merge join --- src/Interpreters/ActionsDAG.cpp | 4 ++-- .../CreateSetAndFilterOnTheFlyStep.cpp | 4 ++++ .../QueryPlan/CreateSetAndFilterOnTheFlyStep.h | 2 ++ .../QueryPlan/Optimizations/filterPushDown.cpp | 11 +++++++++-- .../02861_filter_pushdown_const_bug.reference | 8 ++++++++ .../02861_filter_pushdown_const_bug.sql | 18 ++++++++++++++++++ 6 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/02861_filter_pushdown_const_bug.reference create mode 100644 tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 232721666e7..52bb9aebae6 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -2191,8 +2191,8 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown( /// Replace predicate result to constant 1. Node node; node.type = ActionType::COLUMN; - node.result_name = std::move(predicate->result_name); - node.result_type = std::move(predicate->result_type); + node.result_name = predicate->result_name; + node.result_type = predicate->result_type; node.column = node.result_type->createColumnConst(0, 1); if (predicate->type != ActionType::INPUT) diff --git a/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.cpp b/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.cpp index c5b5dad5aa5..ca46f92eeb4 100644 --- a/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.cpp +++ b/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.cpp @@ -201,5 +201,9 @@ void CreateSetAndFilterOnTheFlyStep::updateOutputStream() output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); } +bool CreateSetAndFilterOnTheFlyStep::isColumnPartOfSetKey(const String & column_name) const +{ + return std::find(column_names.begin(), column_names.end(), column_name) != column_names.end(); +} } diff --git a/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.h b/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.h index b363991c2f6..023901dba02 100644 --- a/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.h +++ b/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.h @@ -35,6 +35,8 @@ public: SetWithStatePtr getSet() const { return own_set; } + bool isColumnPartOfSetKey(const String & column_name) const; + /// Set for another stream. void setFiltering(SetWithStatePtr filtering_set_) { filtering_set = filtering_set_; } diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index af47b6ff4cd..3b31a809f9d 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -428,8 +428,15 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes return updated_steps; } - if (auto updated_steps = simplePushDownOverStep(parent_node, nodes, child)) - return updated_steps; + if (const auto * join_filter_set_step = typeid_cast(child.get())) + { + const auto & filter_column_name = assert_cast(parent_node->step.get())->getFilterColumnName(); + bool can_remove_filter = !join_filter_set_step->isColumnPartOfSetKey(filter_column_name); + + Names allowed_inputs = child->getOutputStream().header.getNames(); + if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, allowed_inputs, can_remove_filter)) + return updated_steps; + } if (auto * union_step = typeid_cast(child.get())) { diff --git a/tests/queries/0_stateless/02861_filter_pushdown_const_bug.reference b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.reference new file mode 100644 index 00000000000..428ba88bff0 --- /dev/null +++ b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.reference @@ -0,0 +1,8 @@ +1 +1 +1 +1 1 +1 +1 +1 +1 1 diff --git a/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql new file mode 100644 index 00000000000..a5ddf830d48 --- /dev/null +++ b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS t1; + +CREATE TABLE t1 (key UInt8) ENGINE = Memory; +INSERT INTO t1 VALUES (1),(2); + +SET join_algorithm = 'full_sorting_merge'; + +SELECT key FROM ( SELECT key FROM t1 ) AS t1 JOIN ( SELECT key FROM t1 ) AS t2 ON t1.key = t2.key WHERE key; +SELECT key FROM ( SELECT 1 AS key ) AS t1 JOIN ( SELECT 1 AS key ) AS t2 ON t1.key = t2.key WHERE key; +SELECT * FROM ( SELECT 1 AS key GROUP BY NULL ) AS t1 INNER JOIN (SELECT 1 AS key) AS t2 ON t1.key = t2.key WHERE t1.key ORDER BY key; + +SET max_rows_in_set_to_optimize_join = 0; + +SELECT key FROM ( SELECT key FROM t1 ) AS t1 JOIN ( SELECT key FROM t1 ) AS t2 ON t1.key = t2.key WHERE key; +SELECT key FROM ( SELECT 1 AS key ) AS t1 JOIN ( SELECT 1 AS key ) AS t2 ON t1.key = t2.key WHERE key; +SELECT * FROM ( SELECT 1 AS key GROUP BY NULL ) AS t1 INNER JOIN (SELECT 1 AS key) AS t2 ON t1.key = t2.key WHERE t1.key ORDER BY key; + +DROP TABLE IF EXISTS t1;