Merge pull request #69560 from Avogar/fix-prewhere-reorder

Keep original order of conditions during move to prewhere

src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp

@@ -361,11 +361,23 @@ std::optional<MergeTreeWhereOptimizer::OptimizeResult> MergeTreeWhereOptimizer::
     UInt64 total_size_of_moved_conditions = 0;
     UInt64 total_number_of_moved_columns = 0;
 
+    /// Remember positions of conditions in where_conditions list
+    /// to keep original order of conditions in prewhere_conditions while moving.
+    std::unordered_map<const Condition *, size_t> condition_positions;
+    size_t position= 0;
+    for (const auto & condition : where_conditions)
+        condition_positions[&condition] = position++;
+
     /// Move condition and all other conditions depend on the same set of columns.
     auto move_condition = [&](Conditions::iterator cond_it)
     {
         LOG_TRACE(log, "Condition {} moved to PREWHERE", cond_it->node.getColumnName());
-        prewhere_conditions.splice(prewhere_conditions.end(), where_conditions, cond_it);
+        /// Keep the original order of conditions in prewhere_conditions.
+        position = condition_positions[&(*cond_it)];
+        auto prewhere_it = prewhere_conditions.begin();
+        while (condition_positions[&(*prewhere_it)] < position && prewhere_it != prewhere_conditions.end())
+            ++prewhere_it;
+        prewhere_conditions.splice(prewhere_it, where_conditions, cond_it);
         total_size_of_moved_conditions += cond_it->columns_size;
         total_number_of_moved_columns += cond_it->table_columns.size();
 
@@ -375,7 +387,12 @@ std::optional<MergeTreeWhereOptimizer::OptimizeResult> MergeTreeWhereOptimizer::
             if (jt->viable && jt->columns_size == cond_it->columns_size && jt->table_columns == cond_it->table_columns)
             {
                 LOG_TRACE(log, "Condition {} moved to PREWHERE", jt->node.getColumnName());
-                prewhere_conditions.splice(prewhere_conditions.end(), where_conditions, jt++);
+                /// Keep the original order of conditions in prewhere_conditions.
+                position = condition_positions[&(*jt)];
+                prewhere_it = prewhere_conditions.begin();
+                while (condition_positions[&(*prewhere_it)] < position && prewhere_it != prewhere_conditions.end())
+                    ++prewhere_it;
+                prewhere_conditions.splice(prewhere_it, where_conditions, jt++);
             }
             else
             {

tests/queries/0_stateless/02156_storage_merge_prewhere.reference

@@ -1,25 +1,25 @@
           Prewhere info
             Prewhere filter
-            Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
+            Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
           Prewhere info
             Prewhere filter
-            Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
+            Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
           Prewhere info
             Prewhere filter
-            Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
+            Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
           Prewhere info
             Prewhere filter
-            Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
+            Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
 2
         Filter column: and(equals(k, 3), notEmpty(v)) (removed)
           Prewhere info
             Prewhere filter
-            Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
+            Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
 2
               Prewhere info
                 Prewhere filter
-                Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
+                Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
               Prewhere info
                 Prewhere filter
-                Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
+                Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
 2

tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.reference

@@ -1,15 +1,15 @@
           Prewhere filter
-          Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
+          Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
 1
           Prewhere filter
-          Prewhere filter column: and(like(d, \'%es%\'), less(c, 20), equals(b, \'3\'), equals(a, 3)) (removed)
+          Prewhere filter column: and(equals(a, 3), equals(b, \'3\'), less(c, 20), like(d, \'%es%\')) (removed)
 1
           Prewhere filter
-          Prewhere filter column: and(like(d, \'%es%\'), less(c, 20), greater(c, 0), equals(a, 3)) (removed)
+          Prewhere filter column: and(equals(a, 3), less(c, 20), greater(c, 0), like(d, \'%es%\')) (removed)
 1
           Prewhere filter
-          Prewhere filter column: and(like(d, \'%es%\'), equals(b, \'3\'), less(c, 20)) (removed)
+          Prewhere filter column: and(equals(b, \'3\'), less(c, 20), like(d, \'%es%\')) (removed)
 1
           Prewhere filter
-          Prewhere filter column: and(like(d, \'%es%\'), equals(b, \'3\'), equals(a, 3)) (removed)
+          Prewhere filter column: and(equals(a, 3), equals(b, \'3\'), like(d, \'%es%\')) (removed)
 1

tests/queries/0_stateless/02864_statistics_delayed_materialization_in_merge.reference

@@ -5,8 +5,8 @@ After insert
 After merge
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed)
+          Prewhere filter column: and(less(b, 10_UInt8), less(a, 10_UInt8)) (removed)
 After truncate, insert, and materialize
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed)
+          Prewhere filter column: and(less(b, 10_UInt8), less(a, 10_UInt8)) (removed)

tests/queries/0_stateless/02864_statistics_usage.reference

@@ -1,7 +1,7 @@
 After insert
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed)
+          Prewhere filter column: and(less(b, 10_UInt8), less(a, 10_UInt8)) (removed)
 After drop statistic
         Prewhere info
           Prewhere filter
@@ -9,12 +9,12 @@ After drop statistic
 After add and materialize statistic
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed)
+          Prewhere filter column: and(less(b, 10_UInt8), less(a, 10_UInt8)) (removed)
 After merge
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed)
+          Prewhere filter column: and(less(b, 10_UInt8), less(a, 10_UInt8)) (removed)
 After rename
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(less(a, 10_UInt8), less(c, 10_UInt8)) (removed)
+          Prewhere filter column: and(less(c, 10_UInt8), less(a, 10_UInt8)) (removed)

tests/queries/0_stateless/03231_prewhere_conditions_order.reference

@@ -0,0 +1 @@
+1	[0,1]	[0,1]

tests/queries/0_stateless/03231_prewhere_conditions_order.sql

@@ -0,0 +1,6 @@
+drop table if exists test;
+create table test (x UInt32, arr1 Array(UInt32), arr2 Array(UInt32)) engine=MergeTree order by x;
+insert into test values (1, [0, 1], [0, 1]), (2, [0], [0, 1]);
+select * from test where x == 1 and arrayExists((x1, x2) -> (x1 == x2), arr1, arr2);
+drop table test;
+

tests/queries/1_stateful/00091_prewhere_two_conditions.sql

@@ -7,7 +7,7 @@ SET optimize_move_to_prewhere = 1;
 SET enable_multiple_prewhere_read_steps = 1;
 
 SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00';
-SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND URL != '' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00';
+SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00' AND URL != '';
 SELECT uniq(*) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00' AND EventDate = '2014-03-21';
 WITH toTimeZone(EventTime, 'Asia/Dubai') AS xyz SELECT uniq(*) FROM test.hits WHERE xyz >= '2014-03-20 00:00:00' AND xyz < '2014-03-21 00:00:00' AND EventDate = '2014-03-21';