thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Backport #69560 to 24.6: Keep original order of conditions during move to prewhere

Browse Source
This commit is contained in:
robot-clickhouse 2024-09-18 11:09:57 +00:00
parent d6d5449890
commit 6068f0433a
6 changed files with 39 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
View File

@ -360,11 +360,23 @@ std::optional<MergeTreeWhereOptimizer::OptimizeResult> MergeTreeWhereOptimizer::
UInt64 total_size_of_moved_conditions = 0;
UInt64 total_number_of_moved_columns = 0;
/// Remember positions of conditions in where_conditions list
/// to keep original order of conditions in prewhere_conditions while moving.
std::unordered_map<const Condition *, size_t> condition_positions;
size_t position= 0;
for (const auto & condition : where_conditions)
condition_positions[&condition] = position++;
/// Move condition and all other conditions depend on the same set of columns.
auto move_condition = [&](Conditions::iterator cond_it)
{
LOG_TRACE(log, "Condition {} moved to PREWHERE", cond_it->node.getColumnName());
prewhere_conditions.splice(prewhere_conditions.end(), where_conditions, cond_it);
/// Keep the original order of conditions in prewhere_conditions.
position = condition_positions[&(*cond_it)];
auto prewhere_it = prewhere_conditions.begin();
while (condition_positions[&(*prewhere_it)] < position && prewhere_it != prewhere_conditions.end())
++prewhere_it;
prewhere_conditions.splice(prewhere_it, where_conditions, cond_it);
total_size_of_moved_conditions += cond_it->columns_size;
total_number_of_moved_columns += cond_it->table_columns.size();
@ -374,7 +386,12 @@ std::optional<MergeTreeWhereOptimizer::OptimizeResult> MergeTreeWhereOptimizer::
if (jt->viable && jt->columns_size == cond_it->columns_size && jt->table_columns == cond_it->table_columns)
{
LOG_TRACE(log, "Condition {} moved to PREWHERE", jt->node.getColumnName());
prewhere_conditions.splice(prewhere_conditions.end(), where_conditions, jt++);
/// Keep the original order of conditions in prewhere_conditions.
position = condition_positions[&(*jt)];
prewhere_it = prewhere_conditions.begin();
while (condition_positions[&(*prewhere_it)] < position && prewhere_it != prewhere_conditions.end())
++prewhere_it;
prewhere_conditions.splice(prewhere_it, where_conditions, jt++);
}
else
{

14
tests/queries/0_stateless/02156_storage_merge_prewhere.reference
View File

@ -1,25 +1,25 @@
Prewhere info
Prewhere filter
Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
Prewhere info
Prewhere filter
Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
Prewhere info
Prewhere filter
Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
Prewhere info
Prewhere filter
Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
2
Filter column: and(equals(k, 3), notEmpty(v)) (removed)
Prewhere info
Prewhere filter
Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
2
Prewhere info
Prewhere filter
Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
Prewhere info
Prewhere filter
Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
2

10
tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.reference
View File

@ -1,15 +1,15 @@
Prewhere filter
Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed)
Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed)
1
Prewhere filter
Prewhere filter column: and(like(d, \'%es%\'), less(c, 20), equals(b, \'3\'), equals(a, 3)) (removed)
Prewhere filter column: and(equals(a, 3), equals(b, \'3\'), less(c, 20), like(d, \'%es%\')) (removed)
1
Prewhere filter
Prewhere filter column: and(like(d, \'%es%\'), less(c, 20), greater(c, 0), equals(a, 3)) (removed)
Prewhere filter column: and(equals(a, 3), less(c, 20), greater(c, 0), like(d, \'%es%\')) (removed)
1
Prewhere filter
Prewhere filter column: and(like(d, \'%es%\'), equals(b, \'3\'), less(c, 20)) (removed)
Prewhere filter column: and(equals(b, \'3\'), less(c, 20), like(d, \'%es%\')) (removed)
1
Prewhere filter
Prewhere filter column: and(like(d, \'%es%\'), equals(b, \'3\'), equals(a, 3)) (removed)
Prewhere filter column: and(equals(a, 3), equals(b, \'3\'), like(d, \'%es%\')) (removed)
1

1
tests/queries/0_stateless/03231_prewhere_conditions_order.reference Normal file
View File

@ -0,0 +1 @@
1 [0,1] [0,1]

6
tests/queries/0_stateless/03231_prewhere_conditions_order.sql Normal file
View File

@ -0,0 +1,6 @@
drop table if exists test;
create table test (x UInt32, arr1 Array(UInt32), arr2 Array(UInt32)) engine=MergeTree order by x;
insert into test values (1, [0, 1], [0, 1]), (2, [0], [0, 1]);
select * from test where x == 1 and arrayExists((x1, x2) -> (x1 == x2), arr1, arr2);
drop table test;

2
tests/queries/1_stateful/00091_prewhere_two_conditions.sql
View File

@ -7,7 +7,7 @@ SET optimize_move_to_prewhere = 1;
SET enable_multiple_prewhere_read_steps = 1;
SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00';
SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND URL != '' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00';
SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00' AND URL != '';
SELECT uniq(*) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00' AND EventDate = '2014-03-21';
WITH toTimeZone(EventTime, 'Asia/Dubai') AS xyz SELECT uniq(*) FROM test.hits WHERE xyz >= '2014-03-20 00:00:00' AND xyz < '2014-03-21 00:00:00' AND EventDate = '2014-03-21';