2023-12-01 20:39:26 +00:00
|
|
|
-- Tags: long, no-s3-storage
|
2023-01-18 18:37:24 +00:00
|
|
|
|
2024-03-04 11:27:12 +00:00
|
|
|
SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;
|
|
|
|
|
2023-01-05 19:22:42 +00:00
|
|
|
set max_threads = 16;
|
2023-01-13 22:52:02 +00:00
|
|
|
set allow_aggregate_partitions_independently = 1;
|
|
|
|
set force_aggregate_partitions_independently = 1;
|
2023-05-10 01:35:13 +00:00
|
|
|
set optimize_use_projections = 0;
|
2023-01-05 19:22:42 +00:00
|
|
|
|
2023-08-29 13:38:51 +00:00
|
|
|
set allow_prefetched_read_pool_for_remote_filesystem = 0;
|
|
|
|
set allow_prefetched_read_pool_for_local_filesystem = 0;
|
|
|
|
|
2023-02-04 21:09:44 +00:00
|
|
|
create table t1(a UInt32) engine=MergeTree order by tuple() partition by a % 4 settings index_granularity = 8192, index_granularity_bytes = 10485760;
|
2023-01-05 19:22:42 +00:00
|
|
|
|
|
|
|
system stop merges t1;
|
|
|
|
|
|
|
|
insert into t1 select number from numbers_mt(1e6);
|
|
|
|
insert into t1 select number from numbers_mt(1e6);
|
|
|
|
|
2024-02-10 11:43:35 +00:00
|
|
|
-- { echoOn }
|
2023-01-05 19:22:42 +00:00
|
|
|
explain pipeline select a from t1 group by a;
|
2024-02-10 11:43:35 +00:00
|
|
|
-- { echoOff }
|
2023-01-05 19:22:42 +00:00
|
|
|
|
|
|
|
select count() from (select throwIf(count() != 2) from t1 group by a);
|
|
|
|
|
|
|
|
drop table t1;
|
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t2(a UInt32) engine=MergeTree order by tuple() partition by a % 8 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-05 19:22:42 +00:00
|
|
|
|
|
|
|
system stop merges t2;
|
|
|
|
|
|
|
|
insert into t2 select number from numbers_mt(1e6);
|
|
|
|
insert into t2 select number from numbers_mt(1e6);
|
|
|
|
|
2024-02-10 11:43:35 +00:00
|
|
|
-- { echoOn }
|
2023-01-05 19:22:42 +00:00
|
|
|
explain pipeline select a from t2 group by a;
|
2024-02-10 11:43:35 +00:00
|
|
|
-- { echoOff }
|
2023-01-05 19:22:42 +00:00
|
|
|
|
|
|
|
select count() from (select throwIf(count() != 2) from t2 group by a);
|
|
|
|
|
|
|
|
drop table t2;
|
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t3(a UInt32) engine=MergeTree order by tuple() partition by a % 16 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-05 19:22:42 +00:00
|
|
|
|
|
|
|
system stop merges t3;
|
|
|
|
|
|
|
|
insert into t3 select number from numbers_mt(1e6);
|
|
|
|
insert into t3 select number from numbers_mt(1e6);
|
|
|
|
|
2024-02-10 11:43:35 +00:00
|
|
|
-- { echoOn }
|
2023-01-05 19:22:42 +00:00
|
|
|
explain pipeline select a from t3 group by a;
|
2024-02-10 11:43:35 +00:00
|
|
|
-- { echoOff }
|
2023-01-05 19:22:42 +00:00
|
|
|
|
|
|
|
select count() from (select throwIf(count() != 2) from t3 group by a);
|
|
|
|
|
|
|
|
select throwIf(count() != 4) from remote('127.0.0.{1,2}', currentDatabase(), t3) group by a format Null;
|
|
|
|
|
2023-01-11 22:32:54 +00:00
|
|
|
-- if we happened to switch to external aggregation at some point, merging will happen as usual
|
|
|
|
select count() from (select throwIf(count() != 2) from t3 group by a) settings max_bytes_before_external_group_by = '1Ki';
|
|
|
|
|
2023-01-05 19:22:42 +00:00
|
|
|
drop table t3;
|
2023-01-11 21:10:51 +00:00
|
|
|
|
|
|
|
-- aggregation in order --
|
|
|
|
|
|
|
|
set optimize_aggregation_in_order = 1;
|
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t4(a UInt32) engine=MergeTree order by a partition by a % 4 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-11 21:10:51 +00:00
|
|
|
|
|
|
|
system stop merges t4;
|
|
|
|
|
|
|
|
insert into t4 select number from numbers_mt(1e6);
|
|
|
|
insert into t4 select number from numbers_mt(1e6);
|
|
|
|
|
2024-02-10 11:43:35 +00:00
|
|
|
-- { echoOn }
|
2023-01-11 21:10:51 +00:00
|
|
|
explain pipeline select a from t4 group by a settings read_in_order_two_level_merge_threshold = 1e12;
|
2024-02-10 11:43:35 +00:00
|
|
|
-- { echoOff }
|
2023-01-11 21:10:51 +00:00
|
|
|
|
|
|
|
select count() from (select throwIf(count() != 2) from t4 group by a);
|
|
|
|
|
|
|
|
drop table t4;
|
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t5(a UInt32) engine=MergeTree order by a partition by a % 8 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-11 21:10:51 +00:00
|
|
|
|
|
|
|
system stop merges t5;
|
|
|
|
|
|
|
|
insert into t5 select number from numbers_mt(1e6);
|
|
|
|
insert into t5 select number from numbers_mt(1e6);
|
|
|
|
|
2024-02-10 11:43:35 +00:00
|
|
|
-- { echoOn }
|
2023-01-11 21:10:51 +00:00
|
|
|
explain pipeline select a from t5 group by a settings read_in_order_two_level_merge_threshold = 1e12;
|
2024-02-10 11:43:35 +00:00
|
|
|
-- { echoOff }
|
2023-01-11 21:10:51 +00:00
|
|
|
|
|
|
|
select count() from (select throwIf(count() != 2) from t5 group by a);
|
|
|
|
|
|
|
|
drop table t5;
|
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t6(a UInt32) engine=MergeTree order by a partition by a % 16 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-11 21:10:51 +00:00
|
|
|
|
|
|
|
system stop merges t6;
|
|
|
|
|
|
|
|
insert into t6 select number from numbers_mt(1e6);
|
|
|
|
insert into t6 select number from numbers_mt(1e6);
|
|
|
|
|
2024-02-10 11:43:35 +00:00
|
|
|
-- { echoOn }
|
2023-01-11 21:10:51 +00:00
|
|
|
explain pipeline select a from t6 group by a settings read_in_order_two_level_merge_threshold = 1e12;
|
2024-02-10 11:43:35 +00:00
|
|
|
-- { echoOff }
|
2023-01-11 21:10:51 +00:00
|
|
|
|
|
|
|
select count() from (select throwIf(count() != 2) from t6 group by a);
|
|
|
|
|
|
|
|
drop table t6;
|
2023-01-13 22:52:02 +00:00
|
|
|
|
2023-01-23 13:09:50 +00:00
|
|
|
set optimize_aggregation_in_order = 0;
|
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t7(a UInt32) engine=MergeTree order by a partition by intDiv(a, 2) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-13 22:52:02 +00:00
|
|
|
|
|
|
|
insert into t7 select number from numbers_mt(100);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select intDiv(a, 2) as a1 from t7 group by a1
|
|
|
|
) where explain like '%Skip merging: %';
|
|
|
|
|
|
|
|
drop table t7;
|
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t8(a UInt32) engine=MergeTree order by a partition by intDiv(a, 2) * 2 + 1 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-13 22:52:02 +00:00
|
|
|
|
|
|
|
insert into t8 select number from numbers_mt(100);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select intDiv(a, 2) + 1 as a1 from t8 group by a1
|
|
|
|
) where explain like '%Skip merging: %';
|
|
|
|
|
|
|
|
drop table t8;
|
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t9(a UInt32) engine=MergeTree order by a partition by intDiv(a, 2) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-13 22:52:02 +00:00
|
|
|
|
|
|
|
insert into t9 select number from numbers_mt(100);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select intDiv(a, 3) as a1 from t9 group by a1
|
|
|
|
) where explain like '%Skip merging: %';
|
|
|
|
|
|
|
|
drop table t9;
|
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t10(a UInt32, b UInt32) engine=MergeTree order by a partition by (intDiv(a, 2), intDiv(b, 3)) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-13 22:52:02 +00:00
|
|
|
|
|
|
|
insert into t10 select number, number from numbers_mt(100);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select intDiv(a, 2) + 1 as a1, intDiv(b, 3) as b1 from t10 group by a1, b1, pi()
|
|
|
|
) where explain like '%Skip merging: %';
|
|
|
|
|
|
|
|
drop table t10;
|
|
|
|
|
|
|
|
-- multiplication by 2 is not injective, so optimization is not applicable
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t11(a UInt32, b UInt32) engine=MergeTree order by a partition by (intDiv(a, 2), intDiv(b, 3)) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-13 22:52:02 +00:00
|
|
|
|
|
|
|
insert into t11 select number, number from numbers_mt(100);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select intDiv(a, 2) + 1 as a1, intDiv(b, 3) * 2 as b1 from t11 group by a1, b1, pi()
|
|
|
|
) where explain like '%Skip merging: %';
|
|
|
|
|
|
|
|
drop table t11;
|
|
|
|
|
|
|
|
create table t12(a UInt32, b UInt32) engine=MergeTree order by a partition by a % 16;
|
|
|
|
|
|
|
|
insert into t12 select number, number from numbers_mt(100);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select a, b from t12 group by a, b, pi()
|
|
|
|
) where explain like '%Skip merging: %';
|
|
|
|
|
|
|
|
drop table t12;
|
2023-01-17 14:50:38 +00:00
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t13(a UInt32, b UInt32) engine=MergeTree order by a partition by (intDiv(a, 2), intDiv(b, 3)) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-17 14:50:38 +00:00
|
|
|
|
|
|
|
insert into t13 select number, number from numbers_mt(100);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select s from t13 group by intDiv(a, 2) + intDiv(b, 3) as s, pi()
|
|
|
|
) where explain like '%Skip merging: %';
|
|
|
|
|
|
|
|
drop table t13;
|
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t14(a UInt32, b UInt32) engine=MergeTree order by a partition by intDiv(a, 2) + intDiv(b, 3) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-17 14:50:38 +00:00
|
|
|
|
|
|
|
insert into t14 select number, number from numbers_mt(100);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select intDiv(a, 2) as a1, intDiv(b, 3) as b1 from t14 group by a1, b1, pi()
|
|
|
|
) where explain like '%Skip merging: %';
|
|
|
|
|
|
|
|
drop table t14;
|
2023-01-18 14:22:32 +00:00
|
|
|
|
|
|
|
-- to few partitions --
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t15(a UInt32, b UInt32) engine=MergeTree order by a partition by a < 90 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-18 14:22:32 +00:00
|
|
|
|
|
|
|
insert into t15 select number, number from numbers_mt(100);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select a from t15 group by a
|
|
|
|
) where explain like '%Skip merging: %'
|
|
|
|
settings force_aggregate_partitions_independently = 0;
|
|
|
|
|
|
|
|
drop table t15;
|
|
|
|
|
|
|
|
-- to many partitions --
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t16(a UInt32, b UInt32) engine=MergeTree order by a partition by a % 16 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-18 14:22:32 +00:00
|
|
|
|
|
|
|
insert into t16 select number, number from numbers_mt(100);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select a from t16 group by a
|
|
|
|
) where explain like '%Skip merging: %'
|
|
|
|
settings force_aggregate_partitions_independently = 0, max_number_of_partitions_for_independent_aggregation = 4;
|
|
|
|
|
|
|
|
drop table t16;
|
|
|
|
|
|
|
|
-- to big skew --
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t17(a UInt32, b UInt32) engine=MergeTree order by a partition by a < 90 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-18 14:22:32 +00:00
|
|
|
|
|
|
|
insert into t17 select number, number from numbers_mt(100);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select a from t17 group by a
|
|
|
|
) where explain like '%Skip merging: %'
|
|
|
|
settings force_aggregate_partitions_independently = 0, max_threads = 4;
|
|
|
|
|
|
|
|
drop table t17;
|
2023-01-22 19:39:24 +00:00
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t18(a UInt32, b UInt32) engine=MergeTree order by a partition by a SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-22 19:39:24 +00:00
|
|
|
|
|
|
|
insert into t18 select number, number from numbers_mt(50);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select a1 from t18 group by intDiv(a, 2) as a1
|
|
|
|
) where explain like '%Skip merging: %';
|
|
|
|
|
|
|
|
drop table t18;
|
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t19(a UInt32, b UInt32) engine=MergeTree order by a partition by a SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-22 19:39:24 +00:00
|
|
|
|
|
|
|
insert into t19 select number, number from numbers_mt(50);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select a1 from t19 group by blockNumber() as a1
|
|
|
|
) where explain like '%Skip merging: %';
|
|
|
|
|
|
|
|
drop table t19;
|
2023-01-22 20:30:20 +00:00
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t20(a UInt32, b UInt32) engine=MergeTree order by a partition by a SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-22 20:30:20 +00:00
|
|
|
|
|
|
|
insert into t20 select number, number from numbers_mt(50);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select a1 from t20 group by rand(a) as a1
|
|
|
|
) where explain like '%Skip merging: %';
|
|
|
|
|
|
|
|
drop table t20;
|
2023-01-23 13:09:50 +00:00
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t21(a UInt64, b UInt64) engine=MergeTree order by a partition by a % 16 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-23 13:09:50 +00:00
|
|
|
|
|
|
|
insert into t21 select number, number from numbers_mt(1e6);
|
|
|
|
|
|
|
|
select a from t21 group by a limit 10 format Null;
|
|
|
|
|
|
|
|
drop table t21;
|
2023-01-23 18:35:42 +00:00
|
|
|
|
2023-08-12 17:31:18 +00:00
|
|
|
create table t22(a UInt32, b UInt32) engine=SummingMergeTree order by a partition by a % 16 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
2023-01-23 18:35:42 +00:00
|
|
|
|
|
|
|
insert into t22 select number, number from numbers_mt(1e6);
|
|
|
|
|
|
|
|
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
|
|
|
|
explain actions=1 select a from t22 final group by a
|
|
|
|
) where explain like '%Skip merging: %';
|
|
|
|
|
|
|
|
drop table t22;
|