ClickHouse/tests/queries/0_stateless/02521_aggregation_by_partitions.sql

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

278 lines
10 KiB
MySQL
Raw Normal View History

-- Tags: long, no-s3-storage
2023-01-18 18:37:24 +00:00
2024-03-04 11:27:12 +00:00
SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;
2023-01-05 19:22:42 +00:00
set max_threads = 16;
2023-01-13 22:52:02 +00:00
set allow_aggregate_partitions_independently = 1;
set force_aggregate_partitions_independently = 1;
2023-05-10 01:35:13 +00:00
set optimize_use_projections = 0;
2023-01-05 19:22:42 +00:00
set allow_prefetched_read_pool_for_remote_filesystem = 0;
set allow_prefetched_read_pool_for_local_filesystem = 0;
2023-02-04 21:09:44 +00:00
create table t1(a UInt32) engine=MergeTree order by tuple() partition by a % 4 settings index_granularity = 8192, index_granularity_bytes = 10485760;
2023-01-05 19:22:42 +00:00
system stop merges t1;
insert into t1 select number from numbers_mt(1e6);
insert into t1 select number from numbers_mt(1e6);
2024-02-10 11:43:35 +00:00
-- { echoOn }
2023-01-05 19:22:42 +00:00
explain pipeline select a from t1 group by a;
2024-02-10 11:43:35 +00:00
-- { echoOff }
2023-01-05 19:22:42 +00:00
select count() from (select throwIf(count() != 2) from t1 group by a);
drop table t1;
2023-08-12 17:31:18 +00:00
create table t2(a UInt32) engine=MergeTree order by tuple() partition by a % 8 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-05 19:22:42 +00:00
system stop merges t2;
insert into t2 select number from numbers_mt(1e6);
insert into t2 select number from numbers_mt(1e6);
2024-02-10 11:43:35 +00:00
-- { echoOn }
2023-01-05 19:22:42 +00:00
explain pipeline select a from t2 group by a;
2024-02-10 11:43:35 +00:00
-- { echoOff }
2023-01-05 19:22:42 +00:00
select count() from (select throwIf(count() != 2) from t2 group by a);
drop table t2;
2023-08-12 17:31:18 +00:00
create table t3(a UInt32) engine=MergeTree order by tuple() partition by a % 16 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-05 19:22:42 +00:00
system stop merges t3;
insert into t3 select number from numbers_mt(1e6);
insert into t3 select number from numbers_mt(1e6);
2024-02-10 11:43:35 +00:00
-- { echoOn }
2023-01-05 19:22:42 +00:00
explain pipeline select a from t3 group by a;
2024-02-10 11:43:35 +00:00
-- { echoOff }
2023-01-05 19:22:42 +00:00
select count() from (select throwIf(count() != 2) from t3 group by a);
select throwIf(count() != 4) from remote('127.0.0.{1,2}', currentDatabase(), t3) group by a format Null;
2023-01-11 22:32:54 +00:00
-- if we happened to switch to external aggregation at some point, merging will happen as usual
select count() from (select throwIf(count() != 2) from t3 group by a) settings max_bytes_before_external_group_by = '1Ki';
2023-01-05 19:22:42 +00:00
drop table t3;
2023-01-11 21:10:51 +00:00
-- aggregation in order --
set optimize_aggregation_in_order = 1;
2023-08-12 17:31:18 +00:00
create table t4(a UInt32) engine=MergeTree order by a partition by a % 4 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-11 21:10:51 +00:00
system stop merges t4;
insert into t4 select number from numbers_mt(1e6);
insert into t4 select number from numbers_mt(1e6);
2024-02-10 11:43:35 +00:00
-- { echoOn }
2023-01-11 21:10:51 +00:00
explain pipeline select a from t4 group by a settings read_in_order_two_level_merge_threshold = 1e12;
2024-02-10 11:43:35 +00:00
-- { echoOff }
2023-01-11 21:10:51 +00:00
select count() from (select throwIf(count() != 2) from t4 group by a);
drop table t4;
2023-08-12 17:31:18 +00:00
create table t5(a UInt32) engine=MergeTree order by a partition by a % 8 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-11 21:10:51 +00:00
system stop merges t5;
insert into t5 select number from numbers_mt(1e6);
insert into t5 select number from numbers_mt(1e6);
2024-02-10 11:43:35 +00:00
-- { echoOn }
2023-01-11 21:10:51 +00:00
explain pipeline select a from t5 group by a settings read_in_order_two_level_merge_threshold = 1e12;
2024-02-10 11:43:35 +00:00
-- { echoOff }
2023-01-11 21:10:51 +00:00
select count() from (select throwIf(count() != 2) from t5 group by a);
drop table t5;
2023-08-12 17:31:18 +00:00
create table t6(a UInt32) engine=MergeTree order by a partition by a % 16 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-11 21:10:51 +00:00
system stop merges t6;
insert into t6 select number from numbers_mt(1e6);
insert into t6 select number from numbers_mt(1e6);
2024-02-10 11:43:35 +00:00
-- { echoOn }
2023-01-11 21:10:51 +00:00
explain pipeline select a from t6 group by a settings read_in_order_two_level_merge_threshold = 1e12;
2024-02-10 11:43:35 +00:00
-- { echoOff }
2023-01-11 21:10:51 +00:00
select count() from (select throwIf(count() != 2) from t6 group by a);
drop table t6;
2023-01-13 22:52:02 +00:00
2023-01-23 13:09:50 +00:00
set optimize_aggregation_in_order = 0;
2023-08-12 17:31:18 +00:00
create table t7(a UInt32) engine=MergeTree order by a partition by intDiv(a, 2) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-13 22:52:02 +00:00
insert into t7 select number from numbers_mt(100);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select intDiv(a, 2) as a1 from t7 group by a1
) where explain like '%Skip merging: %';
drop table t7;
2023-08-12 17:31:18 +00:00
create table t8(a UInt32) engine=MergeTree order by a partition by intDiv(a, 2) * 2 + 1 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-13 22:52:02 +00:00
insert into t8 select number from numbers_mt(100);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select intDiv(a, 2) + 1 as a1 from t8 group by a1
) where explain like '%Skip merging: %';
drop table t8;
2023-08-12 17:31:18 +00:00
create table t9(a UInt32) engine=MergeTree order by a partition by intDiv(a, 2) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-13 22:52:02 +00:00
insert into t9 select number from numbers_mt(100);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select intDiv(a, 3) as a1 from t9 group by a1
) where explain like '%Skip merging: %';
drop table t9;
2023-08-12 17:31:18 +00:00
create table t10(a UInt32, b UInt32) engine=MergeTree order by a partition by (intDiv(a, 2), intDiv(b, 3)) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-13 22:52:02 +00:00
insert into t10 select number, number from numbers_mt(100);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select intDiv(a, 2) + 1 as a1, intDiv(b, 3) as b1 from t10 group by a1, b1, pi()
) where explain like '%Skip merging: %';
drop table t10;
-- multiplication by 2 is not injective, so optimization is not applicable
2023-08-12 17:31:18 +00:00
create table t11(a UInt32, b UInt32) engine=MergeTree order by a partition by (intDiv(a, 2), intDiv(b, 3)) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-13 22:52:02 +00:00
insert into t11 select number, number from numbers_mt(100);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select intDiv(a, 2) + 1 as a1, intDiv(b, 3) * 2 as b1 from t11 group by a1, b1, pi()
) where explain like '%Skip merging: %';
drop table t11;
create table t12(a UInt32, b UInt32) engine=MergeTree order by a partition by a % 16;
insert into t12 select number, number from numbers_mt(100);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select a, b from t12 group by a, b, pi()
) where explain like '%Skip merging: %';
drop table t12;
2023-01-17 14:50:38 +00:00
2023-08-12 17:31:18 +00:00
create table t13(a UInt32, b UInt32) engine=MergeTree order by a partition by (intDiv(a, 2), intDiv(b, 3)) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-17 14:50:38 +00:00
insert into t13 select number, number from numbers_mt(100);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select s from t13 group by intDiv(a, 2) + intDiv(b, 3) as s, pi()
) where explain like '%Skip merging: %';
drop table t13;
2023-08-12 17:31:18 +00:00
create table t14(a UInt32, b UInt32) engine=MergeTree order by a partition by intDiv(a, 2) + intDiv(b, 3) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-17 14:50:38 +00:00
insert into t14 select number, number from numbers_mt(100);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select intDiv(a, 2) as a1, intDiv(b, 3) as b1 from t14 group by a1, b1, pi()
) where explain like '%Skip merging: %';
drop table t14;
2023-01-18 14:22:32 +00:00
-- to few partitions --
2023-08-12 17:31:18 +00:00
create table t15(a UInt32, b UInt32) engine=MergeTree order by a partition by a < 90 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-18 14:22:32 +00:00
insert into t15 select number, number from numbers_mt(100);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select a from t15 group by a
) where explain like '%Skip merging: %'
settings force_aggregate_partitions_independently = 0;
drop table t15;
-- to many partitions --
2023-08-12 17:31:18 +00:00
create table t16(a UInt32, b UInt32) engine=MergeTree order by a partition by a % 16 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-18 14:22:32 +00:00
insert into t16 select number, number from numbers_mt(100);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select a from t16 group by a
) where explain like '%Skip merging: %'
settings force_aggregate_partitions_independently = 0, max_number_of_partitions_for_independent_aggregation = 4;
drop table t16;
-- to big skew --
2023-08-12 17:31:18 +00:00
create table t17(a UInt32, b UInt32) engine=MergeTree order by a partition by a < 90 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-18 14:22:32 +00:00
insert into t17 select number, number from numbers_mt(100);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select a from t17 group by a
) where explain like '%Skip merging: %'
settings force_aggregate_partitions_independently = 0, max_threads = 4;
drop table t17;
2023-01-22 19:39:24 +00:00
2023-08-12 17:31:18 +00:00
create table t18(a UInt32, b UInt32) engine=MergeTree order by a partition by a SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-22 19:39:24 +00:00
insert into t18 select number, number from numbers_mt(50);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select a1 from t18 group by intDiv(a, 2) as a1
) where explain like '%Skip merging: %';
drop table t18;
2023-08-12 17:31:18 +00:00
create table t19(a UInt32, b UInt32) engine=MergeTree order by a partition by a SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-22 19:39:24 +00:00
insert into t19 select number, number from numbers_mt(50);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select a1 from t19 group by blockNumber() as a1
) where explain like '%Skip merging: %';
drop table t19;
2023-08-12 17:31:18 +00:00
create table t20(a UInt32, b UInt32) engine=MergeTree order by a partition by a SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
insert into t20 select number, number from numbers_mt(50);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select a1 from t20 group by rand(a) as a1
) where explain like '%Skip merging: %';
drop table t20;
2023-01-23 13:09:50 +00:00
2023-08-12 17:31:18 +00:00
create table t21(a UInt64, b UInt64) engine=MergeTree order by a partition by a % 16 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-23 13:09:50 +00:00
insert into t21 select number, number from numbers_mt(1e6);
select a from t21 group by a limit 10 format Null;
drop table t21;
2023-01-23 18:35:42 +00:00
2023-08-12 17:31:18 +00:00
create table t22(a UInt32, b UInt32) engine=SummingMergeTree order by a partition by a % 16 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
2023-01-23 18:35:42 +00:00
insert into t22 select number, number from numbers_mt(1e6);
select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from (
explain actions=1 select a from t22 final group by a
) where explain like '%Skip merging: %';
drop table t22;