thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 16:42:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

more stable stateless test + add perf. test

Browse Source
This commit is contained in:
Duc Canh Le 2023-08-31 06:27:06 +00:00
parent 11c94246c7
commit 06afe0c2aa
4 changed files with 143 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
tests/performance/final_with_lonely_parts.xml Normal file
View File

@ -0,0 +1,77 @@
<test>
<settings>
<do_not_merge_across_partitions_select_final>1</do_not_merge_across_partitions_select_final>
<max_threads>1</max_threads>
</settings>
<create_query>
CREATE TABLE with_lonely
(
`id` UInt64,
`dt` Date,
`val` UInt64,
`version` UInt64
)
ENGINE = ReplacingMergeTree(version)
PARTITION BY dt
ORDER BY (id);
</create_query>
<fill_query>
INSERT INTO with_lonely SELECT number, '2022-10-28', number*10, 0 FROM numbers(3000000);
</fill_query>
<fill_query>
INSERT INTO with_lonely SELECT number+500000, '2022-10-28', number*10, 1 FROM
numbers(3000000);
</fill_query>
<fill_query>OPTIMIZE TABLE with_lonely PARTITION '2022-10-28' FINAL;</fill_query>
<fill_query>
INSERT INTO with_lonely SELECT number, '2022-10-29', number*10, 0 FROM numbers(3000000);
</fill_query>
<fill_query>
INSERT INTO with_lonely SELECT number+500000, '2022-10-29', number*10, 1 FROM
numbers(3000000);
</fill_query>
<fill_query>OPTIMIZE TABLE with_lonely PARTITION '2022-10-29' FINAL;</fill_query>
<fill_query>
INSERT INTO with_lonely SELECT number, '2022-10-30', number*10, 0 FROM numbers(3000000);
</fill_query>
<fill_query>
INSERT INTO with_lonely SELECT number+500000, '2022-10-30', number*10, 1 FROM
numbers(3000000);
</fill_query>
<fill_query>OPTIMIZE TABLE with_lonely PARTITION '2022-10-30' FINAL;</fill_query>
<fill_query>
INSERT INTO with_lonely SELECT number, '2022-10-31', number*10, 0 FROM numbers(3000000);
</fill_query>
<fill_query>
INSERT INTO with_lonely SELECT number+500000, '2022-10-31', number*10, 1 FROM
numbers(3000000);
</fill_query>
<fill_query>OPTIMIZE TABLE with_lonely PARTITION '2022-10-31' FINAL;</fill_query>
<fill_query>
INSERT INTO with_lonely SELECT number, '2022-11-01', number*10, 0 FROM numbers(500000);
</fill_query>
<fill_query>
INSERT INTO with_lonely SELECT number+50000, '2022-11-01', number*10, 1 FROM
numbers(500000);
</fill_query>
<fill_query>
INSERT INTO with_lonely SELECT number+60000, '2022-11-01', number*10, 2 FROM
numbers(500000);
</fill_query>
<fill_query>
INSERT INTO with_lonely SELECT number+70000, '2022-11-01', number*10, 3 FROM
numbers(500000);
</fill_query>
<query>SELECT max(val), count(*) FROM with_lonely FINAL;</query>
<query>SELECT max(val), count(*) FROM with_lonely FINAL WHERE dt != '2022-11-01';</query>
<drop_query>DROP TABLE IF EXISTS with_lonely;</drop_query>
</test>

208
tests/queries/0_stateless/02868_no_merge_across_partitions_final_with_lonely.reference
View File

@ -1,202 +1,6 @@
(Expression)
ExpressionTransform × 16
Header: max(val) UInt64: max(val) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
(Aggregating)
Resize 6 → 16
Header × 16 : max(val) UInt64: max(val) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
AggregatingTransform × 6
Header: max(val) UInt64: max(val) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
StrictResize 6 → 6
Header × 6 : val UInt64: val UInt64 UInt64(size = 0)
(Expression)
ExpressionTransform × 6
Header: val UInt64: val UInt64 UInt64(size = 0)
(ReadFromMergeTree)
MergeTreeThread × 4 0 → 1
Header: val UInt64: val UInt64 UInt64(size = 0)
ExpressionTransform × 2
Header: val UInt64: val UInt64 UInt64(size = 0)
ReplacingSorted 2 → 1
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
FilterSortedStreamByRange × 2
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
Description: filter values in [(50000), +inf)
ExpressionTransform × 2
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
MergeTreeInOrder × 2 0 → 1
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
ReplacingSorted 2 → 1
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
FilterSortedStreamByRange × 2
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
Description: filter values in [-inf, (50000))
ExpressionTransform × 2
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
MergeTreeInOrder × 2 0 → 1
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
(Expression)
ExpressionTransform × 16
Header: max(val) UInt64: max(val) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
(Aggregating)
Resize 4 → 16
Header × 16 : max(val) UInt64: max(val) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
AggregatingTransform × 4
Header: max(val) UInt64: max(val) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
StrictResize 4 → 4
Header × 4 : val UInt64: val UInt64 UInt64(size = 0)
(Expression)
ExpressionTransform × 4
Header: val UInt64: val UInt64 UInt64(size = 0)
(Filter)
FilterTransform × 4
Header: val UInt64: val UInt64 UInt64(size = 0)
(ReadFromMergeTree)
MergeTreeThread × 4 0 → 1
Header: dt Date: dt Date UInt16(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
(Expression)
ExpressionTransform
Header: max(val) UInt64: max(val) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
(Aggregating)
Resize 4 → 1
Header: max(val) UInt64: max(val) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
AggregatingTransform × 4
Header: max(val) UInt64: max(val) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
StrictResize 4 → 4
Header × 4 : val UInt64: val UInt64 UInt64(size = 0)
(Expression)
ExpressionTransform × 4
Header: val UInt64: val UInt64 UInt64(size = 0)
(Filter)
FilterTransform × 4
Header: val UInt64: val UInt64 UInt64(size = 0)
(ReadFromMergeTree)
MergeTreeInOrder × 4 0 → 1
Header: dt Date: dt Date UInt16(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
(Expression)
ExpressionTransform × 16
Header: max(val) UInt64: max(val) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
(Aggregating)
Resize 6 → 16
Header × 16 : max(val_0) UInt64: max(val_0) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
AggregatingTransform × 6
Header: max(val_0) UInt64: max(val_0) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
StrictResize 6 → 6
Header × 6 : val_0 UInt64: val_0 UInt64 UInt64(size = 0)
(Expression)
ExpressionTransform × 6
Header: val_0 UInt64: val_0 UInt64 UInt64(size = 0)
(ReadFromMergeTree)
MergeTreeThread × 4 0 → 1
Header: val UInt64: val UInt64 UInt64(size = 0)
ExpressionTransform × 2
Header: val UInt64: val UInt64 UInt64(size = 0)
ReplacingSorted 2 → 1
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
FilterSortedStreamByRange × 2
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
Description: filter values in [(50000), +inf)
ExpressionTransform × 2
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
MergeTreeInOrder × 2 0 → 1
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
ReplacingSorted 2 → 1
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
FilterSortedStreamByRange × 2
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
Description: filter values in [-inf, (50000))
ExpressionTransform × 2
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
MergeTreeInOrder × 2 0 → 1
Header: id UInt64: id UInt64 UInt64(size = 0)
val UInt64: val UInt64 UInt64(size = 0)
version UInt64: version UInt64 UInt64(size = 0)
(Expression)
ExpressionTransform × 16
Header: max(val) UInt64: max(val) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
(Aggregating)
Resize 4 → 16
Header × 16 : max(val_0) UInt64: max(val_0) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
AggregatingTransform × 4
Header: max(val_0) UInt64: max(val_0) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
StrictResize 4 → 4
Header × 4 : val_0 UInt64: val_0 UInt64 UInt64(size = 0)
(Expression)
ExpressionTransform × 4
Header: val_0 UInt64: val_0 UInt64 UInt64(size = 0)
(Filter)
FilterTransform × 4
Header: val_0 UInt64: val_0 UInt64 UInt64(size = 0)
(ReadFromMergeTree)
MergeTreeThread × 4 0 → 1
Header: val UInt64: val UInt64 UInt64(size = 0)
dt Date: dt Date UInt16(size = 0)
(Expression)
ExpressionTransform
Header: max(val) UInt64: max(val) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
(Aggregating)
Resize 4 → 1
Header: max(val_0) UInt64: max(val_0) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
AggregatingTransform × 4
Header: max(val_0) UInt64: max(val_0) UInt64 UInt64(size = 0)
count() UInt64: count() UInt64 UInt64(size = 0)
StrictResize 4 → 4
Header × 4 : val_0 UInt64: val_0 UInt64 UInt64(size = 0)
(Expression)
ExpressionTransform × 4
Header: val_0 UInt64: val_0 UInt64 UInt64(size = 0)
(Filter)
FilterTransform × 4
Header: val_0 UInt64: val_0 UInt64 UInt64(size = 0)
(ReadFromMergeTree)
MergeTreeInOrder × 4 0 → 1
Header: val UInt64: val UInt64 UInt64(size = 0)
dt Date: dt Date UInt16(size = 0)
90 110
90 80
90 80
90 110
90 80
90 80

60
tests/queries/0_stateless/02868_no_merge_across_partitions_final_with_lonely.sh Executable file
View File

@ -0,0 +1,60 @@
#!/usr/bin/env bash
set -e
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} -nm -q """
DROP TABLE IF EXISTS with_lonely;
CREATE TABLE with_lonely
(
id UInt64,
dt Date,
val UInt64,
version UInt64
)
ENGINE = ReplacingMergeTree(version)
PARTITION BY dt
ORDER BY (id);
"""
create_optimize_partition() {
${CLICKHOUSE_CLIENT} -nm -q """
INSERT INTO with_lonely SELECT number, '$1', number*10, 0 FROM numbers(10);
INSERT INTO with_lonely SELECT number+500000, '$1', number*10, 1 FROM numbers(10);
"""
cnt=$(${CLICKHOUSE_CLIENT} -q "SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 'with_lonely' AND partition = '$1' AND active")
while [ "$cnt" -ne "1" ]; do
${CLICKHOUSE_CLIENT} -q "OPTIMIZE TABLE with_lonely PARTITION '$1' FINAL;"
cnt=$(${CLICKHOUSE_CLIENT} -q "SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 'with_lonely' AND partition = '$1' AND active")
done
}
create_optimize_partition "2022-10-28"
create_optimize_partition "2022-10-29"
create_optimize_partition "2022-10-30"
create_optimize_partition "2022-10-31"
${CLICKHOUSE_CLIENT} -nm -q """
SYSTEM STOP MERGES with_lonely;
INSERT INTO with_lonely SELECT number, '2022-11-01', number*10, 0 FROM numbers(10);
INSERT INTO with_lonely SELECT number+50000, '2022-11-01', number*10, 1 FROM numbers(10);
INSERT INTO with_lonely SELECT number+60000, '2022-11-01', number*10, 2 FROM numbers(10);
"""
CLICKHOUSE_CLIENT="${CLICKHOUSE_CLIENT} --do_not_merge_across_partitions_select_final 1"
# mix lonely parts and non-lonely parts
${CLICKHOUSE_CLIENT} --allow_experimental_analyzer 1 --max_bytes_to_read 2000 -q "SELECT max(val), count(*) FROM with_lonely FINAL;"
# only lonely parts
${CLICKHOUSE_CLIENT} --allow_experimental_analyzer 1 --max_bytes_to_read 1000 -q "SELECT max(val), count(*) FROM with_lonely FINAL WHERE dt < '2022-11-01';"
# only lonely parts but max_thread = 1, so reading lonely parts with in-order
${CLICKHOUSE_CLIENT} --allow_experimental_analyzer 1 --max_threads 1 --max_bytes_to_read 1000 -q "SELECT max(val), count(*) FROM with_lonely FINAL WHERE dt < '2022-11-01';"
${CLICKHOUSE_CLIENT} --allow_experimental_analyzer 0 --max_bytes_to_read 2000 -q "SELECT max(val), count(*) FROM with_lonely FINAL;"
${CLICKHOUSE_CLIENT} --allow_experimental_analyzer 0 --max_bytes_to_read 1000 -q "SELECT max(val), count(*) FROM with_lonely FINAL WHERE dt < '2022-11-01';"
${CLICKHOUSE_CLIENT} --allow_experimental_analyzer 0 --max_threads 1 --max_bytes_to_read 1000 -q "SELECT max(val), count(*) FROM with_lonely FINAL WHERE dt < '2022-11-01';"

49
tests/queries/0_stateless/02868_no_merge_across_partitions_final_with_lonely.sql
View File

@ -1,49 +0,0 @@
DROP TABLE IF EXISTS with_lonely;
CREATE TABLE with_lonely
(
`id` UInt64,
`dt` Date,
`val` UInt64,
`version` UInt64
)
ENGINE = ReplacingMergeTree(version)
PARTITION BY dt
ORDER BY (id);
INSERT INTO with_lonely SELECT number, '2022-10-28', number*10, 0 FROM numbers(10000);
INSERT INTO with_lonely SELECT number+500000, '2022-10-28', number*10, 1 FROM numbers(10000);
OPTIMIZE TABLE with_lonely PARTITION '2022-10-28' FINAL;
INSERT INTO with_lonely SELECT number, '2022-10-29', number*10, 0 FROM numbers(10000);
INSERT INTO with_lonely SELECT number+500000, '2022-10-29', number*10, 1 FROM numbers(10000);
OPTIMIZE TABLE with_lonely PARTITION '2022-10-29' FINAL;
INSERT INTO with_lonely SELECT number, '2022-10-30', number*10, 0 FROM numbers(10000);
INSERT INTO with_lonely SELECT number+500000, '2022-10-30', number*10, 1 FROM numbers(10000);
OPTIMIZE TABLE with_lonely PARTITION '2022-10-30' FINAL;
INSERT INTO with_lonely SELECT number, '2022-10-31', number*10, 0 FROM numbers(10000);
INSERT INTO with_lonely SELECT number+500000, '2022-10-31', number*10, 1 FROM numbers(10000);
OPTIMIZE TABLE with_lonely PARTITION '2022-10-31' FINAL;
SYSTEM STOP MERGES with_lonely;
INSERT INTO with_lonely SELECT number, '2022-11-01', number*10, 0 FROM numbers(1000);
INSERT INTO with_lonely SELECT number+50000, '2022-11-01', number*10, 1 FROM numbers(1000);
INSERT INTO with_lonely SELECT number+60000, '2022-11-01', number*10, 2 FROM numbers(1000);
SET do_not_merge_across_partitions_select_final = 1, max_threads = 16;
-- mix lonely parts and non-lonely parts
EXPLAIN PIPELINE header=1 SELECT max(val), count(*) FROM with_lonely FINAL SETTINGS allow_experimental_analyzer = 0;
-- only lonely parts
EXPLAIN PIPELINE header=1 SELECT max(val), count(*) FROM with_lonely FINAL WHERE dt < '2022-11-01' SETTINGS allow_experimental_analyzer = 0;
-- only lonely parts but max_thread = 1, so reading lonely parts with in-order
EXPLAIN PIPELINE header=1 SELECT max(val), count(*) FROM with_lonely FINAL WHERE dt < '2022-11-01' SETTINGS max_threads = 1, allow_experimental_analyzer = 0;
EXPLAIN PIPELINE header=1 SELECT max(val), count(*) FROM with_lonely FINAL SETTINGS allow_experimental_analyzer = 1;
EXPLAIN PIPELINE header=1 SELECT max(val), count(*) FROM with_lonely FINAL WHERE dt < '2022-11-01' SETTINGS allow_experimental_analyzer = 1;
EXPLAIN PIPELINE header=1 SELECT max(val), count(*) FROM with_lonely FINAL WHERE dt < '2022-11-01' SETTINGS max_threads = 1, allow_experimental_analyzer = 1;