mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
simplify perf tests and minor code change
Signed-off-by: Duc Canh Le <duccanh.le@ahrefs.com>
This commit is contained in:
Duc Canh Le
parent
b42c5efc73
commit
458c8d758d
@ -40,8 +40,10 @@ static std::shared_ptr<const ChunkSelectFinalIndices> getSelectByFinalIndices(Ch
|
||||
{
|
||||
if (auto select_final_indices_info = std::dynamic_pointer_cast<const ChunkSelectFinalIndices>(chunk.getChunkInfo()))
|
||||
{
|
||||
const auto & index_column = select_final_indices_info->select_final_indices;
|
||||
chunk.setChunkInfo(nullptr);
|
||||
return select_final_indices_info;
|
||||
if (index_column && index_column->size() != chunk.getNumRows())
|
||||
return select_final_indices_info;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
@ -53,9 +55,8 @@ executeSelectByIndices(Columns & columns, std::shared_ptr<const ChunkSelectFinal
|
||||
{
|
||||
const auto & index_column = select_final_indices_info->select_final_indices;
|
||||
|
||||
if (index_column && index_column->size() != num_rows)
|
||||
for (auto & column : columns)
|
||||
column = column->index(*index_column, 0);
|
||||
for (auto & column : columns)
|
||||
column = column->index(*index_column, 0);
|
||||
|
||||
num_rows = index_column->size();
|
||||
}
|
||||
@ -70,7 +71,7 @@ static std::unique_ptr<IFilterDescription> combineFilterAndIndices(
|
||||
{
|
||||
const auto * index_column = select_final_indices_info->select_final_indices;
|
||||
|
||||
if (index_column && index_column->size() != num_rows && description->hasOne())
|
||||
if (description->hasOne())
|
||||
{
|
||||
const auto & selected_by_indices = index_column->getData();
|
||||
const auto * selected_by_filter = description->data->data();
|
||||
@ -154,7 +155,7 @@ static std::unique_ptr<IFilterDescription> combineFilterAndIndices(
|
||||
{
|
||||
const auto * index_column = select_final_indices_info->select_final_indices;
|
||||
|
||||
if (index_column && index_column->size() != num_rows && description->hasOne())
|
||||
if (description->hasOne())
|
||||
{
|
||||
std::unique_ptr<FilterDescription> res;
|
||||
res->has_one = 0;
|
||||
@ -169,7 +170,7 @@ static std::unique_ptr<IFilterDescription> combineFilterAndIndices(
|
||||
return res;
|
||||
}
|
||||
}
|
||||
return description;
|
||||
return std::move(description);
|
||||
}
|
||||
|
||||
Block FilterTransform::transformHeader(
|
||||
|
||||
@ -1,17 +1,20 @@
|
||||
<test>
|
||||
<settings>
|
||||
<max_insert_threads>1</max_insert_threads>
|
||||
<max_memory_usage>20G</max_memory_usage>
|
||||
</settings>
|
||||
|
||||
<create_query>
|
||||
CREATE TABLE optimized_select_final (d Date, key UInt64, value String)
|
||||
ENGINE = ReplacingMergeTree()
|
||||
PARTITION BY toYYYYMM(d) ORDER BY key
|
||||
</create_query>
|
||||
|
||||
<fill_query>INSERT INTO optimized_select_final SELECT toDate('2000-01-01'), number, randomPrintableASCII(1000) FROM numbers(5000000)</fill_query>
|
||||
<fill_query>INSERT INTO optimized_select_final SELECT toDate('2020-01-01'), number, randomPrintableASCII(1000) FROM numbers(5000000)</fill_query>
|
||||
<fill_query>INSERT INTO optimized_select_final SELECT toDate('2021-01-01'), number, randomPrintableASCII(1000) FROM numbers(5000000)</fill_query>
|
||||
<fill_query>INSERT INTO optimized_select_final SELECT toDate('2022-01-01'), number, randomPrintableASCII(1000) FROM numbers(5000000)</fill_query>
|
||||
<fill_query>INSERT INTO optimized_select_final SELECT toDate('2000-01-01'), 2*number, randomPrintableASCII(1000) FROM numbers(5000000)</fill_query>
|
||||
<fill_query>INSERT INTO optimized_select_final SELECT toDate('2020-01-01'), 2*number+1, randomPrintableASCII(1000) FROM numbers(5000000)</fill_query>
|
||||
|
||||
<query>SELECT * FROM optimized_select_final FINAL FORMAT Null SETTINGS max_threads = 8</query>
|
||||
<query>SELECT * FROM optimized_select_final FINAL WHERE key % 1000 = 0 FORMAT Null</query>
|
||||
<query>SELECT * FROM optimized_select_final FINAL WHERE key % 10 = 0 FORMAT Null</query>
|
||||
|
||||
<drop_query>DROP TABLE IF EXISTS optimized_select_final</drop_query>
|
||||
|
||||
|
||||
@ -9,51 +9,53 @@
|
||||
|
||||
<substitutions>
|
||||
<substitution>
|
||||
<name>collapsing</name>
|
||||
<name>replacing</name>
|
||||
<values>
|
||||
<value>collapsing_final_16p_ord</value>
|
||||
<value>collapsing_final_16p_rnd</value>
|
||||
<value>collapsing_final_16p_int_keys_ord</value>
|
||||
<value>collapsing_final_16p_int_keys_rnd</value>
|
||||
<value>collapsing_final_16p_str_keys_ord</value>
|
||||
<value>collapsing_final_16p_str_keys_rnd</value>
|
||||
<value>collapsing_final_1024p_ord</value>
|
||||
<value>collapsing_final_1024p_rnd</value>
|
||||
<value>collapsing_final_1p_ord</value>
|
||||
<value>replacing_final_16p_ord</value>
|
||||
<value>replacing_final_16p_rnd</value>
|
||||
<value>replacing_final_16p_int_keys_ord</value>
|
||||
<value>replacing_final_16p_int_keys_rnd</value>
|
||||
<value>replacing_final_16p_str_keys_ord</value>
|
||||
<value>replacing_final_16p_str_keys_rnd</value>
|
||||
<value>replacing_final_1024p_ord</value>
|
||||
<value>replacing_final_1024p_rnd</value>
|
||||
<value>replacing_final_1p_ord</value>
|
||||
</values>
|
||||
</substitution>
|
||||
</substitutions>
|
||||
|
||||
<create_query>create table collapsing_final_16p_ord (key1 UInt32, key2 String, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1, key2) partition by intDiv(key1, 8192 * 64) </create_query>
|
||||
<create_query>create table collapsing_final_16p_rnd (key1 UInt32, key2 String, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1, key2) partition by key1 % 16 </create_query>
|
||||
<create_query>create table collapsing_final_16p_int_keys_ord (key1 UInt32, key2 UInt32, key3 UInt32, key4 UInt32, key5 UInt32, key6 UInt32, key7 UInt32, key8 UInt32, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by intDiv(key1, 8192 * 64) </create_query>
|
||||
<create_query>create table collapsing_final_16p_int_keys_rnd (key1 UInt32, key2 UInt32, key3 UInt32, key4 UInt32, key5 UInt32, key6 UInt32, key7 UInt32, key8 UInt32, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by key1 % 16 </create_query>
|
||||
<create_query>create table collapsing_final_16p_str_keys_ord (key1 UInt32, key2 String, key3 String, key4 String, key5 String, key6 String, key7 String, key8 String, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by intDiv(key1, 8192 * 64) </create_query>
|
||||
<create_query>create table collapsing_final_16p_str_keys_rnd (key1 UInt32, key2 String, key3 String, key4 String, key5 String, key6 String, key7 String, key8 String, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by key1 % 16 </create_query>
|
||||
<create_query>create table collapsing_final_1024p_ord (key1 UInt32, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1) partition by intDiv(key1, 8192 * 2) </create_query>
|
||||
<create_query>create table collapsing_final_1024p_rnd (key1 UInt32, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1) partition by key1 % 1024 </create_query>
|
||||
<create_query>create table collapsing_final_1p_ord (key1 UInt64, key2 UInt64, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1, key2)</create_query>
|
||||
<create_query>create table replacing_final_16p_ord (key1 UInt32, key2 String, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2) partition by intDiv(key1, 8192 * 64) </create_query>
|
||||
<create_query>create table replacing_final_16p_rnd (key1 UInt32, key2 String, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2) partition by key1 % 16 </create_query>
|
||||
<create_query>create table replacing_final_16p_int_keys_ord (key1 UInt32, key2 UInt32, key3 UInt32, key4 UInt32, key5 UInt32, key6 UInt32, key7 UInt32, key8 UInt32, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by intDiv(key1, 8192 * 64) </create_query>
|
||||
<create_query>create table replacing_final_16p_int_keys_rnd (key1 UInt32, key2 UInt32, key3 UInt32, key4 UInt32, key5 UInt32, key6 UInt32, key7 UInt32, key8 UInt32, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by key1 % 16 </create_query>
|
||||
<create_query>create table replacing_final_16p_str_keys_ord (key1 UInt32, key2 String, key3 String, key4 String, key5 String, key6 String, key7 String, key8 String, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by intDiv(key1, 8192 * 64) </create_query>
|
||||
<create_query>create table replacing_final_16p_str_keys_rnd (key1 UInt32, key2 String, key3 String, key4 String, key5 String, key6 String, key7 String, key8 String, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by key1 % 16 </create_query>
|
||||
<create_query>create table replacing_final_1024p_ord (key1 UInt32, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1) partition by intDiv(key1, 8192 * 2) </create_query>
|
||||
<create_query>create table replacing_final_1024p_rnd (key1 UInt32, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1) partition by key1 % 1024 </create_query>
|
||||
<create_query>create table replacing_final_1p_ord (key1 UInt64, key2 UInt64, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2)</create_query>
|
||||
|
||||
<!-- 16 parts, 8192 * 1024 rows each -->
|
||||
<fill_query>insert into collapsing_final_16p_ord select number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into collapsing_final_16p_rnd select sipHash64(number), number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into collapsing_final_16p_int_keys_ord select number, number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into collapsing_final_16p_int_keys_rnd select sipHash64(number), number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into collapsing_final_16p_str_keys_ord select number, number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into collapsing_final_16p_str_keys_rnd select sipHash64(number), number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into replacing_final_16p_ord select number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into replacing_final_16p_rnd select sipHash64(number), number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into replacing_final_16p_int_keys_ord select number, number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into replacing_final_16p_int_keys_rnd select sipHash64(number), number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into replacing_final_16p_str_keys_ord select number, number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into replacing_final_16p_str_keys_rnd select sipHash64(number), number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
|
||||
<!-- 1024 parts, 8192 * 2 rows each -->
|
||||
<fill_query>insert into collapsing_final_1024p_ord select number, 1, number from numbers_mt(16777216) </fill_query>
|
||||
<fill_query>insert into collapsing_final_1024p_rnd select number, 1, number from numbers_mt(16777216) </fill_query>
|
||||
<fill_query>insert into replacing_final_1024p_ord select number, 1, number from numbers_mt(16777216) </fill_query>
|
||||
<fill_query>insert into replacing_final_1024p_rnd select number, 1, number from numbers_mt(16777216) </fill_query>
|
||||
|
||||
<!-- 1 big part of 5e7 rows -->
|
||||
<fill_query>insert into collapsing_final_1p_ord select number, number + 1, 1, number from numbers_mt(5e7)</fill_query>
|
||||
<fill_query>insert into replacing_final_1p_ord select number, number + 1, 1, number from numbers_mt(5e7)</fill_query>
|
||||
|
||||
<fill_query>optimize table {collapsing} final</fill_query>
|
||||
<fill_query>optimize table {replacing} final</fill_query>
|
||||
|
||||
<query>SELECT count() FROM {collapsing} final</query>
|
||||
<query>SELECT sum(s) FROM {collapsing} final group by key1 limit 10</query>
|
||||
<query>SELECT sum(s) FROM {collapsing} final group by key1 % 8192 limit 10</query>
|
||||
<query>SELECT count() FROM {replacing} final</query>
|
||||
<query>SELECT sum(s) FROM {replacing} final group by key1 limit 10</query>
|
||||
<query>SELECT sum(s) FROM {replacing} final group by key1 % 8192 limit 10</query>
|
||||
|
||||
<drop_query>DROP TABLE IF EXISTS {collapsing}</drop_query>
|
||||
<drop_query>DROP TABLE IF EXISTS {replacing}</drop_query>
|
||||
|
||||
<drop_query>DROP TABLE IF EXISTS {replacing}</drop_query>
|
||||
</test>
|
||||
|
||||
@ -1,61 +0,0 @@
|
||||
<test>
|
||||
|
||||
<settings>
|
||||
<max_partitions_per_insert_block>1024</max_partitions_per_insert_block>
|
||||
<max_insert_threads>1</max_insert_threads>
|
||||
<max_memory_usage>20G</max_memory_usage>
|
||||
<allow_deprecated_syntax_for_merge_tree>1</allow_deprecated_syntax_for_merge_tree>
|
||||
</settings>
|
||||
|
||||
<substitutions>
|
||||
<substitution>
|
||||
<name>replacing</name>
|
||||
<values>
|
||||
<value>replacing_final_16p_ord</value>
|
||||
<value>replacing_final_16p_rnd</value>
|
||||
<value>replacing_final_16p_int_keys_ord</value>
|
||||
<value>replacing_final_16p_int_keys_rnd</value>
|
||||
<value>replacing_final_16p_str_keys_ord</value>
|
||||
<value>replacing_final_16p_str_keys_rnd</value>
|
||||
<value>replacing_final_1024p_ord</value>
|
||||
<value>replacing_final_1024p_rnd</value>
|
||||
<value>replacing_final_1p_ord</value>
|
||||
</values>
|
||||
</substitution>
|
||||
</substitutions>
|
||||
|
||||
<create_query>create table replacing_final_16p_ord (key1 UInt32, key2 String, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2) partition by intDiv(key1, 8192 * 64) </create_query>
|
||||
<create_query>create table replacing_final_16p_rnd (key1 UInt32, key2 String, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2) partition by key1 % 16 </create_query>
|
||||
<create_query>create table replacing_final_16p_int_keys_ord (key1 UInt32, key2 UInt32, key3 UInt32, key4 UInt32, key5 UInt32, key6 UInt32, key7 UInt32, key8 UInt32, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by intDiv(key1, 8192 * 64) </create_query>
|
||||
<create_query>create table replacing_final_16p_int_keys_rnd (key1 UInt32, key2 UInt32, key3 UInt32, key4 UInt32, key5 UInt32, key6 UInt32, key7 UInt32, key8 UInt32, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by key1 % 16 </create_query>
|
||||
<create_query>create table replacing_final_16p_str_keys_ord (key1 UInt32, key2 String, key3 String, key4 String, key5 String, key6 String, key7 String, key8 String, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by intDiv(key1, 8192 * 64) </create_query>
|
||||
<create_query>create table replacing_final_16p_str_keys_rnd (key1 UInt32, key2 String, key3 String, key4 String, key5 String, key6 String, key7 String, key8 String, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by key1 % 16 </create_query>
|
||||
<create_query>create table replacing_final_1024p_ord (key1 UInt32, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1) partition by intDiv(key1, 8192 * 2) </create_query>
|
||||
<create_query>create table replacing_final_1024p_rnd (key1 UInt32, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1) partition by key1 % 1024 </create_query>
|
||||
<create_query>create table replacing_final_1p_ord (key1 UInt64, key2 UInt64, ver Int8, s UInt64) engine = ReplacingMergeTree(ver) order by (key1, key2)</create_query>
|
||||
|
||||
<!-- 16 parts, 8192 * 1024 rows each -->
|
||||
<fill_query>insert into replacing_final_16p_ord select number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into replacing_final_16p_rnd select sipHash64(number), number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into replacing_final_16p_int_keys_ord select number, number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into replacing_final_16p_int_keys_rnd select sipHash64(number), number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into replacing_final_16p_str_keys_ord select number, number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
<fill_query>insert into replacing_final_16p_str_keys_rnd select sipHash64(number), number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) </fill_query>
|
||||
|
||||
<!-- 1024 parts, 8192 * 2 rows each -->
|
||||
<fill_query>insert into replacing_final_1024p_ord select number, 1, number from numbers_mt(16777216) </fill_query>
|
||||
<fill_query>insert into replacing_final_1024p_rnd select number, 1, number from numbers_mt(16777216) </fill_query>
|
||||
|
||||
<!-- 1 big part of 5e7 rows -->
|
||||
<fill_query>insert into replacing_final_1p_ord select number, number + 1, 1, number from numbers_mt(5e7)</fill_query>
|
||||
|
||||
<fill_query>optimize table {replacing} final</fill_query>
|
||||
|
||||
<query>SELECT * FROM {replacing} final SETTINGS max_threads = 4 FORMAT Null</query>
|
||||
<query>SELECT * FROM {replacing} final FORMAT Null</query>
|
||||
|
||||
<query>SELECT * FROM {replacing} final WHERE s % 100 = 0 FORMAT Null SETTINGS max_threads = 4</query>
|
||||
<query>SELECT * FROM {replacing} final WHERE s % 100 = 0 FORMAT Null</query>
|
||||
|
||||
<drop_query>DROP TABLE IF EXISTS {replacing}</drop_query>
|
||||
</test>
|
||||
Loading…
Reference in New Issue
Block a user