From 53b8ff42c2daac587b7bdef50d376dd216243690 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Tue, 17 Nov 2020 19:56:36 +0300 Subject: [PATCH] Add test and set setting to 0 in some other tests --- .../MergeTree/MergeTreeDataWriter.cpp | 20 +++++++---- .../00083_create_merge_tree_zookeeper.sql | 2 ++ .../00327_summing_composite_nested.sql | 2 ++ .../00443_optimize_final_vertical_merge.sh | 6 ++-- ...ed_storage_definition_syntax_zookeeper.sql | 2 ++ .../00564_versioned_collapsing_merge_tree.sql | 2 ++ ...77_replacing_merge_tree_vertical_merge.sql | 2 ++ .../0_stateless/00616_final_single_part.sql | 2 ++ ...00660_optimize_final_without_partition.sql | 2 ++ ...replicated_without_partition_zookeeper.sql | 2 ++ .../00754_alter_modify_order_by.sql | 1 + ...r_modify_order_by_replicated_zookeeper.sql | 2 ++ ...030_incorrect_count_summing_merge_tree.sql | 2 ++ ...01285_data_skip_index_over_aggregation.sql | 2 ++ .../0_stateless/01323_add_scalars_in_time.sql | 2 ++ ...mming_merge_tree_exclude_partition_key.sql | 2 ++ .../01560_optimize_on_insert.reference | 13 +++++++ .../0_stateless/01560_optimize_on_insert.sql | 35 +++++++++++++++++++ 18 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/01560_optimize_on_insert.reference create mode 100644 tests/queries/0_stateless/01560_optimize_on_insert.sql diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 65846c10ff0..f338b4b7048 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -204,6 +204,9 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(const Block & block Block MergeTreeDataWriter::mergeBlock(const Block & block, SortDescription sort_description, Names & partition_key_columns, IColumn::Permutation ** permutation) { + + LOG_DEBUG(log, "Apply merging algorithm on inserted data"); + size_t block_size = block.rows(); auto get_merging_algorithm = [&]() -> std::shared_ptr @@ -215,23 +218,23 @@ Block MergeTreeDataWriter::mergeBlock(const Block & block, SortDescription sort_ return nullptr; case MergeTreeData::MergingParams::Replacing: return std::make_shared( - block, 1, sort_description, data.merging_params.version_column, block_size); + block, 1, sort_description, data.merging_params.version_column, block_size + 1); case MergeTreeData::MergingParams::Collapsing: return std::make_shared( block, 1, sort_description, data.merging_params.sign_column, - false, block_size, &Poco::Logger::get("MergeTreeBlockOutputStream")); + false, block_size + 1, &Poco::Logger::get("MergeTreeBlockOutputStream")); case MergeTreeData::MergingParams::Summing: return std::make_shared( block, 1, sort_description, data.merging_params.columns_to_sum, - partition_key_columns, block_size); + partition_key_columns, block_size + 1); case MergeTreeData::MergingParams::Aggregating: - return std::make_shared(block, 1, sort_description, block_size); + return std::make_shared(block, 1, sort_description, block_size + 1); case MergeTreeData::MergingParams::VersionedCollapsing: return std::make_shared( - block, 1, sort_description, data.merging_params.sign_column, block_size); + block, 1, sort_description, data.merging_params.sign_column, block_size + 1); case MergeTreeData::MergingParams::Graphite: return std::make_shared( - block, 1, sort_description, block_size, data.merging_params.graphite_params, time(nullptr)); + block, 1, sort_description, block_size + 1, data.merging_params.graphite_params, time(nullptr)); } __builtin_unreachable(); @@ -243,6 +246,8 @@ Block MergeTreeDataWriter::mergeBlock(const Block & block, SortDescription sort_ Chunk chunk(block.getColumns(), block_size); + LOG_DEBUG(log, "chunk size before merge {}, block rows {}", chunk.getNumRows(), block_size); + IMergingAlgorithm::Input input; input.set(std::move(chunk)); input.permutation = *permutation; @@ -255,6 +260,9 @@ Block MergeTreeDataWriter::mergeBlock(const Block & block, SortDescription sort_ while (!status.is_finished) status = merging_algorithm->merge(); + + LOG_DEBUG(log, "chunk size after merge {}", status.chunk.getNumRows()); + /// Merged Block is sorted and we don't need to use permutation anymore *permutation = nullptr; diff --git a/tests/queries/0_stateless/00083_create_merge_tree_zookeeper.sql b/tests/queries/0_stateless/00083_create_merge_tree_zookeeper.sql index 120d599bd35..998a4517163 100644 --- a/tests/queries/0_stateless/00083_create_merge_tree_zookeeper.sql +++ b/tests/queries/0_stateless/00083_create_merge_tree_zookeeper.sql @@ -1,3 +1,5 @@ +SET optimize_on_insert = 0; + DROP TABLE IF EXISTS merge_tree; DROP TABLE IF EXISTS collapsing_merge_tree; DROP TABLE IF EXISTS versioned_collapsing_merge_tree; diff --git a/tests/queries/0_stateless/00327_summing_composite_nested.sql b/tests/queries/0_stateless/00327_summing_composite_nested.sql index b61bc71b892..9be21e87abf 100644 --- a/tests/queries/0_stateless/00327_summing_composite_nested.sql +++ b/tests/queries/0_stateless/00327_summing_composite_nested.sql @@ -1,3 +1,5 @@ +SET optimize_on_insert = 0; + DROP TABLE IF EXISTS summing_composite_key; CREATE TABLE summing_composite_key (d Date, k UInt64, FirstMap Nested(k1 UInt32, k2ID Int8, s Float64), SecondMap Nested(k1ID UInt64, k2Key String, k3Type Int32, s Int64)) ENGINE = SummingMergeTree(d, k, 1); diff --git a/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh b/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh index 1ea7762b813..1bd73cc1f3e 100755 --- a/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh +++ b/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh @@ -43,13 +43,13 @@ $CLICKHOUSE_CLIENT -q "INSERT INTO $name (date, Sign, ki) SELECT toDate(0) AS date, toInt8(1) AS Sign, toUInt64(0) AS ki -FROM system.numbers LIMIT 9000" +FROM system.numbers LIMIT 9000" --server_logs_file=/dev/null $CLICKHOUSE_CLIENT -q "INSERT INTO $name (date, Sign, ki) SELECT toDate(0) AS date, toInt8(1) AS Sign, number AS ki -FROM system.numbers LIMIT 9000, 9000" +FROM system.numbers LIMIT 9000, 9000" --server_logs_file=/dev/null $CLICKHOUSE_CLIENT -q "INSERT INTO $name SELECT toDate(0) AS date, @@ -68,7 +68,7 @@ number AS di09, number AS di10, [number, number+1] AS \`n.i\`, [hex(number), hex(number+1)] AS \`n.s\` -FROM system.numbers LIMIT $res_rows" +FROM system.numbers LIMIT $res_rows" --server_logs_file=/dev/null while [[ $(get_num_parts) -ne 1 ]] ; do $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE $name PARTITION 197001" --server_logs_file=/dev/null; done diff --git a/tests/queries/0_stateless/00509_extended_storage_definition_syntax_zookeeper.sql b/tests/queries/0_stateless/00509_extended_storage_definition_syntax_zookeeper.sql index 4a3dd2981cd..cec6088bd59 100644 --- a/tests/queries/0_stateless/00509_extended_storage_definition_syntax_zookeeper.sql +++ b/tests/queries/0_stateless/00509_extended_storage_definition_syntax_zookeeper.sql @@ -1,3 +1,5 @@ +SET optimize_on_insert = 0; + SELECT '*** Replicated with sampling ***'; DROP TABLE IF EXISTS test.replicated_with_sampling; diff --git a/tests/queries/0_stateless/00564_versioned_collapsing_merge_tree.sql b/tests/queries/0_stateless/00564_versioned_collapsing_merge_tree.sql index b7824e7efdc..634b9781c7a 100644 --- a/tests/queries/0_stateless/00564_versioned_collapsing_merge_tree.sql +++ b/tests/queries/0_stateless/00564_versioned_collapsing_merge_tree.sql @@ -1,3 +1,5 @@ +set optimize_on_insert = 0; + drop table if exists mult_tab; create table mult_tab (date Date, value String, version UInt64, sign Int8) engine = VersionedCollapsingMergeTree(date, (date), 8192, sign, version); insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; diff --git a/tests/queries/0_stateless/00577_replacing_merge_tree_vertical_merge.sql b/tests/queries/0_stateless/00577_replacing_merge_tree_vertical_merge.sql index 1bfbdaf75c5..8c51a6f34da 100644 --- a/tests/queries/0_stateless/00577_replacing_merge_tree_vertical_merge.sql +++ b/tests/queries/0_stateless/00577_replacing_merge_tree_vertical_merge.sql @@ -1,3 +1,5 @@ +set optimize_on_insert = 0; + drop table if exists tab_00577; create table tab_00577 (date Date, version UInt64, val UInt64) engine = ReplacingMergeTree(version) partition by date order by date settings enable_vertical_merge_algorithm = 1, vertical_merge_algorithm_min_rows_to_activate = 1, vertical_merge_algorithm_min_columns_to_activate = 0; insert into tab_00577 values ('2018-01-01', 2, 2), ('2018-01-01', 1, 1); diff --git a/tests/queries/0_stateless/00616_final_single_part.sql b/tests/queries/0_stateless/00616_final_single_part.sql index df65123e29b..6618d0b1252 100644 --- a/tests/queries/0_stateless/00616_final_single_part.sql +++ b/tests/queries/0_stateless/00616_final_single_part.sql @@ -1,3 +1,5 @@ +SET optimize_on_insert = 0; + DROP TABLE IF EXISTS test_00616; DROP TABLE IF EXISTS replacing_00616; diff --git a/tests/queries/0_stateless/00660_optimize_final_without_partition.sql b/tests/queries/0_stateless/00660_optimize_final_without_partition.sql index 8c1f2ebd361..6545ad6e85b 100644 --- a/tests/queries/0_stateless/00660_optimize_final_without_partition.sql +++ b/tests/queries/0_stateless/00660_optimize_final_without_partition.sql @@ -1,3 +1,5 @@ +SET optimize_on_insert = 0; + DROP TABLE IF EXISTS partitioned_by_tuple; CREATE TABLE partitioned_by_tuple (d Date, x UInt8, w String, y UInt8) ENGINE SummingMergeTree (y) PARTITION BY (d, x) ORDER BY (d, x, w); diff --git a/tests/queries/0_stateless/00661_optimize_final_replicated_without_partition_zookeeper.sql b/tests/queries/0_stateless/00661_optimize_final_replicated_without_partition_zookeeper.sql index 77747bc0383..033202e04aa 100644 --- a/tests/queries/0_stateless/00661_optimize_final_replicated_without_partition_zookeeper.sql +++ b/tests/queries/0_stateless/00661_optimize_final_replicated_without_partition_zookeeper.sql @@ -1,3 +1,5 @@ +SET optimize_on_insert = 0; + DROP TABLE IF EXISTS partitioned_by_tuple_replica1_00661; DROP TABLE IF EXISTS partitioned_by_tuple_replica2_00661; CREATE TABLE partitioned_by_tuple_replica1_00661(d Date, x UInt8, w String, y UInt8) ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/test/partitioned_by_tuple_00661', '1') PARTITION BY (d, x) ORDER BY (d, x, w); diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by.sql b/tests/queries/0_stateless/00754_alter_modify_order_by.sql index f8c584ed052..a09d824c928 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by.sql +++ b/tests/queries/0_stateless/00754_alter_modify_order_by.sql @@ -1,4 +1,5 @@ SET send_logs_level = 'fatal'; +SET optimize_on_insert = 0; DROP TABLE IF EXISTS old_style; CREATE TABLE old_style(d Date, x UInt32) ENGINE MergeTree(d, x, 8192); diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql index 0f861749537..b3b746c3b43 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql +++ b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql @@ -1,3 +1,5 @@ +SET optimze_on_insert = 0; + SET send_logs_level = 'fatal'; DROP TABLE IF EXISTS old_style; diff --git a/tests/queries/0_stateless/01030_incorrect_count_summing_merge_tree.sql b/tests/queries/0_stateless/01030_incorrect_count_summing_merge_tree.sql index 0b5845d3b04..90b1660e546 100644 --- a/tests/queries/0_stateless/01030_incorrect_count_summing_merge_tree.sql +++ b/tests/queries/0_stateless/01030_incorrect_count_summing_merge_tree.sql @@ -1,3 +1,5 @@ +SET optimize_on_insert = 0; + select '-- SummingMergeTree with Nullable column without duplicates.'; drop table if exists tst; diff --git a/tests/queries/0_stateless/01285_data_skip_index_over_aggregation.sql b/tests/queries/0_stateless/01285_data_skip_index_over_aggregation.sql index 110c5b65cab..812da74ce75 100644 --- a/tests/queries/0_stateless/01285_data_skip_index_over_aggregation.sql +++ b/tests/queries/0_stateless/01285_data_skip_index_over_aggregation.sql @@ -1,3 +1,5 @@ +SET optimize_on_insert = 0; + DROP TABLE IF EXISTS data_01285; SET max_threads=1; diff --git a/tests/queries/0_stateless/01323_add_scalars_in_time.sql b/tests/queries/0_stateless/01323_add_scalars_in_time.sql index 2d7cf270017..2ee5603f760 100644 --- a/tests/queries/0_stateless/01323_add_scalars_in_time.sql +++ b/tests/queries/0_stateless/01323_add_scalars_in_time.sql @@ -1,3 +1,5 @@ +SET optimize_on_insert = 0; + DROP TABLE IF EXISTS tags; CREATE TABLE tags ( diff --git a/tests/queries/0_stateless/01373_summing_merge_tree_exclude_partition_key.sql b/tests/queries/0_stateless/01373_summing_merge_tree_exclude_partition_key.sql index 790fbca6b73..c5a874efe09 100644 --- a/tests/queries/0_stateless/01373_summing_merge_tree_exclude_partition_key.sql +++ b/tests/queries/0_stateless/01373_summing_merge_tree_exclude_partition_key.sql @@ -1,3 +1,5 @@ +SET optimize_on_insert = 0; + DROP TABLE IF EXISTS tt_01373; CREATE TABLE tt_01373 diff --git a/tests/queries/0_stateless/01560_optimize_on_insert.reference b/tests/queries/0_stateless/01560_optimize_on_insert.reference new file mode 100644 index 00000000000..b1f46ac9068 --- /dev/null +++ b/tests/queries/0_stateless/01560_optimize_on_insert.reference @@ -0,0 +1,13 @@ +Replacing Merge Tree +1 2020-01-02 00:00:00 +1 2020-01-01 00:00:00 +Collapsing Merge Tree +1 1 2020-01-01 00:00:00 +Versioned Collapsing Merge Tree +1 1 2 2020-01-01 00:00:00 +Summing Merge Tree +1 6 2020-01-02 00:00:00 +1 6 2020-01-01 00:00:00 +Aggregating Merge Tree +1 5 2020-01-02 00:00:00 +1 5 2020-01-01 00:00:00 diff --git a/tests/queries/0_stateless/01560_optimize_on_insert.sql b/tests/queries/0_stateless/01560_optimize_on_insert.sql new file mode 100644 index 00000000000..6cfdda1205b --- /dev/null +++ b/tests/queries/0_stateless/01560_optimize_on_insert.sql @@ -0,0 +1,35 @@ +SELECT 'Replacing Merge Tree'; +DROP TABLE IF EXISTS replacing_merge_tree; +CREATE TABLE replacing_merge_tree (key UInt32, date Datetime) ENGINE=ReplacingMergeTree() PARTITION BY date ORDER BY key; +INSERT INTO replacing_merge_tree VALUES (1, '2020-01-01'), (1, '2020-01-02'), (1, '2020-01-01'), (1, '2020-01-02'); +SELECT * FROM replacing_merge_tree; +DROP TABLE replacing_merge_tree; + +SELECT 'Collapsing Merge Tree'; +DROP TABLE IF EXISTS collapsing_merge_tree; +CREATE TABLE collapsing_merge_tree (key UInt32, sign Int8, date Datetime) ENGINE=CollapsingMergeTree(sign) PARTITION BY date ORDER BY key; +INSERT INTO collapsing_merge_tree VALUES (1, 1, '2020-01-01'), (1, 1, '2020-01-02'), (1, -1, '2020-01-01'), (1, -1, '2020-01-02'), (1, 1, '2020-01-01'); +SELECT * FROM collapsing_merge_tree; +DROP TABLE collapsing_merge_tree; + +SELECT 'Versioned Collapsing Merge Tree'; +DROP TABLE IF EXISTS versioned_collapsing_merge_tree; +CREATE TABLE versioned_collapsing_merge_tree (key UInt32, sign Int8, version Int32, date Datetime) ENGINE=VersionedCollapsingMergeTree(sign, version) PARTITION BY date ORDER BY (key, version); +INSERT INTO versioned_collapsing_merge_tree VALUES (1, 1, 1, '2020-01-01'), (1, -1, 1, '2020-01-01'), (1, 1, 2, '2020-01-01'); +SELECT * FROM versioned_collapsing_merge_tree; +DROP TABLE versioned_collapsing_merge_tree; + +SELECT 'Summing Merge Tree'; +DROP TABLE IF EXISTS summing_merge_tree; +CREATE TABLE summing_merge_tree (key UInt32, val UInt32, date Datetime) ENGINE=SummingMergeTree(val) PARTITION BY date ORDER BY key; +INSERT INTO summing_merge_tree VALUES (1, 1, '2020-01-01'), (1, 1, '2020-01-02'), (1, 5, '2020-01-01'), (1, 5, '2020-01-02'); +SELECT * FROM summing_merge_tree; +DROP TABLE summing_merge_tree; + +SELECT 'Aggregating Merge Tree'; +DROP TABLE IF EXISTS aggregating_merge_tree; +CREATE TABLE aggregating_merge_tree (key UInt32, val SimpleAggregateFunction(max, UInt32), date Datetime) ENGINE=AggregatingMergeTree() PARTITION BY date ORDER BY key; +INSERT INTO aggregating_merge_tree VALUES (1, 1, '2020-01-01'), (1, 1, '2020-01-02'), (1, 5, '2020-01-01'), (1, 5, '2020-01-02'); +SELECT * FROM aggregating_merge_tree; +DROP TABLE aggregating_merge_tree; +