From 78c4b9f2bca3fb3fb26fecaa17b9b675a84f43c8 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 23 Dec 2020 03:11:32 +0300 Subject: [PATCH] do not allow merge from wide to compact parts --- .../MergeTree/MergeTreeDataMergerMutator.cpp | 5 ++- .../MergeTree/MergeTreeDataPartType.h | 12 ++++++- ...01606_merge_from_wide_to_compact.reference | 9 +++++ .../01606_merge_from_wide_to_compact.sql | 36 +++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01606_merge_from_wide_to_compact.reference create mode 100644 tests/queries/0_stateless/01606_merge_from_wide_to_compact.sql diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 2365ef141b6..3f63d34b1c6 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -79,13 +79,16 @@ void FutureMergedMutatedPart::assign(MergeTreeData::DataPartsVector parts_) size_t sum_rows = 0; size_t sum_bytes_uncompressed = 0; + MergeTreeDataPartType future_part_type; for (const auto & part : parts_) { sum_rows += part->rows_count; sum_bytes_uncompressed += part->getTotalColumnsSize().data_uncompressed; + future_part_type = std::min(future_part_type, part->getType()); } - auto future_part_type = parts_.front()->storage.choosePartTypeOnDisk(sum_bytes_uncompressed, sum_rows); + auto chosen_type = parts_.front()->storage.choosePartTypeOnDisk(sum_bytes_uncompressed, sum_rows); + future_part_type = std::min(future_part_type, chosen_type); assign(std::move(parts_), future_part_type); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartType.h b/src/Storages/MergeTree/MergeTreeDataPartType.h index 44e170141f6..6aec9c97f44 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartType.h +++ b/src/Storages/MergeTree/MergeTreeDataPartType.h @@ -18,7 +18,7 @@ public: /// Data of all columns is stored in one file. Marks are also stored in single file. COMPACT, - /// Format with buffering data in RAM. Not implemented yet. + /// Format with buffering data in RAM. IN_MEMORY, UNKNOWN, @@ -37,6 +37,16 @@ public: return !(*this == other); } + bool operator<(const MergeTreeDataPartType & other) const + { + return value < other.value; + } + + bool operator>(const MergeTreeDataPartType & other) const + { + return value > other.value; + } + void fromString(const String & str); String toString() const; diff --git a/tests/queries/0_stateless/01606_merge_from_wide_to_compact.reference b/tests/queries/0_stateless/01606_merge_from_wide_to_compact.reference new file mode 100644 index 00000000000..d1d987f7ad0 --- /dev/null +++ b/tests/queries/0_stateless/01606_merge_from_wide_to_compact.reference @@ -0,0 +1,9 @@ +all_1_1_0 Wide +all_2_2_0 Wide +all_3_3_0 Wide +all_1_3_1 Wide +300000 +all_1_3_1 Wide +all_4_4_0 Compact +all_1_4_2 Wide +400000 diff --git a/tests/queries/0_stateless/01606_merge_from_wide_to_compact.sql b/tests/queries/0_stateless/01606_merge_from_wide_to_compact.sql new file mode 100644 index 00000000000..15596d3f6db --- /dev/null +++ b/tests/queries/0_stateless/01606_merge_from_wide_to_compact.sql @@ -0,0 +1,36 @@ +DROP TABLE IF EXISTS wide_to_comp; + +CREATE TABLE wide_to_comp (a Int, b Int, c Int) + ENGINE = MergeTree ORDER BY a + settings vertical_merge_algorithm_min_rows_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1, + min_bytes_for_wide_part = 0; + +SYSTEM STOP merges wide_to_comp; + +INSERT INTO wide_to_comp SELECT number, number, number FROM numbers(100000); +INSERT INTO wide_to_comp SELECT number, number, number FROM numbers(100000); +INSERT INTO wide_to_comp SELECT number, number, number FROM numbers(100000); + +SELECT name, part_type FROM system.parts WHERE table = 'wide_to_comp' AND database = currentDatabase() AND active ORDER BY name; + +ALTER TABLE wide_to_comp MODIFY setting min_rows_for_wide_part = 10000000; +SYSTEM START merges wide_to_comp; +OPTIMIZE TABLE wide_to_comp FINAL; + +SELECT name, part_type FROM system.parts WHERE table = 'wide_to_comp' AND database = currentDatabase() AND active ORDER BY name; +SELECT count() FROM wide_to_comp WHERE not ignore(*); + +SYSTEM STOP merges wide_to_comp; +INSERT INTO wide_to_comp SELECT number, number, number FROM numbers(100000); + +SELECT name, part_type FROM system.parts WHERE table = 'wide_to_comp' AND database = currentDatabase() AND active ORDER BY name; + +ALTER TABLE wide_to_comp MODIFY setting min_rows_for_wide_part = 10000000; +SYSTEM START merges wide_to_comp; +OPTIMIZE TABLE wide_to_comp FINAL; + +SELECT name, part_type FROM system.parts WHERE table = 'wide_to_comp' AND database = currentDatabase() AND active ORDER BY name; +SELECT count() FROM wide_to_comp WHERE not ignore(*); + +DROP TABLE wide_to_comp;