From 0606b7e4df76dbbf8530a43e7401a50ef5f40d95 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Apr 2020 21:24:11 +0300 Subject: [PATCH] Renames for compact parts --- dbms/Storages/MergeTree/MergeTreeData.cpp | 5 --- .../MergeTree/MergeTreeDataMergerMutator.cpp | 17 +++++++++- .../MergeTree/MergeTreeReaderCompact.cpp | 23 +++++++++++-- .../MergeTree/StorageFromMergeTreeDataPart.h | 2 -- dbms/Storages/StorageMergeTree.cpp | 11 +++++++ dbms/Storages/StorageMergeTree.h | 2 +- .../01213_alter_rename_compact_part.reference | 7 ++++ .../01213_alter_rename_compact_part.sql | 33 +++++++++++++++++++ 8 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 tests/queries/0_stateless/01213_alter_rename_compact_part.reference create mode 100644 tests/queries/0_stateless/01213_alter_rename_compact_part.sql diff --git a/dbms/Storages/MergeTree/MergeTreeData.cpp b/dbms/Storages/MergeTree/MergeTreeData.cpp index bf0eedc4264..2e00dc864f8 100644 --- a/dbms/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/Storages/MergeTree/MergeTreeData.cpp @@ -3608,13 +3608,8 @@ MergeTreeData::AlterConversions MergeTreeData::getAlterConversionsForPart(const AlterConversions result{}; for (const auto & command : commands) - { if (command.type == MutationCommand::Type::RENAME_COLUMN) - { result.rename_map[command.rename_to] = command.column_name; - LOG_DEBUG(log, "Add to rename map:" << command.column_name); - } - } return result; } diff --git a/dbms/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index d1dd2861202..2b732d879b0 100644 --- a/dbms/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1261,6 +1261,20 @@ void MergeTreeDataMergerMutator::splitMutationCommands( { removed_columns_from_compact_part.emplace(command.column_name); } + else if (command.type == MutationCommand::Type::RENAME_COLUMN) + { + if (is_compact_part) + { + for_interpreter.push_back( + { + .type = MutationCommand::Type::READ_COLUMN, + .column_name = command.rename_to, + }); + already_changed_columns.emplace(command.column_name); + } + else + for_file_renames.push_back(command); + } else { for_file_renames.push_back(command); @@ -1273,7 +1287,8 @@ void MergeTreeDataMergerMutator::splitMutationCommands( /// we just don't read dropped columns for (const auto & column : part->getColumns()) { - if (!removed_columns_from_compact_part.count(column.name) && !already_changed_columns.count(column.name)) + if (!removed_columns_from_compact_part.count(column.name) + && !already_changed_columns.count(column.name)) { for_interpreter.emplace_back(MutationCommand { diff --git a/dbms/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/Storages/MergeTree/MergeTreeReaderCompact.cpp index 496371b6e4b..a59a87386a5 100644 --- a/dbms/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -81,14 +81,23 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( const auto & [name, type] = *name_and_type; auto position = data_part->getColumnPosition(name); - /// If array of Nested column is missing in part, - /// we have to read it's offsets if they exists. + if (!position) + { + auto renamed_it = alter_conversions.rename_map.find(name); + + if (renamed_it != alter_conversions.rename_map.end()) + position = data_part->getColumnPosition(renamed_it->second); + } + if (!position && typeid_cast(type.get())) { + /// If array of Nested column is missing in part, + /// we have to read it's offsets if they exists. position = findColumnForOffsets(name); read_only_offsets[i] = (position != std::nullopt); } + column_positions[i] = std::move(position); } @@ -125,7 +134,15 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, if (!res_columns[pos]) continue; - const auto & [name, type] = *name_and_type; + auto [name, type] = *name_and_type; + + if (alter_conversions.rename_map.count(name)) + { + String original_name = alter_conversions.rename_map[name]; + if (!data_part->getColumnPosition(name) && data_part->getColumnPosition(original_name)) + name = original_name; + } + auto & column = mutable_columns[pos]; try diff --git a/dbms/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/dbms/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 0e93301e124..4d799522920 100644 --- a/dbms/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/dbms/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -52,12 +52,10 @@ public: return part->storage.getInMemoryMetadata(); } - bool hasSortingKey() const { return part->storage.hasSortingKey(); } Names getSortingKeyColumns() const override { return part->storage.getSortingKeyColumns(); } - protected: StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) : IStorage(getIDFromPart(part_), part_->storage.getVirtuals()) diff --git a/dbms/Storages/StorageMergeTree.cpp b/dbms/Storages/StorageMergeTree.cpp index a0ca6964cbf..5742b3a811b 100644 --- a/dbms/Storages/StorageMergeTree.cpp +++ b/dbms/Storages/StorageMergeTree.cpp @@ -1260,4 +1260,15 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, const Context & c return results; } + +MutationCommands StorageMergeTree::getFirtsAlterMutationCommandsForPart(const DataPartPtr & part) const +{ + std::lock_guard lock(currently_processing_in_background_mutex); + + auto it = current_mutations_by_version.upper_bound(part->info.getDataVersion()); + if (it == current_mutations_by_version.end()) + return {}; + return it->second.commands; +} + } diff --git a/dbms/Storages/StorageMergeTree.h b/dbms/Storages/StorageMergeTree.h index d6b8f98bff8..44ebce03802 100644 --- a/dbms/Storages/StorageMergeTree.h +++ b/dbms/Storages/StorageMergeTree.h @@ -166,7 +166,7 @@ protected: std::unique_ptr settings_, bool has_force_restore_data_flag); - MutationCommands getFirtsAlterMutationCommandsForPart(const DataPartPtr & /* part */) const override { return {}; } + MutationCommands getFirtsAlterMutationCommandsForPart(const DataPartPtr & part) const override; }; } diff --git a/tests/queries/0_stateless/01213_alter_rename_compact_part.reference b/tests/queries/0_stateless/01213_alter_rename_compact_part.reference new file mode 100644 index 00000000000..a5e642f56ad --- /dev/null +++ b/tests/queries/0_stateless/01213_alter_rename_compact_part.reference @@ -0,0 +1,7 @@ +1 +1 +date key renamed_value1 value2 value3 +2019-10-02 1 1 1 1 +7 7 +date key renamed_value1 renamed_value2 renamed_value3 +2019-10-02 7 7 7 7 diff --git a/tests/queries/0_stateless/01213_alter_rename_compact_part.sql b/tests/queries/0_stateless/01213_alter_rename_compact_part.sql new file mode 100644 index 00000000000..07188ece519 --- /dev/null +++ b/tests/queries/0_stateless/01213_alter_rename_compact_part.sql @@ -0,0 +1,33 @@ +DROP TABLE IF EXISTS table_with_compact_parts; + +CREATE TABLE table_with_compact_parts +( + date Date, + key UInt64, + value1 String, + value2 String, + value3 String +) +ENGINE = MergeTree() +PARTITION BY date +ORDER BY key +settings index_granularity = 8, +min_rows_for_wide_part = 10; + +INSERT INTO table_with_compact_parts SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number), toString(number) from numbers(9); + +SELECT value1 FROM table_with_compact_parts WHERE key = 1; + +ALTER TABLE table_with_compact_parts RENAME COLUMN value1 to renamed_value1; + +SELECT renamed_value1 FROM table_with_compact_parts WHERE key = 1; + +SELECT * FROM table_with_compact_parts WHERE key = 1 FORMAT TSVWithNames; + +ALTER TABLE table_with_compact_parts RENAME COLUMN value2 TO renamed_value2, RENAME COLUMN value3 TO renamed_value3; + +SELECT renamed_value2, renamed_value3 FROM table_with_compact_parts WHERE key = 7; + +SELECT * FROM table_with_compact_parts WHERE key = 7 FORMAT TSVWithNames; + +DROP TABLE IF EXISTS table_with_compact_parts;