From 4159fa0382830aee5e934da2d48a267363807c45 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 24 Mar 2020 20:05:38 +0300 Subject: [PATCH] Preparation for rename --- dbms/src/Access/AccessType.h | 2 ++ .../Interpreters/InterpreterAlterQuery.cpp | 5 ++++ dbms/src/Parsers/ASTAlterQuery.cpp | 14 +++++++++++ dbms/src/Parsers/ASTAlterQuery.h | 5 ++++ dbms/src/Storages/AlterCommands.cpp | 18 ++++++++++++++- dbms/src/Storages/AlterCommands.h | 4 ++++ dbms/src/Storages/ColumnsDescription.cpp | 23 +++++++++++++++++++ dbms/src/Storages/ColumnsDescription.h | 3 +++ dbms/src/Storages/MergeTree/MergeTreeData.cpp | 4 ++++ dbms/src/Storages/MergeTree/MergeTreeData.h | 9 ++++++++ .../MergeTree/ReplicatedMergeTreeQueue.cpp | 15 ++++++++++++ .../MergeTree/ReplicatedMergeTreeQueue.h | 3 +++ dbms/src/Storages/StorageMergeTree.h | 2 ++ .../Storages/StorageReplicatedMergeTree.cpp | 5 ++++ .../src/Storages/StorageReplicatedMergeTree.h | 4 ++++ 15 files changed, 115 insertions(+), 1 deletion(-) diff --git a/dbms/src/Access/AccessType.h b/dbms/src/Access/AccessType.h index 27892076d59..29a289ac235 100644 --- a/dbms/src/Access/AccessType.h +++ b/dbms/src/Access/AccessType.h @@ -28,6 +28,7 @@ enum class AccessType ADD_COLUMN, DROP_COLUMN, MODIFY_COLUMN, + RENAME_COLUMN, COMMENT_COLUMN, CLEAR_COLUMN, ALTER_COLUMN, /// allow to execute ALTER {ADD|DROP|MODIFY...} COLUMN @@ -195,6 +196,7 @@ namespace impl ACCESS_TYPE_TO_KEYWORD_CASE(ADD_COLUMN); ACCESS_TYPE_TO_KEYWORD_CASE(DROP_COLUMN); ACCESS_TYPE_TO_KEYWORD_CASE(MODIFY_COLUMN); + ACCESS_TYPE_TO_KEYWORD_CASE(RENAME_COLUMN); ACCESS_TYPE_TO_KEYWORD_CASE(COMMENT_COLUMN); ACCESS_TYPE_TO_KEYWORD_CASE(CLEAR_COLUMN); ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_COLUMN); diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 315527765ef..298f3fc4097 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -280,6 +280,11 @@ AccessRightsElements InterpreterAlterQuery::getRequiredAccessForCommand(const AS required_access.emplace_back(AccessType::REFRESH_VIEW, database, table); break; } + case ASTAlterCommand::RENAME_COLUMN: + { + required_access.emplace_back(AccessType::RENAME_COLUMN, database, table); + break; + } case ASTAlterCommand::NO_TYPE: break; } diff --git a/dbms/src/Parsers/ASTAlterQuery.cpp b/dbms/src/Parsers/ASTAlterQuery.cpp index 50d751a9c3b..9ec2fad5768 100644 --- a/dbms/src/Parsers/ASTAlterQuery.cpp +++ b/dbms/src/Parsers/ASTAlterQuery.cpp @@ -56,6 +56,11 @@ ASTPtr ASTAlterCommand::clone() const res->values = values->clone(); res->children.push_back(res->values); } + if (rename_to) + { + res->rename_to = rename_to->clone(); + res->children.push_back(res->rename_to); + } return res; } @@ -285,6 +290,15 @@ void ASTAlterCommand::formatImpl( { settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "REFRESH " << (settings.hilite ? hilite_none : ""); } + else if (type == ASTAlterCommand::RENAME_COLUMN) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "RENAME COLUMN " << (if_exists ? "IF EXISTS " : "") + << (settings.hilite ? hilite_none : ""); + column->formatImpl(settings, state, frame); + + settings.ostr << (settings.hilite ? hilite_keyword : "") << " TO "; + rename_to->formatImpl(settings, state, frame); + } else throw Exception("Unexpected type of ALTER", ErrorCodes::UNEXPECTED_AST_STRUCTURE); } diff --git a/dbms/src/Parsers/ASTAlterQuery.h b/dbms/src/Parsers/ASTAlterQuery.h index de36394a9c3..85e9a4d7552 100644 --- a/dbms/src/Parsers/ASTAlterQuery.h +++ b/dbms/src/Parsers/ASTAlterQuery.h @@ -29,6 +29,7 @@ public: DROP_COLUMN, MODIFY_COLUMN, COMMENT_COLUMN, + RENAME_COLUMN, MODIFY_ORDER_BY, MODIFY_TTL, MATERIALIZE_TTL, @@ -69,6 +70,7 @@ public: /** The ADD COLUMN query here optionally stores the name of the column following AFTER * The DROP query stores the column name for deletion here + * Also used for RENAME COLUMN. */ ASTPtr column; @@ -155,6 +157,9 @@ public: String to_database; String to_table; + /// Target column name + ASTPtr rename_to; + String getID(char delim) const override { return "AlterCommand" + (delim + std::to_string(static_cast(type))); } ASTPtr clone() const override; diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index a02e5b5a879..4af0e3e49c9 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -231,10 +231,20 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::MODIFY_QUERY) { AlterCommand command; + command.ast = command_ast->clone(); command.type = AlterCommand::MODIFY_QUERY; command.select = command_ast->select; return command; } + else if (command_ast->type == ASTAlterCommand::RENAME_COLUMN) + { + AlterCommand command; + command.ast = command_ast->clone(); + command.type = AlterCommand::RENAME_COLUMN; + command.column_name = command_ast->column->as().name; + command.rename_to = command_ast->rename_to->as().name; + return command; + } else return {}; } @@ -437,6 +447,10 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata) const settings_from_storage.push_back(change); } } + else if (type == RENAME_COLUMN) + { + metadata.columns.rename(column_name, rename_to); + } else throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR); } @@ -519,7 +533,7 @@ bool AlterCommand::isRequireMutationStage(const StorageInMemoryMetadata & metada if (ignore) return false; - if (type == DROP_COLUMN || type == DROP_INDEX) + if (type == DROP_COLUMN || type == DROP_INDEX || type == RENAME_COLUMN) return true; if (type != MODIFY_COLUMN || data_type == nullptr) @@ -619,6 +633,8 @@ String alterTypeToString(const AlterCommand::Type type) return "MODIFY SETTING"; case AlterCommand::Type::MODIFY_QUERY: return "MODIFY QUERY"; + case AlterCommand::Type::RENAME_COLUMN: + return "RENAME COLUMN"; } __builtin_unreachable(); } diff --git a/dbms/src/Storages/AlterCommands.h b/dbms/src/Storages/AlterCommands.h index 886c8beaed9..be27ba6ac2b 100644 --- a/dbms/src/Storages/AlterCommands.h +++ b/dbms/src/Storages/AlterCommands.h @@ -35,6 +35,7 @@ struct AlterCommand MODIFY_TTL, MODIFY_SETTING, MODIFY_QUERY, + RENAME_COLUMN, }; Type type; @@ -96,6 +97,9 @@ struct AlterCommand /// For MODIFY_QUERY ASTPtr select = nullptr; + /// Target column name + String rename_to; + static std::optional parse(const ASTAlterCommand * command); void apply(StorageInMemoryMetadata & metadata) const; diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp index 2b2281c9663..ea8217c5f18 100644 --- a/dbms/src/Storages/ColumnsDescription.cpp +++ b/dbms/src/Storages/ColumnsDescription.cpp @@ -1,4 +1,6 @@ #include + +#include #include #include #include @@ -195,6 +197,27 @@ void ColumnsDescription::remove(const String & column_name) list_it = columns.get<0>().erase(list_it); } +void ColumnsDescription::rename(const String & column_from, const String & column_to) +{ + auto range = getNameRange(columns, column_from); + + if (range.first == range.second) + throw Exception("There is no column " + column_from + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); + + std::vector iterators; + for (auto list_it = range.first; list_it != range.second;) + { + iterators.push_back(*list_it); + list_it = columns.get<0>().erase(list_it); + } + + for (auto & col_desc : iterators) + { + boost::replace_all(col_desc.name, column_from, column_to); + add(col_desc); + } +} + void ColumnsDescription::flattenNested() { diff --git a/dbms/src/Storages/ColumnsDescription.h b/dbms/src/Storages/ColumnsDescription.h index f930b333577..b6315bfa6eb 100644 --- a/dbms/src/Storages/ColumnsDescription.h +++ b/dbms/src/Storages/ColumnsDescription.h @@ -57,6 +57,9 @@ public: /// `column_name` can be a Nested column name; void remove(const String & column_name); + /// TODO(alesap) + void rename(const String & column_from, const String & column_to); + void flattenNested(); /// TODO: remove, insert already flattened Nested columns. bool operator==(const ColumnsDescription & other) const { return columns == other.columns; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 2279618c9a0..83442f4dff1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -3589,4 +3589,8 @@ bool MergeTreeData::canUsePolymorphicParts(const MergeTreeSettings & settings, S return true; } +MergeTreeData::AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPartPtr /*part*/) const +{ + return AlterConversions{}; +} } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index fbc42de5517..5e4fb5c8430 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -33,6 +33,7 @@ namespace DB class MergeListEntry; class AlterCommands; class MergeTreePartsMover; +class MutationCommands; class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; @@ -124,6 +125,11 @@ public: STRONG_TYPEDEF(String, PartitionID) + struct AlterConversions + { + std::unordered_map rename_map; + }; + struct LessDataPart { using is_transparent = void; @@ -647,6 +653,8 @@ public: /// Reserves 0 bytes ReservationPtr makeEmptyReservationOnLargestDisk() { return getStoragePolicy()->makeEmptyReservationOnLargestDisk(); } + AlterConversions getAlterConversionsForPart(const MergeTreeDataPartPtr part) const; + MergeTreeDataFormatVersion format_version; Context & global_context; @@ -908,6 +916,7 @@ protected: /// mechanisms for parts locking virtual bool partIsAssignedToBackgroundOperation(const DataPartPtr & part) const = 0; + virtual MutationCommands getFirtsAlterMutationCommandsForPart(const DataPartPtr & part) const = 0; /// Moves part to specified space, used in ALTER ... MOVE ... queries bool movePartsToSpace(const DataPartsVector & parts, SpacePtr space); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 73ea2098c71..1f5c5ea9a98 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1309,6 +1309,21 @@ ReplicatedMergeTreeMergePredicate ReplicatedMergeTreeQueue::getMergePredicate(zk } +MutationCommands ReplicatedMergeTreeQueue::getFirstAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const +{ + std::lock_guard lock(state_mutex); + auto in_partition = mutations_by_partition.find(part->info.partition_id); + if (in_partition == mutations_by_partition.end()) + return MutationCommands{}; + + Int64 part_version = part->info.getDataVersion(); + for (auto [mutation_version, mutation_status] : in_partition->second) + if (mutation_version > part_version && mutation_status->entry->alter_version != -1) + return mutation_status->entry->commands; + + return MutationCommands{}; +} + MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( const MergeTreeData::DataPartPtr & part, Int64 desired_mutation_version) const { diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 22d198b9f19..7a3c70023da 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -331,6 +331,9 @@ public: MutationCommands getMutationCommands(const MergeTreeData::DataPartPtr & part, Int64 desired_mutation_version) const; + /// TODO(alesap) + MutationCommands getFirstAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const; + /// Mark finished mutations as done. If the function needs to be called again at some later time /// (because some mutations are probably done but we are not sure yet), returns true. bool tryFinalizeMutations(zkutil::ZooKeeperPtr zookeeper); diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 93d7ac89832..bfbf4d6ab7e 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -164,6 +164,8 @@ protected: const MergingParams & merging_params_, std::unique_ptr settings_, bool has_force_restore_data_flag); + + MutationCommands getFirtsAlterMutationCommandsForPart(const DataPartPtr & /* part */) const override { return {}; } }; } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 68cc98cb1b9..2e70d9037f0 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -5297,4 +5297,9 @@ StorageReplicatedMergeTree::getMetadataFromSharedZookeeper(const String & metada } +MutationCommands StorageReplicatedMergeTree::getFirtsAlterMutationCommandsForPart(const DataPartPtr & part) const +{ + return queue.getFirstAlterMutationCommandsForPart(part); +} + } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index f6483baf353..3ec72d2f7b6 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -527,6 +527,9 @@ private: StorageInMemoryMetadata getMetadataFromSharedZookeeper(const String & metadata_str, const String & columns_str) const; + + MutationCommands getFirtsAlterMutationCommandsForPart(const DataPartPtr & part) const override; + protected: /** If not 'attach', either creates a new table in ZK, or adds a replica to an existing table. */ @@ -542,6 +545,7 @@ protected: const MergingParams & merging_params_, std::unique_ptr settings_, bool has_force_restore_data_flag); + };