From 68cb59227725457df9074189d693232eca317d95 Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Wed, 6 Sep 2017 23:34:26 +0300 Subject: [PATCH] allow arbitrary partition literals in ALTER PARTITION queries [#CLICKHOUSE-3000] --- dbms/src/Common/AutoArray.h | 2 +- dbms/src/Core/ErrorCodes.cpp | 2 +- .../Interpreters/InterpreterAlterQuery.cpp | 31 +++---- dbms/src/Interpreters/InterpreterAlterQuery.h | 14 ++-- .../Interpreters/InterpreterOptimizeQuery.cpp | 4 +- dbms/src/Parsers/ASTAlterQuery.h | 3 +- dbms/src/Parsers/ASTOptimizeQuery.h | 26 ++++-- dbms/src/Parsers/ASTPartition.cpp | 44 ++++++++++ dbms/src/Parsers/ASTPartition.h | 29 +++++++ dbms/src/Parsers/ExpressionElementParsers.cpp | 4 +- dbms/src/Parsers/ParserAlterQuery.cpp | 21 +++-- dbms/src/Parsers/ParserOptimizeQuery.cpp | 14 ++-- dbms/src/Parsers/ParserPartition.cpp | 81 ++++++++++++++++++ dbms/src/Parsers/ParserPartition.h | 17 ++++ dbms/src/Parsers/TokenIterator.h | 1 + dbms/src/Storages/IStorage.h | 14 ++-- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 83 ++++++++++++++++--- dbms/src/Storages/MergeTree/MergeTreeData.h | 4 +- .../Storages/MergeTree/MergeTreeDataPart.cpp | 16 +++- dbms/src/Storages/StorageBuffer.cpp | 8 +- dbms/src/Storages/StorageBuffer.h | 2 +- dbms/src/Storages/StorageDistributed.cpp | 5 +- dbms/src/Storages/StorageDistributed.h | 2 +- dbms/src/Storages/StorageMaterializedView.cpp | 4 +- dbms/src/Storages/StorageMaterializedView.h | 2 +- dbms/src/Storages/StorageMergeTree.cpp | 34 +++++--- dbms/src/Storages/StorageMergeTree.h | 13 ++- .../Storages/StorageReplicatedMergeTree.cpp | 51 ++++++------ .../src/Storages/StorageReplicatedMergeTree.h | 14 ++-- dbms/src/Storages/StorageTrivialBuffer.cpp | 4 +- dbms/src/Storages/StorageTrivialBuffer.h | 2 +- 31 files changed, 404 insertions(+), 147 deletions(-) create mode 100644 dbms/src/Parsers/ASTPartition.cpp create mode 100644 dbms/src/Parsers/ASTPartition.h create mode 100644 dbms/src/Parsers/ParserPartition.cpp create mode 100644 dbms/src/Parsers/ParserPartition.h diff --git a/dbms/src/Common/AutoArray.h b/dbms/src/Common/AutoArray.h index 6f8d67539f3..a972559543f 100644 --- a/dbms/src/Common/AutoArray.h +++ b/dbms/src/Common/AutoArray.h @@ -48,7 +48,7 @@ public: setEmpty(); } - AutoArray(size_t size_) + explicit AutoArray(size_t size_) { init(size_, false); } diff --git a/dbms/src/Core/ErrorCodes.cpp b/dbms/src/Core/ErrorCodes.cpp index 8fbb1956941..6afa1361a5a 100644 --- a/dbms/src/Core/ErrorCodes.cpp +++ b/dbms/src/Core/ErrorCodes.cpp @@ -253,7 +253,7 @@ namespace ErrorCodes extern const int INVALID_NESTED_NAME = 245; extern const int CORRUPTED_DATA = 246; extern const int INCORRECT_MARK = 247; - extern const int INVALID_PARTITION_NAME = 248; + extern const int INVALID_PARTITION_VALUE = 248; extern const int NOT_ENOUGH_BLOCK_NUMBERS = 250; extern const int NO_SUCH_REPLICA = 251; extern const int TOO_MUCH_PARTS = 252; diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 2834fb4636a..378f19eddad 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -60,19 +60,19 @@ BlockIO InterpreterAlterQuery::execute() switch (command.type) { case PartitionCommand::DROP_PARTITION: - table->dropPartition(query_ptr, command.partition, command.detach, context.getSettingsRef()); + table->dropPartition(query_ptr, command.partition, command.detach, context); break; case PartitionCommand::ATTACH_PARTITION: - table->attachPartition(query_ptr, command.partition, command.part, context.getSettingsRef()); + table->attachPartition(command.partition, command.part, context); break; case PartitionCommand::FETCH_PARTITION: - table->fetchPartition(command.partition, command.from, context.getSettingsRef()); + table->fetchPartition(command.partition, command.from, context); break; case PartitionCommand::FREEZE_PARTITION: - table->freezePartition(command.partition, command.with_name, context.getSettingsRef()); + table->freezePartition(command.partition, command.with_name, context); break; case PartitionCommand::RESHARD_PARTITION: @@ -82,7 +82,7 @@ BlockIO InterpreterAlterQuery::execute() break; case PartitionCommand::CLEAR_COLUMN: - table->clearColumnInPartition(query_ptr, command.partition, command.column_name, context.getSettingsRef()); + table->clearColumnInPartition(command.partition, command.column_name, context); break; } } @@ -136,10 +136,9 @@ void InterpreterAlterQuery::parseAlter( if (!params.clear_column) throw Exception("Can't DROP COLUMN from partition. It is possible only CLEAR COLUMN in partition", ErrorCodes::BAD_ARGUMENTS); - const Field & partition = typeid_cast(*(params.partition)).value; const Field & column_name = typeid_cast(*(params.column)).name; - out_partition_commands.emplace_back(PartitionCommand::clearColumn(partition, column_name)); + out_partition_commands.emplace_back(PartitionCommand::clearColumn(params.partition, column_name)); } else { @@ -185,30 +184,22 @@ void InterpreterAlterQuery::parseAlter( } else if (params.type == ASTAlterQuery::DROP_PARTITION) { - const Field & partition = dynamic_cast(*params.partition).value; - out_partition_commands.emplace_back(PartitionCommand::dropPartition(partition, params.detach)); + out_partition_commands.emplace_back(PartitionCommand::dropPartition(params.partition, params.detach)); } else if (params.type == ASTAlterQuery::ATTACH_PARTITION) { - const Field & partition = dynamic_cast(*params.partition).value; - out_partition_commands.emplace_back(PartitionCommand::attachPartition(partition, params.part)); + out_partition_commands.emplace_back(PartitionCommand::attachPartition(params.partition, params.part)); } else if (params.type == ASTAlterQuery::FETCH_PARTITION) { - const Field & partition = dynamic_cast(*params.partition).value; - out_partition_commands.emplace_back(PartitionCommand::fetchPartition(partition, params.from)); + out_partition_commands.emplace_back(PartitionCommand::fetchPartition(params.partition, params.from)); } else if (params.type == ASTAlterQuery::FREEZE_PARTITION) { - const Field & partition = dynamic_cast(*params.partition).value; - out_partition_commands.emplace_back(PartitionCommand::freezePartition(partition, params.with_name)); + out_partition_commands.emplace_back(PartitionCommand::freezePartition(params.partition, params.with_name)); } else if (params.type == ASTAlterQuery::RESHARD_PARTITION) { - Field partition; - if (params.partition) - partition = dynamic_cast(*params.partition).value; - WeightedZooKeeperPaths weighted_zookeeper_paths; const ASTs & ast_weighted_zookeeper_paths = typeid_cast(*params.weighted_zookeeper_paths).children; @@ -223,7 +214,7 @@ void InterpreterAlterQuery::parseAlter( coordinator = dynamic_cast(*params.coordinator).value; out_partition_commands.emplace_back(PartitionCommand::reshardPartitions( - partition, weighted_zookeeper_paths, params.sharding_key_expr, + params.partition, weighted_zookeeper_paths, params.sharding_key_expr, params.do_copy, coordinator)); } else diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.h b/dbms/src/Interpreters/InterpreterAlterQuery.h index 4fd718cd781..438554692af 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.h +++ b/dbms/src/Interpreters/InterpreterAlterQuery.h @@ -35,7 +35,7 @@ private: Type type; - Field partition; + ASTPtr partition; Field column_name; bool detach = false; /// true for DETACH PARTITION. @@ -52,7 +52,7 @@ private: /// For FREEZE PARTITION String with_name; - static PartitionCommand dropPartition(const Field & partition, bool detach) + static PartitionCommand dropPartition(const ASTPtr & partition, bool detach) { PartitionCommand res; res.type = DROP_PARTITION; @@ -61,7 +61,7 @@ private: return res; } - static PartitionCommand clearColumn(const Field & partition, const Field & column_name) + static PartitionCommand clearColumn(const ASTPtr & partition, const Field & column_name) { PartitionCommand res; res.type = CLEAR_COLUMN; @@ -70,7 +70,7 @@ private: return res; } - static PartitionCommand attachPartition(const Field & partition, bool part) + static PartitionCommand attachPartition(const ASTPtr & partition, bool part) { PartitionCommand res; res.type = ATTACH_PARTITION; @@ -79,7 +79,7 @@ private: return res; } - static PartitionCommand fetchPartition(const Field & partition, const String & from) + static PartitionCommand fetchPartition(const ASTPtr & partition, const String & from) { PartitionCommand res; res.type = FETCH_PARTITION; @@ -88,7 +88,7 @@ private: return res; } - static PartitionCommand freezePartition(const Field & partition, const String & with_name) + static PartitionCommand freezePartition(const ASTPtr & partition, const String & with_name) { PartitionCommand res; res.type = FREEZE_PARTITION; @@ -97,7 +97,7 @@ private: return res; } - static PartitionCommand reshardPartitions(const Field & partition_, + static PartitionCommand reshardPartitions(const ASTPtr & partition_, const WeightedZooKeeperPaths & weighted_zookeeper_paths_, const ASTPtr & sharding_key_expr_, bool do_copy_, const Field & coordinator_) { diff --git a/dbms/src/Interpreters/InterpreterOptimizeQuery.cpp b/dbms/src/Interpreters/InterpreterOptimizeQuery.cpp index c9bce30271c..8504ffe3821 100644 --- a/dbms/src/Interpreters/InterpreterOptimizeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterOptimizeQuery.cpp @@ -18,12 +18,12 @@ BlockIO InterpreterOptimizeQuery::execute() { const ASTOptimizeQuery & ast = typeid_cast(*query_ptr); - if (ast.final && ast.partition.empty()) + if (ast.final && !ast.partition) throw Exception("FINAL flag for OPTIMIZE query is meaningful only with specified PARTITION", ErrorCodes::BAD_ARGUMENTS); StoragePtr table = context.getTable(ast.database, ast.table); auto table_lock = table->lockStructure(true, __PRETTY_FUNCTION__); - table->optimize(query_ptr, ast.partition, ast.final, ast.deduplicate, context.getSettings()); + table->optimize(query_ptr, ast.partition, ast.final, ast.deduplicate, context); return {}; } diff --git a/dbms/src/Parsers/ASTAlterQuery.h b/dbms/src/Parsers/ASTAlterQuery.h index f49bc4c1435..8c9bcd09d82 100644 --- a/dbms/src/Parsers/ASTAlterQuery.h +++ b/dbms/src/Parsers/ASTAlterQuery.h @@ -60,9 +60,10 @@ public: */ ASTPtr primary_key; - /** In DROP PARTITION and RESHARD PARTITION queries, the name of the partition is stored here. + /** In DROP PARTITION and RESHARD PARTITION queries, the value or ID of the partition is stored here. */ ASTPtr partition; + bool detach = false; /// true for DETACH PARTITION bool part = false; /// true for ATTACH PART diff --git a/dbms/src/Parsers/ASTOptimizeQuery.h b/dbms/src/Parsers/ASTOptimizeQuery.h index c395dc18ee7..3caae258b41 100644 --- a/dbms/src/Parsers/ASTOptimizeQuery.h +++ b/dbms/src/Parsers/ASTOptimizeQuery.h @@ -16,7 +16,7 @@ public: String table; /// The partition to optimize can be specified. - String partition; + ASTPtr partition; /// A flag can be specified - perform optimization "to the end" instead of one step. bool final; /// Do deduplicate (default: false) @@ -26,9 +26,21 @@ public: ASTOptimizeQuery(const StringRange range_) : IAST(range_) {} /** Get the text that identifies this element. */ - String getID() const override { return "OptimizeQuery_" + database + "_" + table + "_" + partition + (final ? "_final" : "") + (deduplicate ? "_deduplicate" : ""); }; + String getID() const override { return "OptimizeQuery_" + database + "_" + table + (final ? "_final" : "") + (deduplicate ? "_deduplicate" : ""); }; - ASTPtr clone() const override { return std::make_shared(*this); } + ASTPtr clone() const override + { + auto res = std::make_shared(*this); + res->children.clear(); + + if (partition) + { + res->partition = partition->clone(); + res->children.push_back(res->partition); + } + + return res; + } protected: void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override @@ -36,9 +48,11 @@ protected: settings.ostr << (settings.hilite ? hilite_keyword : "") << "OPTIMIZE TABLE " << (settings.hilite ? hilite_none : "") << (!database.empty() ? backQuoteIfNeed(database) + "." : "") << backQuoteIfNeed(table); - if (!partition.empty()) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " PARTITION " << (settings.hilite ? hilite_none : "") - << partition; + if (partition) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(settings, state, frame); + } if (final) settings.ostr << (settings.hilite ? hilite_keyword : "") << " FINAL" << (settings.hilite ? hilite_none : ""); diff --git a/dbms/src/Parsers/ASTPartition.cpp b/dbms/src/Parsers/ASTPartition.cpp new file mode 100644 index 00000000000..6f0a256ec3b --- /dev/null +++ b/dbms/src/Parsers/ASTPartition.cpp @@ -0,0 +1,44 @@ +#include +#include + +namespace DB +{ + +String ASTPartition::getID() const +{ + if (value) + return "Partition"; + else + return "Partition_ID_" + id; +} + +ASTPtr ASTPartition::clone() const +{ + auto res = std::make_shared(*this); + res->children.clear(); + + if (value) + { + res->value = value->clone(); + res->children.push_back(res->value); + } + + return res; +} + +void ASTPartition::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +{ + if (value) + { + value->formatImpl(settings, state, frame); + } + else + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << "ID " << (settings.hilite ? hilite_none : ""); + WriteBufferFromOwnString id_buf; + writeQuoted(id, id_buf); + settings.ostr << id_buf.str(); + } +} + +} diff --git a/dbms/src/Parsers/ASTPartition.h b/dbms/src/Parsers/ASTPartition.h new file mode 100644 index 00000000000..9f78d56fca1 --- /dev/null +++ b/dbms/src/Parsers/ASTPartition.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +/// Either a (possibly compound) expression representing a partition value or a partition ID. +class ASTPartition : public IAST +{ +public: + ASTPtr value; + StringRef fields_str; /// The extent of comma-separated partition expression fields without parentheses. + size_t fields_count = 0; + + String id; + + ASTPartition() = default; + ASTPartition(StringRange range_) : IAST(range_) {} + String getID() const override; + ASTPtr clone() const override; + +protected: + void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; +}; + +} diff --git a/dbms/src/Parsers/ExpressionElementParsers.cpp b/dbms/src/Parsers/ExpressionElementParsers.cpp index c2203ae4f25..49d1e6da9c9 100644 --- a/dbms/src/Parsers/ExpressionElementParsers.cpp +++ b/dbms/src/Parsers/ExpressionElementParsers.cpp @@ -498,6 +498,8 @@ bool ParserStringLiteral::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte if (pos->type != TokenType::StringLiteral) return false; + Pos begin = pos; + String s; ReadBufferFromMemory in(pos->begin, pos->size()); @@ -518,7 +520,7 @@ bool ParserStringLiteral::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte } ++pos; - node = std::make_shared(StringRange(pos->begin, pos->end), s); + node = std::make_shared(StringRange(begin, pos), s); return true; } diff --git a/dbms/src/Parsers/ParserAlterQuery.cpp b/dbms/src/Parsers/ParserAlterQuery.cpp index 3f2c4334523..7b36f80d6bf 100644 --- a/dbms/src/Parsers/ParserAlterQuery.cpp +++ b/dbms/src/Parsers/ParserAlterQuery.cpp @@ -2,8 +2,8 @@ #include #include #include +#include #include -#include #include #include @@ -46,8 +46,7 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserIdentifier table_parser; ParserCompoundIdentifier parser_name; ParserCompoundColumnDeclaration parser_col_decl; - ParserLiteral parser_literal; - ParserUnsignedInteger parser_uint; + ParserPartition parser_partition; ParserStringLiteral parser_string_literal; ASTPtr table; @@ -106,7 +105,7 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } else if (s_drop_partition.ignore(pos, expected)) { - if (!parser_literal.parse(pos, params.partition, expected)) + if (!parser_partition.parse(pos, params.partition, expected)) return false; params.type = ASTAlterQuery::DROP_PARTITION; @@ -130,13 +129,13 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (s_in_partition.ignore(pos, expected)) { - if (!parser_literal.parse(pos, params.partition, expected)) + if (!parser_partition.parse(pos, params.partition, expected)) return false; } } else if (s_detach_partition.ignore(pos, expected)) { - if (!parser_literal.parse(pos, params.partition, expected)) + if (!parser_partition.parse(pos, params.partition, expected)) return false; params.type = ASTAlterQuery::DROP_PARTITION; @@ -144,14 +143,14 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } else if (s_attach_partition.ignore(pos, expected)) { - if (!parser_literal.parse(pos, params.partition, expected)) + if (!parser_partition.parse(pos, params.partition, expected)) return false; params.type = ASTAlterQuery::ATTACH_PARTITION; } else if (s_attach_part.ignore(pos, expected)) { - if (!parser_literal.parse(pos, params.partition, expected)) + if (!parser_string_literal.parse(pos, params.partition, expected)) return false; params.part = true; @@ -159,7 +158,7 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } else if (s_fetch_partition.ignore(pos, expected)) { - if (!parser_literal.parse(pos, params.partition, expected)) + if (!parser_partition.parse(pos, params.partition, expected)) return false; if (!s_from.ignore(pos, expected)) @@ -174,7 +173,7 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } else if (s_freeze_partition.ignore(pos, expected)) { - if (!parser_literal.parse(pos, params.partition, expected)) + if (!parser_partition.parse(pos, params.partition, expected)) return false; /// WITH NAME 'name' - place local backup to directory with specified name @@ -225,7 +224,7 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (s_partition.ignore(pos, expected)) { - if (!parser_uint.parse(pos, params.partition, expected)) + if (!parser_partition.parse(pos, params.partition, expected)) return false; } diff --git a/dbms/src/Parsers/ParserOptimizeQuery.cpp b/dbms/src/Parsers/ParserOptimizeQuery.cpp index eb4580755bf..90e9146210a 100644 --- a/dbms/src/Parsers/ParserOptimizeQuery.cpp +++ b/dbms/src/Parsers/ParserOptimizeQuery.cpp @@ -1,8 +1,9 @@ -#include -#include - -#include #include +#include +#include + +#include +#include #include #include @@ -22,7 +23,7 @@ bool ParserOptimizeQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte ParserKeyword s_deduplicate("DEDUPLICATE"); ParserToken s_dot(TokenType::Dot); ParserIdentifier name_p; - ParserLiteral partition_p; + ParserPartition partition_p; ASTPtr database; ASTPtr table; @@ -62,8 +63,7 @@ bool ParserOptimizeQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte query->database = typeid_cast(*database).name; if (table) query->table = typeid_cast(*table).name; - if (partition) - query->partition = applyVisitor(FieldVisitorToString(), typeid_cast(*partition).value); + query->partition = partition; query->final = final; query->deduplicate = deduplicate; diff --git a/dbms/src/Parsers/ParserPartition.cpp b/dbms/src/Parsers/ParserPartition.cpp new file mode 100644 index 00000000000..9ecd03d9cd8 --- /dev/null +++ b/dbms/src/Parsers/ParserPartition.cpp @@ -0,0 +1,81 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +bool ParserPartition::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword s_id("ID"); + ParserStringLiteral parser_string_literal; + ParserExpression parser_expr; + + Pos begin = pos; + + auto partition = std::make_shared(); + + if (s_id.ignore(pos, expected)) + { + ASTPtr partition_id; + if (!parser_string_literal.parse(pos, partition_id, expected)) + return false; + + partition->id = dynamic_cast(*partition_id).value.get(); + } + else + { + ASTPtr value; + if (!parser_expr.parse(pos, value, expected)) + return false; + + size_t fields_count; + StringRef fields_str; + + const auto * tuple_ast = typeid_cast(value.get()); + if (tuple_ast && tuple_ast->name == "tuple") + { + const auto * arguments_ast = dynamic_cast(tuple_ast->arguments.get()); + if (arguments_ast) + fields_count = arguments_ast->children.size(); + else + fields_count = 0; + + Pos left_paren = begin; + Pos right_paren = pos; + + while (left_paren != right_paren && left_paren->type != TokenType::OpeningRoundBracket) + ++left_paren; + if (left_paren->type != TokenType::OpeningRoundBracket) + return false; + + while (right_paren != left_paren && right_paren->type != TokenType::ClosingRoundBracket) + --right_paren; + if (right_paren->type != TokenType::ClosingRoundBracket) + return false; + + fields_str = StringRef(left_paren->end, right_paren->begin - left_paren->end); + } + else + { + fields_count = 1; + fields_str = StringRef(begin->begin, pos->begin - begin->begin); + } + + partition->value = value; + partition->children.push_back(value); + partition->fields_str = fields_str; + partition->fields_count = fields_count; + } + + partition->range = StringRange(begin, pos); + node = partition; + return true; +} + +} diff --git a/dbms/src/Parsers/ParserPartition.h b/dbms/src/Parsers/ParserPartition.h new file mode 100644 index 00000000000..2bb7048fd81 --- /dev/null +++ b/dbms/src/Parsers/ParserPartition.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +namespace DB +{ + +/// Parse either a partition value as a (possibly compound) literal or a partition ID. +/// Produce ASTPartition. +class ParserPartition : public IParserBase +{ +protected: + const char * getName() const override { return "partition"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} diff --git a/dbms/src/Parsers/TokenIterator.h b/dbms/src/Parsers/TokenIterator.h index 71d2edfef42..8430d7282e0 100644 --- a/dbms/src/Parsers/TokenIterator.h +++ b/dbms/src/Parsers/TokenIterator.h @@ -69,6 +69,7 @@ public: bool operator< (const TokenIterator & rhs) const { return index < rhs.index; } bool operator<= (const TokenIterator & rhs) const { return index <= rhs.index; } bool operator== (const TokenIterator & rhs) const { return index == rhs.index; } + bool operator!= (const TokenIterator & rhs) const { return index != rhs.index; } bool isValid() { return get().type < TokenType::EndOfStream; } diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 473792376c6..9d3e72d64ea 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -221,35 +221,35 @@ public: } /** Execute CLEAR COLUMN ... IN PARTITION query which removes column from given partition. */ - virtual void clearColumnInPartition(const ASTPtr & query, const Field & partition, const Field & column_name, const Settings & settings) + virtual void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context) { throw Exception("Method dropColumnFromPartition is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } /** Run the query (DROP|DETACH) PARTITION. */ - virtual void dropPartition(const ASTPtr & query, const Field & partition, bool detach, const Settings & settings) + virtual void dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & context) { throw Exception("Method dropPartition is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } /** Run the ATTACH request (PART|PARTITION). */ - virtual void attachPartition(const ASTPtr & query, const Field & partition, bool part, const Settings & settings) + virtual void attachPartition(const ASTPtr & partition, bool part, const Context & context) { throw Exception("Method attachPartition is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } /** Run the FETCH PARTITION query. */ - virtual void fetchPartition(const Field & partition, const String & from, const Settings & settings) + virtual void fetchPartition(const ASTPtr & partition, const String & from, const Context & context) { throw Exception("Method fetchPartition is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } /** Run the FREEZE PARTITION request. That is, create a local backup (snapshot) of data using the `localBackup` function (see localBackup.h) */ - virtual void freezePartition(const Field & partition, const String & with_name, const Settings & settings) + virtual void freezePartition(const ASTPtr & partition, const String & with_name, const Context & context) { throw Exception("Method freezePartition is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } @@ -258,7 +258,7 @@ public: */ virtual void reshardPartitions( const ASTPtr & query, const String & database_name, - const Field & partition, + const ASTPtr & partition, const WeightedZooKeeperPaths & weighted_zookeeper_paths, const ASTPtr & sharding_key_expr, bool do_copy, const Field & coordinator, const Context & context) @@ -269,7 +269,7 @@ public: /** Perform any background work. For example, combining parts in a MergeTree type table. * Returns whether any work has been done. */ - virtual bool optimize(const ASTPtr & query, const String & partition_id, bool final, bool deduplicate, const Settings & settings) + virtual bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) { throw Exception("Method optimize is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 8bdf87e6f07..2e6d36ab66d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -6,13 +6,19 @@ #include #include #include +#include +#include +#include #include #include #include +#include #include #include #include #include +#include +#include #include #include #include @@ -63,6 +69,7 @@ namespace ErrorCodes extern const int MEMORY_LIMIT_EXCEEDED; extern const int SYNTAX_ERROR; extern const int CORRUPTED_DATA; + extern const int INVALID_PARTITION_VALUE; } @@ -1714,8 +1721,21 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part } -void MergeTreeData::freezePartition(const std::string & prefix, const String & with_name) +void MergeTreeData::freezePartition(const ASTPtr & partition_ast, const String & with_name, const Context & context) { + String prefix; + if (format_version == 0) + { + const auto & partition = dynamic_cast(*partition_ast); + /// Month-partitioning specific - allow partition ID can be passed in the partition value. + if (const auto * partition_lit = dynamic_cast(partition.value.get())) + prefix = partition_lit->value.getType() == Field::Types::UInt64 + ? toString(partition_lit->value.get()) + : partition_lit->value.safeGet(); + } + else + prefix = getPartitionIDFromQuery(partition_ast, context); + LOG_DEBUG(log, "Freezing parts with prefix " + prefix); String clickhouse_path = Poco::Path(context.getPath()).makeAbsolute().toString(); @@ -1777,18 +1797,61 @@ size_t MergeTreeData::getPartitionSize(const std::string & partition_id) const return size; } -String MergeTreeData::getPartitionIDFromQuery(const Field & partition) +String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & partition_ast, const Context & context) { - /// Month-partitioning specific, TODO: generalize. - String partition_id = partition.getType() == Field::Types::UInt64 - ? toString(partition.get()) - : partition.safeGet(); + const auto & partition = typeid_cast(*partition_ast); - if (partition_id.size() != 6 || !std::all_of(partition_id.begin(), partition_id.end(), isNumericASCII)) - throw Exception("Invalid partition format: " + partition_id + ". Partition should consist of 6 digits: YYYYMM", - ErrorCodes::INVALID_PARTITION_NAME); + if (!partition.value) + return partition.id; - return partition_id; + if (format_version == 0) + { + /// Month-partitioning specific - allow partition ID can be passed in the partition value. + const auto * partition_lit = typeid_cast(partition.value.get()); + if (partition_lit && partition_lit->value.getType() == Field::Types::String) + { + String partition_id = partition_lit->value.get(); + if (partition_id.size() != 6 || !std::all_of(partition_id.begin(), partition_id.end(), isNumericASCII)) + throw Exception( + "Invalid partition format: " + partition_id + ". Partition should consist of 6 digits: YYYYMM", + ErrorCodes::INVALID_PARTITION_VALUE); + return partition_id; + } + } + + /// Re-parse partition key fields using the information about expected field types. + + size_t fields_count = partition_expr_column_types.size(); + if (partition.fields_count != fields_count) + throw Exception( + "Wrong number of fields in the partition expression: " + toString(partition.fields_count) + + ", must be: " + toString(fields_count), + ErrorCodes::INVALID_PARTITION_VALUE); + + Row partition_row(fields_count); + + if (fields_count) + { + ReadBufferFromMemory left_paren_buf("(", 1); + ReadBufferFromMemory fields_buf(partition.fields_str.data, partition.fields_str.size); + ReadBufferFromMemory right_paren_buf(")", 1); + ConcatReadBuffer buf({&left_paren_buf, &fields_buf, &right_paren_buf}); + + ValuesRowInputStream input_stream(buf, context, /* interpret_expressions = */true); + Block block; + for (size_t i = 0; i < fields_count; ++i) + block.insert(ColumnWithTypeAndName(partition_expr_column_types[i], partition_expr_columns[i])); + + if (!input_stream.read(block)) + throw Exception( + "Could not parse partition value: `" + partition.fields_str.toString() + "`", + ErrorCodes::INVALID_PARTITION_VALUE); + + for (size_t i = 0; i < fields_count; ++i) + block.getByPosition(i).column->get(0, partition_row[i]); + } + + return MergeTreeDataPart::Partition(std::move(partition_row)).getID(*this); } void MergeTreeData::Transaction::rollback() diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 3b653153e34..d5cfd4e2ead 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -410,7 +410,7 @@ public: * Backup is created in directory clickhouse_dir/shadow/i/, where i - incremental number, * or if 'with_name' is specified - backup is created in directory with specified name. */ - void freezePartition(const std::string & prefix, const String & with_name); + void freezePartition(const ASTPtr & partition, const String & with_name, const Context & context); /// Returns the size of partition in bytes. size_t getPartitionSize(const std::string & partition_id) const; @@ -461,7 +461,7 @@ public: } /// For ATTACH/DETACH/DROP/RESHARD PARTITION. - String getPartitionIDFromQuery(const Field & partition); + String getPartitionIDFromQuery(const ASTPtr & partition, const Context & context); MergeTreeDataFormatVersion format_version; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index 65907f7e5cd..673bf977a83 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -351,7 +351,18 @@ String MergeTreeDataPart::Partition::getID(const MergeTreeData & storage) const void MergeTreeDataPart::Partition::serializeTextQuoted(const MergeTreeData & storage, WriteBuffer & out) const { - for (size_t i = 0; i < storage.partition_expr_column_types.size(); ++i) + size_t key_size = storage.partition_expr_column_types.size(); + + if (key_size == 0) + { + writeCString("tuple()", out); + return; + } + + if (key_size > 1) + writeChar('(', out); + + for (size_t i = 0; i < key_size; ++i) { if (i > 0) writeCString(", ", out); @@ -361,6 +372,9 @@ void MergeTreeDataPart::Partition::serializeTextQuoted(const MergeTreeData & sto column->insert(value[i]); type->serializeTextQuoted(*column, 0, out); } + + if (key_size > 1) + writeChar(')', out); } void MergeTreeDataPart::Partition::load(const MergeTreeData & storage, const String & part_path) diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 1ceb2d8ac1e..90026d72d21 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -346,7 +346,7 @@ void StorageBuffer::shutdown() try { - optimize(nullptr /*query*/, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context.getSettings()); + optimize(nullptr /*query*/, {} /*partition*/, false /*final*/, false /*deduplicate*/, context); } catch (...) { @@ -365,9 +365,9 @@ void StorageBuffer::shutdown() * * This kind of race condition make very hard to implement proper tests. */ -bool StorageBuffer::optimize(const ASTPtr & query, const String & partition_id, bool final, bool deduplicate, const Settings & settings) +bool StorageBuffer::optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) { - if (!partition_id.empty()) + if (partition) throw Exception("Partition cannot be specified when optimizing table of type Buffer", ErrorCodes::NOT_IMPLEMENTED); if (final) @@ -593,7 +593,7 @@ void StorageBuffer::alter(const AlterCommands & params, const String & database_ auto lock = lockStructureForAlter(__PRETTY_FUNCTION__); /// So that no blocks of the old structure remain. - optimize({} /*query*/, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context.getSettings()); + optimize({} /*query*/, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context); params.apply(*columns, materialized_columns, alias_columns, column_defaults); diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h index d1c588c71e0..0e67dc4fb02 100644 --- a/dbms/src/Storages/StorageBuffer.h +++ b/dbms/src/Storages/StorageBuffer.h @@ -69,7 +69,7 @@ public: void startup() override; /// Flush all buffers into the subordinate table and stop background thread. void shutdown() override; - bool optimize(const ASTPtr & query, const String & partition_id, bool final, bool deduplicate, const Settings & settings) override; + bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override; void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override { name = new_table_name; } diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 7eb6e0b117e..141c7cccc71 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -270,7 +270,7 @@ void StorageDistributed::shutdown() void StorageDistributed::reshardPartitions( const ASTPtr & query, const String & database_name, - const Field & partition, + const ASTPtr & partition, const WeightedZooKeeperPaths & weighted_zookeeper_paths, const ASTPtr & sharding_key_expr, bool do_copy, const Field & coordinator, const Context & context) @@ -322,8 +322,7 @@ void StorageDistributed::reshardPartitions( ASTAlterQuery::Parameters & parameters = alter_query.parameters.back(); parameters.type = ASTAlterQuery::RESHARD_PARTITION; - if (!partition.isNull()) - parameters.partition = std::make_shared(StringRange(), partition); + parameters.partition = partition->clone(); ASTPtr expr_list = std::make_shared(); for (const auto & entry : weighted_zookeeper_paths) diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index cd986803434..6c6fd662eff 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -76,7 +76,7 @@ public: void reshardPartitions( const ASTPtr & query, const String & database_name, - const Field & partition, + const ASTPtr & partition, const WeightedZooKeeperPaths & weighted_zookeeper_paths, const ASTPtr & sharding_key_expr, bool do_copy, const Field & coordinator, const Context & context) override; diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 60ef05b707b..c59b84125f8 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -166,9 +166,9 @@ void StorageMaterializedView::drop() } } -bool StorageMaterializedView::optimize(const ASTPtr & query, const String & partition_id, bool final, bool deduplicate, const Settings & settings) +bool StorageMaterializedView::optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) { - return getInnerTable()->optimize(query, partition_id, final, deduplicate, settings); + return getInnerTable()->optimize(query, partition, final, deduplicate, context); } StoragePtr StorageMaterializedView::getInnerTable() const diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h index 113a23bf3c1..efc6927e26e 100644 --- a/dbms/src/Storages/StorageMaterializedView.h +++ b/dbms/src/Storages/StorageMaterializedView.h @@ -35,7 +35,7 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override; void drop() override; - bool optimize(const ASTPtr & query, const String & partition_id, bool final, bool deduplicate, const Settings & settings) override; + bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override; BlockInputStreams read( const Names & column_names, diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 9353cc66bea..4b0c2d1fc3a 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -396,7 +397,7 @@ bool StorageMergeTree::mergeTask() } -void StorageMergeTree::clearColumnInPartition(const ASTPtr & query, const Field & partition, const Field & column_name, const Settings &) +void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context) { /// Asks to complete merges and does not allow them to start. /// This protects against "revival" of data for a removed partition after completion of merge. @@ -405,7 +406,7 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & query, const Field /// We don't change table structure, only data in some parts, parts are locked inside alterDataPart() function auto lock_read_structure = lockStructure(false, __PRETTY_FUNCTION__); - String partition_id = data.getPartitionIDFromQuery(partition); + String partition_id = data.getPartitionIDFromQuery(partition, context); MergeTreeData::DataParts parts = data.getDataParts(); std::vector transactions; @@ -446,7 +447,17 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & query, const Field } -void StorageMergeTree::dropPartition(const ASTPtr & query, const Field & partition, bool detach, const Settings & settings) +bool StorageMergeTree::optimize( + const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) +{ + String partition_id; + if (partition) + partition_id = data.getPartitionIDFromQuery(partition, context); + return merge(context.getSettingsRef().min_bytes_to_use_direct_io, true, partition_id, final, deduplicate); +} + + +void StorageMergeTree::dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & context) { /// Asks to complete merges and does not allow them to start. /// This protects against "revival" of data for a removed partition after completion of merge. @@ -454,7 +465,7 @@ void StorageMergeTree::dropPartition(const ASTPtr & query, const Field & partiti /// Waits for completion of merge and does not start new ones. auto lock = lockForAlter(__PRETTY_FUNCTION__); - String partition_id = data.getPartitionIDFromQuery(partition); + String partition_id = data.getPartitionIDFromQuery(partition, context); size_t removed_parts = 0; MergeTreeData::DataParts parts = data.getDataParts(); @@ -473,18 +484,18 @@ void StorageMergeTree::dropPartition(const ASTPtr & query, const Field & partiti data.replaceParts({part}, {}, false); } - LOG_INFO(log, (detach ? "Detached " : "Removed ") << removed_parts << " parts inside " << applyVisitor(FieldVisitorToString(), partition) << "."); + LOG_INFO(log, (detach ? "Detached " : "Removed ") << removed_parts << " parts inside partition ID " << partition_id << "."); } -void StorageMergeTree::attachPartition(const ASTPtr & query, const Field & field, bool part, const Settings & settings) +void StorageMergeTree::attachPartition(const ASTPtr & partition, bool part, const Context & context) { String partition_id; if (part) - partition_id = field.getType() == Field::Types::UInt64 ? toString(field.get()) : field.safeGet(); + partition_id = typeid_cast(*partition).value.safeGet(); else - partition_id = data.getPartitionIDFromQuery(field); + partition_id = data.getPartitionIDFromQuery(partition, context); String source_dir = "detached/"; @@ -532,12 +543,9 @@ void StorageMergeTree::attachPartition(const ASTPtr & query, const Field & field } -void StorageMergeTree::freezePartition(const Field & partition, const String & with_name, const Settings & settings) +void StorageMergeTree::freezePartition(const ASTPtr & partition, const String & with_name, const Context & context) { - /// The prefix can be arbitrary. Not necessarily a month - you can specify only a year. - data.freezePartition(partition.getType() == Field::Types::UInt64 - ? toString(partition.get()) - : partition.safeGet(), with_name); + data.freezePartition(partition, with_name, context); } } diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index bc0f3070f97..fefe84a71f7 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -61,15 +61,12 @@ public: /** Perform the next step in combining the parts. */ - bool optimize(const ASTPtr & query, const String & partition_id, bool final, bool deduplicate, const Settings & settings) override - { - return merge(settings.min_bytes_to_use_direct_io, true, partition_id, final, deduplicate); - } + bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override; - void dropPartition(const ASTPtr & query, const Field & partition, bool detach, const Settings & settings) override; - void clearColumnInPartition(const ASTPtr & query, const Field & partition, const Field & column_name, const Settings & settings) override; - void attachPartition(const ASTPtr & query, const Field & partition, bool part, const Settings & settings) override; - void freezePartition(const Field & partition, const String & with_name, const Settings & settings) override; + void dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & context) override; + void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context) override; + void attachPartition(const ASTPtr & partition, bool part, const Context & context) override; + void freezePartition(const ASTPtr & partition, const String & with_name, const Context & context) override; void drop() override; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index efb9fc08c96..13874669452 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -2348,13 +2349,13 @@ BlockOutputStreamPtr StorageReplicatedMergeTree::write(const ASTPtr & query, con } -bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const String & partition_id, bool final, bool deduplicate, const Settings & settings) +bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) { assertNotReadonly(); if (!is_leader_node) { - sendRequestToLeaderReplica(query, settings); + sendRequestToLeaderReplica(query, context.getSettingsRef()); return true; } @@ -2375,13 +2376,14 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const String & p MergeTreeDataMerger::FuturePart future_merged_part; bool selected = false; - if (partition_id.empty()) + if (!partition) { selected = merger.selectPartsToMerge( future_merged_part, false, data.settings.max_bytes_to_merge_at_max_space_in_pool, can_merge); } else { + String partition_id = data.getPartitionIDFromQuery(partition, context); selected = merger.selectAllPartsToMergeWithinPartition(future_merged_part, disk_space, can_merge, partition_id, final); } @@ -2619,13 +2621,13 @@ String StorageReplicatedMergeTree::getFakePartNameCoveringAllPartsInPartition(co void StorageReplicatedMergeTree::clearColumnInPartition( - const ASTPtr & query, const Field & partition, const Field & column_name, const Settings & settings) + const ASTPtr & partition, const Field & column_name, const Context & context) { assertNotReadonly(); /// We don't block merges, so anyone can manage this task (not only leader) - String partition_id = data.getPartitionIDFromQuery(partition); + String partition_id = data.getPartitionIDFromQuery(partition, context); String fake_part_name = getFakePartNameCoveringAllPartsInPartition(partition_id); if (fake_part_name.empty()) @@ -2646,26 +2648,26 @@ void StorageReplicatedMergeTree::clearColumnInPartition( entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); /// If necessary, wait until the operation is performed on itself or on all replicas. - if (settings.replication_alter_partitions_sync != 0) + if (context.getSettingsRef().replication_alter_partitions_sync != 0) { - if (settings.replication_alter_partitions_sync == 1) + if (context.getSettingsRef().replication_alter_partitions_sync == 1) waitForReplicaToProcessLogEntry(replica_name, entry); else waitForAllReplicasToProcessLogEntry(entry); } } -void StorageReplicatedMergeTree::dropPartition(const ASTPtr & query, const Field & partition, bool detach, const Settings & settings) +void StorageReplicatedMergeTree::dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & context) { assertNotReadonly(); if (!is_leader_node) { - sendRequestToLeaderReplica(query, settings); + sendRequestToLeaderReplica(query, context.getSettingsRef()); return; } - String partition_id = data.getPartitionIDFromQuery(partition); + String partition_id = data.getPartitionIDFromQuery(partition, context); String fake_part_name = getFakePartNameCoveringAllPartsInPartition(partition_id); if (fake_part_name.empty()) @@ -2696,9 +2698,9 @@ void StorageReplicatedMergeTree::dropPartition(const ASTPtr & query, const Field entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); /// If necessary, wait until the operation is performed on itself or on all replicas. - if (settings.replication_alter_partitions_sync != 0) + if (context.getSettingsRef().replication_alter_partitions_sync != 0) { - if (settings.replication_alter_partitions_sync == 1) + if (context.getSettingsRef().replication_alter_partitions_sync == 1) waitForReplicaToProcessLogEntry(replica_name, entry); else waitForAllReplicasToProcessLogEntry(entry); @@ -2706,16 +2708,16 @@ void StorageReplicatedMergeTree::dropPartition(const ASTPtr & query, const Field } -void StorageReplicatedMergeTree::attachPartition(const ASTPtr & query, const Field & field, bool attach_part, const Settings & settings) +void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool attach_part, const Context & context) { assertNotReadonly(); String partition_id; if (attach_part) - partition_id = field.safeGet(); + partition_id = typeid_cast(*partition).value.safeGet(); else - partition_id = data.getPartitionIDFromQuery(field); + partition_id = data.getPartitionIDFromQuery(partition, context); String source_dir = "detached/"; @@ -3219,9 +3221,9 @@ void StorageReplicatedMergeTree::getReplicaDelays(time_t & out_absolute_delay, t } -void StorageReplicatedMergeTree::fetchPartition(const Field & partition, const String & from_, const Settings & settings) +void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const String & from_, const Context & context) { - String partition_id = data.getPartitionIDFromQuery(partition); + String partition_id = data.getPartitionIDFromQuery(partition, context); String from = from_; if (from.back() == '/') @@ -3373,20 +3375,15 @@ void StorageReplicatedMergeTree::fetchPartition(const Field & partition, const S } -void StorageReplicatedMergeTree::freezePartition(const Field & partition, const String & with_name, const Settings & settings) +void StorageReplicatedMergeTree::freezePartition(const ASTPtr & partition, const String & with_name, const Context & context) { - /// The prefix can be arbitrary. Not necessarily a month - you can specify only a year. - String prefix = partition.getType() == Field::Types::UInt64 - ? toString(partition.get()) - : partition.safeGet(); - - data.freezePartition(prefix, with_name); + data.freezePartition(partition, with_name, context); } void StorageReplicatedMergeTree::reshardPartitions( const ASTPtr & query, const String & database_name, - const Field & partition, + const ASTPtr & partition, const WeightedZooKeeperPaths & weighted_zookeeper_paths, const ASTPtr & sharding_key_expr, bool do_copy, const Field & coordinator, const Context & context) @@ -3466,8 +3463,8 @@ void StorageReplicatedMergeTree::reshardPartitions( throw Exception{"Shard paths must be distinct", ErrorCodes::DUPLICATE_SHARD_PATHS}; } - bool include_all = partition.isNull(); - String partition_id = !partition.isNull() ? data.getPartitionIDFromQuery(partition) : String(); + bool include_all = !partition; + String partition_id = partition ? data.getPartitionIDFromQuery(partition, context) : String(); /// Make a list of local partitions that need to be resharded. std::set unique_partition_list; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index ce0489124de..8fc9a9ce754 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -133,19 +133,19 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override; - bool optimize(const ASTPtr & query, const String & partition_id, bool final, bool deduplicate, const Settings & settings) override; + bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override; void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override; - void clearColumnInPartition(const ASTPtr & query, const Field & partition, const Field & column_name, const Settings & settings) override; - void dropPartition(const ASTPtr & query, const Field & partition, bool detach, const Settings & settings) override; - void attachPartition(const ASTPtr & query, const Field & partition, bool part, const Settings & settings) override; - void fetchPartition(const Field & partition, const String & from, const Settings & settings) override; - void freezePartition(const Field & partition, const String & with_name, const Settings & settings) override; + void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context) override; + void dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & context) override; + void attachPartition(const ASTPtr & partition, bool part, const Context & context) override; + void fetchPartition(const ASTPtr & partition, const String & from, const Context & context) override; + void freezePartition(const ASTPtr & partition, const String & with_name, const Context & context) override; void reshardPartitions( const ASTPtr & query, const String & database_name, - const Field & partition, + const ASTPtr & partition, const WeightedZooKeeperPaths & weighted_zookeeper_paths, const ASTPtr & sharding_key_expr, bool do_copy, const Field & coordinator, const Context & context) override; diff --git a/dbms/src/Storages/StorageTrivialBuffer.cpp b/dbms/src/Storages/StorageTrivialBuffer.cpp index 06e693b3494..271af6e1d8f 100644 --- a/dbms/src/Storages/StorageTrivialBuffer.cpp +++ b/dbms/src/Storages/StorageTrivialBuffer.cpp @@ -380,9 +380,9 @@ void StorageTrivialBuffer::shutdown() * * This kind of race condition make very hard to implement proper tests. */ -bool StorageTrivialBuffer::optimize(const ASTPtr & query, const String & partition_id, bool final, bool deduplicate, const Settings & settings) +bool StorageTrivialBuffer::optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) { - if (!partition_id.empty()) + if (partition) throw Exception("Partition cannot be specified when optimizing table of type TrivialBuffer", ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Storages/StorageTrivialBuffer.h b/dbms/src/Storages/StorageTrivialBuffer.h index c0e4753b24a..b1ac18360ab 100644 --- a/dbms/src/Storages/StorageTrivialBuffer.h +++ b/dbms/src/Storages/StorageTrivialBuffer.h @@ -69,7 +69,7 @@ public: void startup() override; /// Writes all the blocks in buffer into the destination table. Stop flushing thread. void shutdown() override; - bool optimize(const ASTPtr & query, const String & partition_id, bool final, bool deduplicate, const Settings & settings) override; + bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override; void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override { name = new_table_name; }