From 3effb74d317ac88809de1e9dd6fb3574d5984c1e Mon Sep 17 00:00:00 2001 From: songenjie Date: Thu, 8 Apr 2021 11:29:36 +0800 Subject: [PATCH 01/15] [clickhouse][server][ddl] add fetch part docs --- .../sql-reference/statements/alter/partition.md | 16 +++++++++++----- docs/en/sql-reference/statements/grant.md | 2 +- .../en/2016/how-to-update-data-in-clickhouse.md | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index f7183ba525c..23f10684ae7 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -16,7 +16,7 @@ The following operations with [partitions](../../../engines/table-engines/merget - [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) — Resets the value of a specified column in a partition. - [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) — Resets the specified secondary index in a partition. - [FREEZE PARTITION](#alter_freeze-partition) — Creates a backup of a partition. -- [FETCH PARTITION](#alter_fetch-partition) — Downloads a partition from another server. +- [FETCH PART\|PARTITION](#alter_fetch-partition) — Downloads a part or partition from another server. - [MOVE PARTITION\|PART](#alter_move-partition) — Move partition/data part to another disk or volume. @@ -198,29 +198,35 @@ ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr The query works similar to `CLEAR COLUMN`, but it resets an index instead of a column data. -## FETCH PARTITION {#alter_fetch-partition} +## FETCH PART|PARTITION {#alter_fetch-partition} ``` sql -ALTER TABLE table_name FETCH PARTITION partition_expr FROM 'path-in-zookeeper' +ALTER TABLE table_name FETCH PART|PARTITION partition_expr FROM 'path-in-zookeeper' ``` Downloads a partition from another server. This query only works for the replicated tables. The query does the following: -1. Downloads the partition from the specified shard. In ‘path-in-zookeeper’ you must specify a path to the shard in ZooKeeper. +1. Downloads the partition|part from the specified shard. In ‘path-in-zookeeper’ you must specify a path to the shard in ZooKeeper. 2. Then the query puts the downloaded data to the `detached` directory of the `table_name` table. Use the [ATTACH PARTITION\|PART](#alter_attach-partition) query to add the data to the table. For example: +1. FETCH PARTITION ``` sql ALTER TABLE users FETCH PARTITION 201902 FROM '/clickhouse/tables/01-01/visits'; ALTER TABLE users ATTACH PARTITION 201902; ``` +2. FETCH PART +``` sql +ALTER TABLE users FETCH PART 201901_2_2_0 FROM '/clickhouse/tables/01-01/visits'; +ALTER TABLE users ATTACH PART 201901_2_2_0; +``` Note that: -- The `ALTER ... FETCH PARTITION` query isn’t replicated. It places the partition to the `detached` directory only on the local server. +- The `ALTER ... FETCH PART|PARTITION` query isn’t replicated. It places the part or partition to the `detached` directory only on the local server. - The `ALTER TABLE ... ATTACH` query is replicated. It adds the data to all replicas. The data is added to one of the replicas from the `detached` directory, and to the others - from neighboring replicas. Before downloading, the system checks if the partition exists and the table structure matches. The most appropriate replica is selected automatically from the healthy replicas. diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 0afc9b5b95f..daa020f9469 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -279,7 +279,7 @@ Allows executing [ALTER](../../sql-reference/statements/alter/index.md) queries - `ALTER MATERIALIZE TTL`. Level: `TABLE`. Aliases: `MATERIALIZE TTL` - `ALTER SETTINGS`. Level: `TABLE`. Aliases: `ALTER SETTING`, `ALTER MODIFY SETTING`, `MODIFY SETTING` - `ALTER MOVE PARTITION`. Level: `TABLE`. Aliases: `ALTER MOVE PART`, `MOVE PARTITION`, `MOVE PART` - - `ALTER FETCH PARTITION`. Level: `TABLE`. Aliases: `FETCH PARTITION` + - `ALTER FETCH PARTITION`. Level: `TABLE`. Aliases: `FETCH PARTITION`, `FETCH PART` - `ALTER FREEZE PARTITION`. Level: `TABLE`. Aliases: `FREEZE PARTITION` - `ALTER VIEW` Level: `GROUP` - `ALTER VIEW REFRESH`. Level: `VIEW`. Aliases: `ALTER LIVE VIEW REFRESH`, `REFRESH VIEW` diff --git a/website/blog/en/2016/how-to-update-data-in-clickhouse.md b/website/blog/en/2016/how-to-update-data-in-clickhouse.md index 22c2fa3ccc1..7e9b938203f 100644 --- a/website/blog/en/2016/how-to-update-data-in-clickhouse.md +++ b/website/blog/en/2016/how-to-update-data-in-clickhouse.md @@ -67,7 +67,7 @@ There is a nice set of operations to work with partitions: - `DROP PARTITION` - Delete a partition. - `ATTACH PART|PARTITION` -- Add a new part or partition from the 'detached' directory to the table. - `FREEZE PARTITION` - Create a backup of a partition. -- `FETCH PARTITION` - Download a partition from another server. +- `FETCH PART|PARTITION` - Download a part or partition from another server. We can do any data management operations on partitions level: move, copy and delete. Also, special DETACH and ATTACH operations are created to simplify data manipulation. DETACH detaches partition from table, moving all data to detached directory. Data is still there and you can copy it anywhere but detached data is not visible on request level. ATTACH is the opposite: attaches data from detached directory so it become visible. From 564136ec46ed7847ff296869c8b6156ee202f34b Mon Sep 17 00:00:00 2001 From: songenjie Date: Tue, 13 Apr 2021 12:40:33 +0800 Subject: [PATCH 02/15] [clickhouse][server][dll][alter]support fetch part --- src/Access/AccessType.h | 2 +- src/Common/ErrorCodes.cpp | 2 + src/Parsers/ASTAlterQuery.cpp | 2 +- src/Parsers/ParserAlterQuery.cpp | 16 +++ src/Storages/MergeTree/MergeTreeData.cpp | 9 +- src/Storages/MergeTree/MergeTreeData.h | 7 +- src/Storages/PartitionCommands.cpp | 6 +- src/Storages/StorageReplicatedMergeTree.cpp | 102 ++++++++++++++---- src/Storages/StorageReplicatedMergeTree.h | 10 +- tests/fuzz/ast.dict | 1 + .../__init__.py | 0 .../configs/zookeeper_config.xml | 28 +++++ .../test.py | 40 +++++++ .../test.py | 2 +- .../rbac/tests/privileges/grant_option.py | 2 +- 15 files changed, 198 insertions(+), 31 deletions(-) create mode 100644 tests/integration/test_fetch_part_from_auxiliary_zookeeper/__init__.py create mode 100644 tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml create mode 100644 tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py diff --git a/src/Access/AccessType.h b/src/Access/AccessType.h index 40740b3164e..c7311997ba2 100644 --- a/src/Access/AccessType.h +++ b/src/Access/AccessType.h @@ -62,7 +62,7 @@ enum class AccessType enabled implicitly by the grant ALTER_TABLE */\ M(ALTER_SETTINGS, "ALTER SETTING, ALTER MODIFY SETTING, MODIFY SETTING", TABLE, ALTER_TABLE) /* allows to execute ALTER MODIFY SETTING */\ M(ALTER_MOVE_PARTITION, "ALTER MOVE PART, MOVE PARTITION, MOVE PART", TABLE, ALTER_TABLE) \ - M(ALTER_FETCH_PARTITION, "FETCH PARTITION", TABLE, ALTER_TABLE) \ + M(ALTER_FETCH_PARTITION, "ALTER FETCH PART, FETCH PARTITION, FETCH PART", TABLE, ALTER_TABLE) \ M(ALTER_FREEZE_PARTITION, "FREEZE PARTITION, UNFREEZE", TABLE, ALTER_TABLE) \ \ M(ALTER_TABLE, "", GROUP, ALTER) \ diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index ad0463db889..7658448976b 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -549,6 +549,8 @@ M(579, INCORRECT_PART_TYPE) \ M(580, CANNOT_SET_ROUNDING_MODE) \ M(581, TOO_LARGE_DISTRIBUTED_DEPTH) \ + M(582, PART_DOESNT_EXIST) \ + M(583, PART_ALREADY_EXISTS) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index df4a9a5f99a..5b052bae856 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -245,7 +245,7 @@ void ASTAlterCommand::formatImpl( else if (type == ASTAlterCommand::FETCH_PARTITION) { settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "FETCH " - << "PARTITION " << (settings.hilite ? hilite_none : ""); + << (part ? "PART " : "PARTITION ") << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << DB::quote << from; diff --git a/src/Parsers/ParserAlterQuery.cpp b/src/Parsers/ParserAlterQuery.cpp index e5cc4b1b95e..de524342fb4 100644 --- a/src/Parsers/ParserAlterQuery.cpp +++ b/src/Parsers/ParserAlterQuery.cpp @@ -61,6 +61,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_drop_detached_partition("DROP DETACHED PARTITION"); ParserKeyword s_drop_detached_part("DROP DETACHED PART"); ParserKeyword s_fetch_partition("FETCH PARTITION"); + ParserKeyword s_fetch_part("FETCH PART"); ParserKeyword s_replace_partition("REPLACE PARTITION"); ParserKeyword s_freeze("FREEZE"); ParserKeyword s_unfreeze("UNFREEZE"); @@ -428,6 +429,21 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected command->from = ast_from->as().value.get(); command->type = ASTAlterCommand::FETCH_PARTITION; } + else if (s_fetch_part.ignore(pos, expected)) + { + if (!parser_string_literal.parse(pos, command->partition, expected)) + return false; + + if (!s_from.ignore(pos, expected)) + return false; + + ASTPtr ast_from; + if (!parser_string_literal.parse(pos, ast_from, expected)) + return false; + command->from = ast_from->as().value.get(); + command->part = true; + command->type = ASTAlterCommand::FETCH_PARTITION; + } else if (s_freeze.ignore(pos, expected)) { if (s_partition.ignore(pos, expected)) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index ee8e15008cb..b5da72c517f 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2909,7 +2909,12 @@ void MergeTreeData::movePartitionToVolume(const ASTPtr & partition, const String throw Exception("Cannot move parts because moves are manually disabled", ErrorCodes::ABORTED); } -void MergeTreeData::fetchPartition(const ASTPtr & /*partition*/, const StorageMetadataPtr & /*metadata_snapshot*/, const String & /*from*/, ContextPtr /*query_context*/) +void MergeTreeData::fetchPartition( + const ASTPtr & /*partition*/, + const StorageMetadataPtr & /*metadata_snapshot*/, + const String & /*from*/, + bool /*fetch_part*/, + ContextPtr /*query_context*/) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "FETCH PARTITION is not supported by storage {}", getName()); } @@ -2972,7 +2977,7 @@ Pipe MergeTreeData::alterPartition( break; case PartitionCommand::FETCH_PARTITION: - fetchPartition(command.partition, metadata_snapshot, command.from_zookeeper_path, query_context); + fetchPartition(command.partition, metadata_snapshot, command.from_zookeeper_path, command.part, query_context); break; case PartitionCommand::FREEZE_PARTITION: diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 6c0bda07bb1..46c0014d9f7 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -970,7 +970,12 @@ protected: virtual void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr context) = 0; /// Makes sense only for replicated tables - virtual void fetchPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, const String & from, ContextPtr query_context); + virtual void fetchPartition( + const ASTPtr & partition, + const StorageMetadataPtr & metadata_snapshot, + const String & from, + bool fetch_part, + ContextPtr query_context); void writePartLog( PartLogElement::Type type, diff --git a/src/Storages/PartitionCommands.cpp b/src/Storages/PartitionCommands.cpp index e51a64d5d81..f09f60887e8 100644 --- a/src/Storages/PartitionCommands.cpp +++ b/src/Storages/PartitionCommands.cpp @@ -82,6 +82,7 @@ std::optional PartitionCommand::parse(const ASTAlterCommand * res.type = FETCH_PARTITION; res.partition = command_ast->partition; res.from_zookeeper_path = command_ast->from; + res.part = command_ast->part; return res; } else if (command_ast->type == ASTAlterCommand::FREEZE_PARTITION) @@ -140,7 +141,10 @@ std::string PartitionCommand::typeToString() const else return "DROP DETACHED PARTITION"; case PartitionCommand::Type::FETCH_PARTITION: - return "FETCH PARTITION"; + if (part) + return "FETCH PART"; + else + return "FETCH PARTITION"; case PartitionCommand::Type::FREEZE_ALL_PARTITIONS: return "FREEZE ALL"; case PartitionCommand::Type::FREEZE_PARTITION: diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a2b1f1737a2..2d836e00a08 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -112,9 +112,11 @@ namespace ErrorCodes extern const int NOT_A_LEADER; extern const int TABLE_WAS_NOT_DROPPED; extern const int PARTITION_ALREADY_EXISTS; + extern const int PART_ALREADY_EXISTS; extern const int TOO_MANY_RETRIES_TO_FETCH_PARTS; extern const int RECEIVED_ERROR_FROM_REMOTE_IO_SERVER; extern const int PARTITION_DOESNT_EXIST; + extern const int PART_DOESNT_EXIST; extern const int UNFINISHED; extern const int RECEIVED_ERROR_TOO_MANY_REQUESTS; extern const int TOO_MANY_FETCHES; @@ -5356,11 +5358,11 @@ void StorageReplicatedMergeTree::getReplicaDelays(time_t & out_absolute_delay, t } } - void StorageReplicatedMergeTree::fetchPartition( const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, const String & from_, + bool fetch_part, ContextPtr query_context) { Macros::MacroExpansionInfo info; @@ -5373,40 +5375,54 @@ void StorageReplicatedMergeTree::fetchPartition( if (from.empty()) throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - String partition_id = getPartitionIDFromQuery(partition, query_context); zkutil::ZooKeeperPtr zookeeper; if (auxiliary_zookeeper_name != default_zookeeper_name) - { zookeeper = getContext()->getAuxiliaryZooKeeper(auxiliary_zookeeper_name); - - LOG_INFO(log, "Will fetch partition {} from shard {} (auxiliary zookeeper '{}')", partition_id, from_, auxiliary_zookeeper_name); - } else - { zookeeper = getZooKeeper(); - LOG_INFO(log, "Will fetch partition {} from shard {}", partition_id, from_); - } - if (from.back() == '/') from.resize(from.size() - 1); + if (fetch_part) + { + String part_name = partition->as().value.safeGet(); + auto replica_path_ = findReplicaHavingPart(part_name, from, zookeeper); + + if (replica_path_.empty()) + throw Exception("fetch part " + part_name + " not exists !", ErrorCodes::PART_DOESNT_EXIST); + /** Let's check that there is no such part in the `detached` directory (where we will write the downloaded parts). + * Unreliable (there is a race condition) - such a part may appear a little later. + */ + if (checkDetachPartIfExists(part_name)) + throw Exception("Detached part " + part_name + " already exists.", ErrorCodes::PART_ALREADY_EXISTS); + LOG_INFO(log, "Will fetch part {} from shard {} (zookeeper '{}')", part_name, from_, auxiliary_zookeeper_name); + + try + { + /// part name , metadata, replica path , true, 0, zookeeper + if (!fetchPart(part_name, metadata_snapshot, replica_path_, true, 0, zookeeper)) + throw Exception("fetch part " + part_name + " failed! ", ErrorCodes::UNFINISHED); + } + catch (const DB::Exception & e) + { + if (e.code() != ErrorCodes::RECEIVED_ERROR_FROM_REMOTE_IO_SERVER && e.code() != ErrorCodes::RECEIVED_ERROR_TOO_MANY_REQUESTS + && e.code() != ErrorCodes::CANNOT_READ_ALL_DATA) + throw; + + LOG_INFO(log, e.displayText()); + } + return; + } + + String partition_id = getPartitionIDFromQuery(partition, query_context); + LOG_INFO(log, "Will fetch partition {} from shard {} (zookeeper '{}')", partition_id, from_, auxiliary_zookeeper_name); /** Let's check that there is no such partition in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a partition may appear a little later. */ - Poco::DirectoryIterator dir_end; - for (const std::string & path : getDataPaths()) - { - for (Poco::DirectoryIterator dir_it{path + "detached/"}; dir_it != dir_end; ++dir_it) - { - MergeTreePartInfo part_info; - if (MergeTreePartInfo::tryParsePartName(dir_it.name(), &part_info, format_version) - && part_info.partition_id == partition_id) - throw Exception("Detached partition " + partition_id + " already exists.", ErrorCodes::PARTITION_ALREADY_EXISTS); - } - - } + if (checkDetachPartitionIfExists(partition_id)) + throw Exception("Detached partition " + partition_id + " already exists.", ErrorCodes::PARTITION_ALREADY_EXISTS); zkutil::Strings replicas; zkutil::Strings active_replicas; @@ -6913,4 +6929,46 @@ String StorageReplicatedMergeTree::getSharedDataReplica( return best_replica; } +String StorageReplicatedMergeTree::findReplicaHavingPart( + const String & part_name, const String & zookeeper_path_, zkutil::ZooKeeper::Ptr zookeeper_) +{ + Strings replicas = zookeeper_->getChildren(zookeeper_path_ + "/replicas"); + + /// Select replicas in uniformly random order. + std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); + + for (const String & replica : replicas) + { + if (zookeeper_->exists(zookeeper_path_ + "/replicas/" + replica + "/parts/" + part_name) + && zookeeper_->exists(zookeeper_path_ + "/replicas/" + replica + "/is_active")) + return zookeeper_path_ + "/replicas/" + replica; + } + + return {}; +} + +bool StorageReplicatedMergeTree::checkDetachPartIfExists(const String & part_name) +{ + Poco::DirectoryIterator dir_end; + for (const std::string & path : getDataPaths()) + for (Poco::DirectoryIterator dir_it{path + "detached/"}; dir_it != dir_end; ++dir_it) + if (dir_it.name() == part_name) + return true; + return false; +} + +bool StorageReplicatedMergeTree::checkDetachPartitionIfExists(const String & partition_name) +{ + Poco::DirectoryIterator dir_end; + for (const std::string & path : getDataPaths()) + { + for (Poco::DirectoryIterator dir_it{path + "detached/"}; dir_it != dir_end; ++dir_it) + { + MergeTreePartInfo part_info; + if (MergeTreePartInfo::tryParsePartName(dir_it.name(), &part_info, format_version) && part_info.partition_id == partition_name) + return true; + } + } + return false; +} } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 0180fa8bc1a..673c713fabc 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -522,8 +522,11 @@ private: /** Returns an empty string if no one has a part. */ String findReplicaHavingPart(const String & part_name, bool active); + String findReplicaHavingPart(const String & part_name, const String & zookeeper_path_, zkutil::ZooKeeper::Ptr zookeeper_); bool checkReplicaHavePart(const String & replica, const String & part_name); + bool checkDetachPartIfExists(const String & part_name); + bool checkDetachPartitionIfExists(const String & partition_name); /** Find replica having specified part or any part that covers it. * If active = true, consider only active replicas. @@ -626,7 +629,12 @@ private: PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, bool part, ContextPtr query_context) override; void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, ContextPtr query_context) override; void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr query_context) override; - void fetchPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, const String & from, ContextPtr query_context) override; + void fetchPartition( + const ASTPtr & partition, + const StorageMetadataPtr & metadata_snapshot, + const String & from, + bool fetch_part, + ContextPtr query_context) override; /// Check granularity of already existing replicated table in zookeeper if it exists /// return true if it's fixed diff --git a/tests/fuzz/ast.dict b/tests/fuzz/ast.dict index 8327f276b31..7befb36c840 100644 --- a/tests/fuzz/ast.dict +++ b/tests/fuzz/ast.dict @@ -156,6 +156,7 @@ "extractURLParameterNames" "extractURLParameters" "FETCH PARTITION" +"FETCH PART" "FINAL" "FIRST" "firstSignificantSubdomain" diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/__init__.py b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml new file mode 100644 index 00000000000..b2b0667ebbf --- /dev/null +++ b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml @@ -0,0 +1,28 @@ + + + + zoo1 + 2181 + + + zoo2 + 2181 + + + zoo3 + 2181 + + + + + + zoo1 + 2181 + + + zoo2 + 2181 + + + + diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py new file mode 100644 index 00000000000..3d13da49a63 --- /dev/null +++ b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py @@ -0,0 +1,40 @@ + + +import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + + yield cluster + finally: + cluster.shutdown() + + +def test_fetch_part_from_allowed_zookeeper(start_cluster): + node.query( + "CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" + ) + node.query("INSERT INTO simple VALUES ('2020-08-27', 1)") + + node.query( + "CREATE TABLE simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;" + ) + node.query( + "ALTER TABLE simple2 FETCH PART '20200827_1_1_0' FROM 'zookeeper2:/clickhouse/tables/0/simple';" + ) + node.query("ALTER TABLE simple2 ATTACH PART '20200827_1_1_0';") + + with pytest.raises(QueryRuntimeException): + node.query( + "ALTER TABLE simple2 FETCH PART '20200827_1_1_0' FROM 'zookeeper:/clickhouse/tables/0/simple';" + ) + + assert node.query("SELECT id FROM simple2").strip() == "1" diff --git a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py index f9c10d68fe3..0c94dfd3c48 100644 --- a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py +++ b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py @@ -18,7 +18,7 @@ def start_cluster(): cluster.shutdown() -def test_fetch_part_from_allowed_zookeeper(start_cluster): +def test_fetch_partition_from_allowed_zookeeper(start_cluster): node.query( "CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) diff --git a/tests/testflows/rbac/tests/privileges/grant_option.py b/tests/testflows/rbac/tests/privileges/grant_option.py index f337aec2619..a28350a78b9 100644 --- a/tests/testflows/rbac/tests/privileges/grant_option.py +++ b/tests/testflows/rbac/tests/privileges/grant_option.py @@ -89,7 +89,7 @@ def grant_option_check(grant_option_target, grant_target, user_name, table_type, @Examples("privilege", [ ("ALTER MOVE PARTITION",), ("ALTER MOVE PART",), ("MOVE PARTITION",), ("MOVE PART",), ("ALTER DELETE",), ("DELETE",), - ("ALTER FETCH PARTITION",), ("FETCH PARTITION",), + ("ALTER FETCH PARTITION",), ("ALTER FETCH PART",), ("FETCH PARTITION",), ("FETCH PART",), ("ALTER FREEZE PARTITION",), ("FREEZE PARTITION",), ("ALTER UPDATE",), ("UPDATE",), ("ALTER ADD COLUMN",), ("ADD COLUMN",), From fbd26789afe5b2aac3c9a808a24049dcd465755f Mon Sep 17 00:00:00 2001 From: songenjie Date: Tue, 13 Apr 2021 14:08:26 +0800 Subject: [PATCH 03/15] [test] show privileges --- tests/queries/0_stateless/01271_show_privileges.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 7928f531a7d..e2784f1dfd5 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -28,7 +28,7 @@ ALTER TTL ['ALTER MODIFY TTL','MODIFY TTL'] TABLE ALTER TABLE ALTER MATERIALIZE TTL ['MATERIALIZE TTL'] TABLE ALTER TABLE ALTER SETTINGS ['ALTER SETTING','ALTER MODIFY SETTING','MODIFY SETTING'] TABLE ALTER TABLE ALTER MOVE PARTITION ['ALTER MOVE PART','MOVE PARTITION','MOVE PART'] TABLE ALTER TABLE -ALTER FETCH PARTITION ['FETCH PARTITION'] TABLE ALTER TABLE +ALTER FETCH PARTITION ['ALTER FETCH PART','FETCH PARTITION','FETCH PART'] TABLE ALTER TABLE ALTER FREEZE PARTITION ['FREEZE PARTITION','UNFREEZE'] TABLE ALTER TABLE ALTER TABLE [] \N ALTER ALTER VIEW REFRESH ['ALTER LIVE VIEW REFRESH','REFRESH VIEW'] VIEW ALTER VIEW From 520f4f39eccfcc16491fbc58d9cb44856deb5f53 Mon Sep 17 00:00:00 2001 From: songenjie Date: Tue, 13 Apr 2021 15:17:03 +0800 Subject: [PATCH 04/15] [test][integration] fetch part --- .../test_fetch_part_from_auxiliary_zookeeper/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py index 3d13da49a63..17617f1c45c 100644 --- a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py +++ b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py @@ -28,13 +28,13 @@ def test_fetch_part_from_allowed_zookeeper(start_cluster): "CREATE TABLE simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) node.query( - "ALTER TABLE simple2 FETCH PART '20200827_1_1_0' FROM 'zookeeper2:/clickhouse/tables/0/simple';" + "ALTER TABLE simple2 FETCH PART '20200827_0_0_0' FROM 'zookeeper2:/clickhouse/tables/0/simple';" ) - node.query("ALTER TABLE simple2 ATTACH PART '20200827_1_1_0';") + node.query("ALTER TABLE simple2 ATTACH PART '20200827_0_0_0';") with pytest.raises(QueryRuntimeException): node.query( - "ALTER TABLE simple2 FETCH PART '20200827_1_1_0' FROM 'zookeeper:/clickhouse/tables/0/simple';" + "ALTER TABLE simple2 FETCH PART '20200827_0_0_0' FROM 'zookeeper:/clickhouse/tables/0/simple';" ) assert node.query("SELECT id FROM simple2").strip() == "1" From 7c32cc1e18e364553ed31a76ca24c280df45e025 Mon Sep 17 00:00:00 2001 From: songenjie Date: Tue, 13 Apr 2021 17:34:04 +0800 Subject: [PATCH 05/15] fix case style for local variable --- src/Storages/StorageReplicatedMergeTree.cpp | 8 ++++---- src/Storages/StorageReplicatedMergeTree.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 2d836e00a08..98f1ee5560e 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5387,9 +5387,9 @@ void StorageReplicatedMergeTree::fetchPartition( if (fetch_part) { String part_name = partition->as().value.safeGet(); - auto replica_path_ = findReplicaHavingPart(part_name, from, zookeeper); + auto part_path = findReplicaHavingPart(part_name, from, zookeeper); - if (replica_path_.empty()) + if (part_path.empty()) throw Exception("fetch part " + part_name + " not exists !", ErrorCodes::PART_DOESNT_EXIST); /** Let's check that there is no such part in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a part may appear a little later. @@ -5400,8 +5400,8 @@ void StorageReplicatedMergeTree::fetchPartition( try { - /// part name , metadata, replica path , true, 0, zookeeper - if (!fetchPart(part_name, metadata_snapshot, replica_path_, true, 0, zookeeper)) + /// part name , metadata, part_path , true, 0, zookeeper + if (!fetchPart(part_name, metadata_snapshot, part_path, true, 0, zookeeper)) throw Exception("fetch part " + part_name + " failed! ", ErrorCodes::UNFINISHED); } catch (const DB::Exception & e) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 673c713fabc..bc59a1bdf5f 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -522,7 +522,7 @@ private: /** Returns an empty string if no one has a part. */ String findReplicaHavingPart(const String & part_name, bool active); - String findReplicaHavingPart(const String & part_name, const String & zookeeper_path_, zkutil::ZooKeeper::Ptr zookeeper_); + static String findReplicaHavingPart(const String & part_name, const String & zookeeper_path_, zkutil::ZooKeeper::Ptr zookeeper_); bool checkReplicaHavePart(const String & replica, const String & part_name); bool checkDetachPartIfExists(const String & part_name); From f731739fab70c31bf02bc41af35db9d0a3115808 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 10:05:41 +0800 Subject: [PATCH 06/15] fix suggests --- .../sql-reference/statements/alter/partition.md | 8 ++++---- src/Access/AccessType.h | 2 +- src/Common/ErrorCodes.cpp | 2 -- src/Storages/StorageReplicatedMergeTree.cpp | 15 +++++++-------- src/Storages/StorageReplicatedMergeTree.h | 4 ++-- .../0_stateless/01271_show_privileges.reference | 2 +- 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index 23f10684ae7..948711e6d9e 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -16,7 +16,7 @@ The following operations with [partitions](../../../engines/table-engines/merget - [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) — Resets the value of a specified column in a partition. - [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) — Resets the specified secondary index in a partition. - [FREEZE PARTITION](#alter_freeze-partition) — Creates a backup of a partition. -- [FETCH PART\|PARTITION](#alter_fetch-partition) — Downloads a part or partition from another server. +- [FETCH PARTITION\|PART](#alter_fetch-partition) — Downloads a part or partition from another server. - [MOVE PARTITION\|PART](#alter_move-partition) — Move partition/data part to another disk or volume. @@ -198,10 +198,10 @@ ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr The query works similar to `CLEAR COLUMN`, but it resets an index instead of a column data. -## FETCH PART|PARTITION {#alter_fetch-partition} +## FETCH PARTITION|PART {#alter_fetch-partition} ``` sql -ALTER TABLE table_name FETCH PART|PARTITION partition_expr FROM 'path-in-zookeeper' +ALTER TABLE table_name FETCH PARTITION|PART partition_expr FROM 'path-in-zookeeper' ``` Downloads a partition from another server. This query only works for the replicated tables. @@ -226,7 +226,7 @@ ALTER TABLE users ATTACH PART 201901_2_2_0; Note that: -- The `ALTER ... FETCH PART|PARTITION` query isn’t replicated. It places the part or partition to the `detached` directory only on the local server. +- The `ALTER ... FETCH PARTITION|PART` query isn’t replicated. It places the part or partition to the `detached` directory only on the local server. - The `ALTER TABLE ... ATTACH` query is replicated. It adds the data to all replicas. The data is added to one of the replicas from the `detached` directory, and to the others - from neighboring replicas. Before downloading, the system checks if the partition exists and the table structure matches. The most appropriate replica is selected automatically from the healthy replicas. diff --git a/src/Access/AccessType.h b/src/Access/AccessType.h index c7311997ba2..952cddba5f5 100644 --- a/src/Access/AccessType.h +++ b/src/Access/AccessType.h @@ -62,7 +62,7 @@ enum class AccessType enabled implicitly by the grant ALTER_TABLE */\ M(ALTER_SETTINGS, "ALTER SETTING, ALTER MODIFY SETTING, MODIFY SETTING", TABLE, ALTER_TABLE) /* allows to execute ALTER MODIFY SETTING */\ M(ALTER_MOVE_PARTITION, "ALTER MOVE PART, MOVE PARTITION, MOVE PART", TABLE, ALTER_TABLE) \ - M(ALTER_FETCH_PARTITION, "ALTER FETCH PART, FETCH PARTITION, FETCH PART", TABLE, ALTER_TABLE) \ + M(ALTER_FETCH_PARTITION, "ALTER FETCH PART, FETCH PARTITION", TABLE, ALTER_TABLE) \ M(ALTER_FREEZE_PARTITION, "FREEZE PARTITION, UNFREEZE", TABLE, ALTER_TABLE) \ \ M(ALTER_TABLE, "", GROUP, ALTER) \ diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 7658448976b..ad0463db889 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -549,8 +549,6 @@ M(579, INCORRECT_PART_TYPE) \ M(580, CANNOT_SET_ROUNDING_MODE) \ M(581, TOO_LARGE_DISTRIBUTED_DEPTH) \ - M(582, PART_DOESNT_EXIST) \ - M(583, PART_ALREADY_EXISTS) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 98f1ee5560e..977ade9f1b8 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -112,11 +112,9 @@ namespace ErrorCodes extern const int NOT_A_LEADER; extern const int TABLE_WAS_NOT_DROPPED; extern const int PARTITION_ALREADY_EXISTS; - extern const int PART_ALREADY_EXISTS; extern const int TOO_MANY_RETRIES_TO_FETCH_PARTS; extern const int RECEIVED_ERROR_FROM_REMOTE_IO_SERVER; extern const int PARTITION_DOESNT_EXIST; - extern const int PART_DOESNT_EXIST; extern const int UNFINISHED; extern const int RECEIVED_ERROR_TOO_MANY_REQUESTS; extern const int TOO_MANY_FETCHES; @@ -132,6 +130,7 @@ namespace ErrorCodes extern const int UNKNOWN_POLICY; extern const int NO_SUCH_DATA_PART; extern const int INTERSERVER_SCHEME_DOESNT_MATCH; + extern const int DUPLICATED_PART_UUIDS; } namespace ActionLocks @@ -5390,12 +5389,12 @@ void StorageReplicatedMergeTree::fetchPartition( auto part_path = findReplicaHavingPart(part_name, from, zookeeper); if (part_path.empty()) - throw Exception("fetch part " + part_name + " not exists !", ErrorCodes::PART_DOESNT_EXIST); + throw Exception("fetch part " + part_name + " not exists !", ErrorCodes::NO_REPLICA_HAS_PART); /** Let's check that there is no such part in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a part may appear a little later. */ - if (checkDetachPartIfExists(part_name)) - throw Exception("Detached part " + part_name + " already exists.", ErrorCodes::PART_ALREADY_EXISTS); + if (checkIfDetachedPartExists(part_name)) + throw Exception("Detached part " + part_name + " already exists.", ErrorCodes::DUPLICATED_PART_UUIDS); LOG_INFO(log, "Will fetch part {} from shard {} (zookeeper '{}')", part_name, from_, auxiliary_zookeeper_name); try @@ -5421,7 +5420,7 @@ void StorageReplicatedMergeTree::fetchPartition( /** Let's check that there is no such partition in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a partition may appear a little later. */ - if (checkDetachPartitionIfExists(partition_id)) + if (checkIfDetachedPartitionExists(partition_id)) throw Exception("Detached partition " + partition_id + " already exists.", ErrorCodes::PARTITION_ALREADY_EXISTS); zkutil::Strings replicas; @@ -6947,7 +6946,7 @@ String StorageReplicatedMergeTree::findReplicaHavingPart( return {}; } -bool StorageReplicatedMergeTree::checkDetachPartIfExists(const String & part_name) +bool StorageReplicatedMergeTree::checkIfDetachedPartExists(const String & part_name) { Poco::DirectoryIterator dir_end; for (const std::string & path : getDataPaths()) @@ -6957,7 +6956,7 @@ bool StorageReplicatedMergeTree::checkDetachPartIfExists(const String & part_nam return false; } -bool StorageReplicatedMergeTree::checkDetachPartitionIfExists(const String & partition_name) +bool StorageReplicatedMergeTree::checkIfDetachedPartitionExists(const String & partition_name) { Poco::DirectoryIterator dir_end; for (const std::string & path : getDataPaths()) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index bc59a1bdf5f..9122bdafbf0 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -525,8 +525,8 @@ private: static String findReplicaHavingPart(const String & part_name, const String & zookeeper_path_, zkutil::ZooKeeper::Ptr zookeeper_); bool checkReplicaHavePart(const String & replica, const String & part_name); - bool checkDetachPartIfExists(const String & part_name); - bool checkDetachPartitionIfExists(const String & partition_name); + bool checkIfDetachedPartExists(const String & part_name); + bool checkIfDetachedPartitionExists(const String & partition_name); /** Find replica having specified part or any part that covers it. * If active = true, consider only active replicas. diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index e2784f1dfd5..c8b8662dc3e 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -28,7 +28,7 @@ ALTER TTL ['ALTER MODIFY TTL','MODIFY TTL'] TABLE ALTER TABLE ALTER MATERIALIZE TTL ['MATERIALIZE TTL'] TABLE ALTER TABLE ALTER SETTINGS ['ALTER SETTING','ALTER MODIFY SETTING','MODIFY SETTING'] TABLE ALTER TABLE ALTER MOVE PARTITION ['ALTER MOVE PART','MOVE PARTITION','MOVE PART'] TABLE ALTER TABLE -ALTER FETCH PARTITION ['ALTER FETCH PART','FETCH PARTITION','FETCH PART'] TABLE ALTER TABLE +ALTER FETCH PARTITION ['ALTER FETCH PART','FETCH PARTITION'] TABLE ALTER TABLE ALTER FREEZE PARTITION ['FREEZE PARTITION','UNFREEZE'] TABLE ALTER TABLE ALTER TABLE [] \N ALTER ALTER VIEW REFRESH ['ALTER LIVE VIEW REFRESH','REFRESH VIEW'] VIEW ALTER VIEW From 96a95b05dbc80ab558181725b7a14c198670d5b9 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 10:14:13 +0800 Subject: [PATCH 07/15] fix suggests --- tests/testflows/rbac/tests/privileges/grant_option.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testflows/rbac/tests/privileges/grant_option.py b/tests/testflows/rbac/tests/privileges/grant_option.py index a28350a78b9..bc8b73eb32f 100644 --- a/tests/testflows/rbac/tests/privileges/grant_option.py +++ b/tests/testflows/rbac/tests/privileges/grant_option.py @@ -89,7 +89,7 @@ def grant_option_check(grant_option_target, grant_target, user_name, table_type, @Examples("privilege", [ ("ALTER MOVE PARTITION",), ("ALTER MOVE PART",), ("MOVE PARTITION",), ("MOVE PART",), ("ALTER DELETE",), ("DELETE",), - ("ALTER FETCH PARTITION",), ("ALTER FETCH PART",), ("FETCH PARTITION",), ("FETCH PART",), + ("ALTER FETCH PARTITION",), ("ALTER FETCH PART",), ("FETCH PARTITION",), ("ALTER FREEZE PARTITION",), ("FREEZE PARTITION",), ("ALTER UPDATE",), ("UPDATE",), ("ALTER ADD COLUMN",), ("ADD COLUMN",), From ff8958965a095c5d7d1c67a523e1702c5df57ba6 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 10:19:08 +0800 Subject: [PATCH 08/15] fix some docs --- docs/en/sql-reference/statements/grant.md | 2 +- website/blog/en/2016/how-to-update-data-in-clickhouse.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index daa020f9469..0a5c737b550 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -279,7 +279,7 @@ Allows executing [ALTER](../../sql-reference/statements/alter/index.md) queries - `ALTER MATERIALIZE TTL`. Level: `TABLE`. Aliases: `MATERIALIZE TTL` - `ALTER SETTINGS`. Level: `TABLE`. Aliases: `ALTER SETTING`, `ALTER MODIFY SETTING`, `MODIFY SETTING` - `ALTER MOVE PARTITION`. Level: `TABLE`. Aliases: `ALTER MOVE PART`, `MOVE PARTITION`, `MOVE PART` - - `ALTER FETCH PARTITION`. Level: `TABLE`. Aliases: `FETCH PARTITION`, `FETCH PART` + - `ALTER FETCH PARTITION`. Level: `TABLE`. Aliases: `ALTER FETCH PART`, `FETCH PARTITION`, `FETCH PART` - `ALTER FREEZE PARTITION`. Level: `TABLE`. Aliases: `FREEZE PARTITION` - `ALTER VIEW` Level: `GROUP` - `ALTER VIEW REFRESH`. Level: `VIEW`. Aliases: `ALTER LIVE VIEW REFRESH`, `REFRESH VIEW` diff --git a/website/blog/en/2016/how-to-update-data-in-clickhouse.md b/website/blog/en/2016/how-to-update-data-in-clickhouse.md index 7e9b938203f..ed713db2aee 100644 --- a/website/blog/en/2016/how-to-update-data-in-clickhouse.md +++ b/website/blog/en/2016/how-to-update-data-in-clickhouse.md @@ -67,7 +67,7 @@ There is a nice set of operations to work with partitions: - `DROP PARTITION` - Delete a partition. - `ATTACH PART|PARTITION` -- Add a new part or partition from the 'detached' directory to the table. - `FREEZE PARTITION` - Create a backup of a partition. -- `FETCH PART|PARTITION` - Download a part or partition from another server. +- `FETCH PARTITION|PART` - Download a part or partition from another server. We can do any data management operations on partitions level: move, copy and delete. Also, special DETACH and ATTACH operations are created to simplify data manipulation. DETACH detaches partition from table, moving all data to detached directory. Data is still there and you can copy it anywhere but detached data is not visible on request level. ATTACH is the opposite: attaches data from detached directory so it become visible. From 8566df9b7d4eb6592a496608852145f78bcb24da Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 17:11:59 +0800 Subject: [PATCH 09/15] fix some Suggest --- src/Storages/StorageReplicatedMergeTree.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 977ade9f1b8..d5c12628848 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -130,7 +130,7 @@ namespace ErrorCodes extern const int UNKNOWN_POLICY; extern const int NO_SUCH_DATA_PART; extern const int INTERSERVER_SCHEME_DOESNT_MATCH; - extern const int DUPLICATED_PART_UUIDS; + extern const int DUPLICATE_DATA_PART; } namespace ActionLocks @@ -5394,7 +5394,7 @@ void StorageReplicatedMergeTree::fetchPartition( * Unreliable (there is a race condition) - such a part may appear a little later. */ if (checkIfDetachedPartExists(part_name)) - throw Exception("Detached part " + part_name + " already exists.", ErrorCodes::DUPLICATED_PART_UUIDS); + throw Exception("Detached part " + part_name + " already exists.", ErrorCodes::DUPLICATE_DATA_PART); LOG_INFO(log, "Will fetch part {} from shard {} (zookeeper '{}')", part_name, from_, auxiliary_zookeeper_name); try From 88b4994c65740d7e0f35cfff67edd1ede8f82d18 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 17:49:06 +0800 Subject: [PATCH 10/15] fix some Suggestfix some Suggest --- website/blog/en/2016/how-to-update-data-in-clickhouse.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/blog/en/2016/how-to-update-data-in-clickhouse.md b/website/blog/en/2016/how-to-update-data-in-clickhouse.md index ed713db2aee..22c2fa3ccc1 100644 --- a/website/blog/en/2016/how-to-update-data-in-clickhouse.md +++ b/website/blog/en/2016/how-to-update-data-in-clickhouse.md @@ -67,7 +67,7 @@ There is a nice set of operations to work with partitions: - `DROP PARTITION` - Delete a partition. - `ATTACH PART|PARTITION` -- Add a new part or partition from the 'detached' directory to the table. - `FREEZE PARTITION` - Create a backup of a partition. -- `FETCH PARTITION|PART` - Download a part or partition from another server. +- `FETCH PARTITION` - Download a partition from another server. We can do any data management operations on partitions level: move, copy and delete. Also, special DETACH and ATTACH operations are created to simplify data manipulation. DETACH detaches partition from table, moving all data to detached directory. Data is still there and you can copy it anywhere but detached data is not visible on request level. ATTACH is the opposite: attaches data from detached directory so it become visible. From 1fc040ac58c2cd00d5c6fa7bf64e0875e93e26ea Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 17:54:56 +0800 Subject: [PATCH 11/15] fix some Suggestfix some Suggest --- src/Storages/StorageReplicatedMergeTree.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index d5c12628848..f729f0a5be5 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5389,19 +5389,19 @@ void StorageReplicatedMergeTree::fetchPartition( auto part_path = findReplicaHavingPart(part_name, from, zookeeper); if (part_path.empty()) - throw Exception("fetch part " + part_name + " not exists !", ErrorCodes::NO_REPLICA_HAS_PART); + throw Exception(ErrorCodes::PART_DOESNT_EXIST, "Part {} does not exist on any replica", part_name); /** Let's check that there is no such part in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a part may appear a little later. */ if (checkIfDetachedPartExists(part_name)) - throw Exception("Detached part " + part_name + " already exists.", ErrorCodes::DUPLICATE_DATA_PART); + throw Exception(ErrorCodes::DUPLICATE_DATA_PART, "Detached part " + part_name + " already exists."); LOG_INFO(log, "Will fetch part {} from shard {} (zookeeper '{}')", part_name, from_, auxiliary_zookeeper_name); try { /// part name , metadata, part_path , true, 0, zookeeper if (!fetchPart(part_name, metadata_snapshot, part_path, true, 0, zookeeper)) - throw Exception("fetch part " + part_name + " failed! ", ErrorCodes::UNFINISHED); + throw Exception(ErrorCodes::UNFINISHED, "Failed to fetch part {} from {}", part_name, from_); } catch (const DB::Exception & e) { From 2534ed84d58c914dee424a843d9f32225f47ac25 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 18:40:15 +0800 Subject: [PATCH 12/15] test_fetch_partition_from_auxiliary_zookeeper --- .../__init__.py | 0 .../configs/zookeeper_config.xml | 28 ------------- .../test.py | 40 ------------------- .../test.py | 22 +++++++--- 4 files changed, 16 insertions(+), 74 deletions(-) delete mode 100644 tests/integration/test_fetch_part_from_auxiliary_zookeeper/__init__.py delete mode 100644 tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml delete mode 100644 tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/__init__.py b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml deleted file mode 100644 index b2b0667ebbf..00000000000 --- a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - zoo1 - 2181 - - - zoo2 - 2181 - - - zoo3 - 2181 - - - - - - zoo1 - 2181 - - - zoo2 - 2181 - - - - diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py deleted file mode 100644 index 17617f1c45c..00000000000 --- a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py +++ /dev/null @@ -1,40 +0,0 @@ - - -import pytest -from helpers.client import QueryRuntimeException -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance("node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True) - - -@pytest.fixture(scope="module") -def start_cluster(): - try: - cluster.start() - - yield cluster - finally: - cluster.shutdown() - - -def test_fetch_part_from_allowed_zookeeper(start_cluster): - node.query( - "CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" - ) - node.query("INSERT INTO simple VALUES ('2020-08-27', 1)") - - node.query( - "CREATE TABLE simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;" - ) - node.query( - "ALTER TABLE simple2 FETCH PART '20200827_0_0_0' FROM 'zookeeper2:/clickhouse/tables/0/simple';" - ) - node.query("ALTER TABLE simple2 ATTACH PART '20200827_0_0_0';") - - with pytest.raises(QueryRuntimeException): - node.query( - "ALTER TABLE simple2 FETCH PART '20200827_0_0_0' FROM 'zookeeper:/clickhouse/tables/0/simple';" - ) - - assert node.query("SELECT id FROM simple2").strip() == "1" diff --git a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py index 0c94dfd3c48..d8d240349fc 100644 --- a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py +++ b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py @@ -18,7 +18,14 @@ def start_cluster(): cluster.shutdown() -def test_fetch_partition_from_allowed_zookeeper(start_cluster): +@pytest.mark.parametrize( + ('part', 'part_name'), + [ + ('PARTITION', '2020-08-27'), + ('PART', '20200827_0_0_0'), + ] +) +def test_fetch_part_from_allowed_zookeeper(start_cluster, part, part_name): node.query( "CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) @@ -27,14 +34,17 @@ def test_fetch_partition_from_allowed_zookeeper(start_cluster): node.query( "CREATE TABLE simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) + node.query( - "ALTER TABLE simple2 FETCH PARTITION '2020-08-27' FROM 'zookeeper2:/clickhouse/tables/0/simple';" - ) - node.query("ALTER TABLE simple2 ATTACH PARTITION '2020-08-27';") + """ALTER TABLE simple2 FETCH {part} '{part_name}' FROM 'zookeeper2:/clickhouse/tables/0/simple';""".format( + part=part, part_name=part_name)) + + node.query("""ALTER TABLE simple2 ATTACH {part} '{part_name}';""".format( + part=part, part_name=part_name)) with pytest.raises(QueryRuntimeException): node.query( - "ALTER TABLE simple2 FETCH PARTITION '2020-08-27' FROM 'zookeeper:/clickhouse/tables/0/simple';" - ) + """ALTER TABLE simple2 FETCH {part} '{part_name}' FROM 'zookeeper:/clickhouse/tables/0/simple';""".format( + part=part, part_name=part_name)) assert node.query("SELECT id FROM simple2").strip() == "1" From c06c624fc7ae0acbde4b7daf4f044308e91df359 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 18:55:42 +0800 Subject: [PATCH 13/15] fix build --- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index f729f0a5be5..10061af22e7 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5389,7 +5389,7 @@ void StorageReplicatedMergeTree::fetchPartition( auto part_path = findReplicaHavingPart(part_name, from, zookeeper); if (part_path.empty()) - throw Exception(ErrorCodes::PART_DOESNT_EXIST, "Part {} does not exist on any replica", part_name); + throw Exception(ErrorCodes::NO_REPLICA_HAS_PART, "Part {} does not exist on any replica", part_name); /** Let's check that there is no such part in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a part may appear a little later. */ From 382f702f592345789c071ba0ab28e26f4a247443 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 20:04:59 +0800 Subject: [PATCH 14/15] test add param date --- .../test.py | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py index d8d240349fc..9553b0b64d3 100644 --- a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py +++ b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py @@ -1,5 +1,3 @@ - - import pytest from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster @@ -19,32 +17,32 @@ def start_cluster(): @pytest.mark.parametrize( - ('part', 'part_name'), + ('part', 'date', 'part_name'), [ - ('PARTITION', '2020-08-27'), - ('PART', '20200827_0_0_0'), + ('PARTITION', '2020-08-27', '2020-08-27'), + ('PART', '2020-08-28' '20200828_0_0_0'), ] ) -def test_fetch_part_from_allowed_zookeeper(start_cluster, part, part_name): +def test_fetch_part_from_allowed_zookeeper(start_cluster, part, date, part_name): node.query( - "CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" + "CREATE TABLE IF NOT EXISTS simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) - node.query("INSERT INTO simple VALUES ('2020-08-27', 1)") + + node.query("""INSERT INTO simple VALUES ('{date}', 1)""".format(date=date)) node.query( - "CREATE TABLE simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;" + "CREATE TABLE IF NOT EXISTS simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) node.query( """ALTER TABLE simple2 FETCH {part} '{part_name}' FROM 'zookeeper2:/clickhouse/tables/0/simple';""".format( part=part, part_name=part_name)) - node.query("""ALTER TABLE simple2 ATTACH {part} '{part_name}';""".format( - part=part, part_name=part_name)) + node.query("""ALTER TABLE simple2 ATTACH {part} '{part_name}';""".format(part=part, part_name=part_name)) with pytest.raises(QueryRuntimeException): node.query( """ALTER TABLE simple2 FETCH {part} '{part_name}' FROM 'zookeeper:/clickhouse/tables/0/simple';""".format( part=part, part_name=part_name)) - assert node.query("SELECT id FROM simple2").strip() == "1" + assert node.query("""SELECT id FROM simple2 where date = '{date}'""".format(date=date)).strip() == "1" From ffd3b3d445036afe43bc941ac1b88a9b0f5cad2b Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 21:15:53 +0800 Subject: [PATCH 15/15] fix some docs --- .../test_fetch_partition_from_auxiliary_zookeeper/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py index 9553b0b64d3..7bce2d50011 100644 --- a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py +++ b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py @@ -20,7 +20,7 @@ def start_cluster(): ('part', 'date', 'part_name'), [ ('PARTITION', '2020-08-27', '2020-08-27'), - ('PART', '2020-08-28' '20200828_0_0_0'), + ('PART', '2020-08-28', '20200828_0_0_0'), ] ) def test_fetch_part_from_allowed_zookeeper(start_cluster, part, date, part_name):