From a0683ce460eb103212fbc7e2c89e77d755973536 Mon Sep 17 00:00:00 2001 From: Peng Jian Date: Mon, 16 Nov 2020 16:27:33 +0800 Subject: [PATCH 1/7] Support mulitple ZooKeeper clusters --- src/Interpreters/Context.cpp | 4 ++ src/Interpreters/Context.h | 2 + .../ReplicatedMergeTreeRestartingThread.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 41 +++++++++++++++---- src/Storages/StorageReplicatedMergeTree.h | 4 +- 5 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 54ee7713e95..90be787fd35 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1574,6 +1574,10 @@ bool Context::hasZooKeeper() const return getConfigRef().has("zookeeper"); } +bool Context::hasAuxiliaryZooKeeper(const String & name) const +{ + return getConfigRef().has("auxiliary_zookeepers." + name); +} void Context::setInterserverIOAddress(const String & host, UInt16 port) { diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 02a57b5d966..0a21534a995 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -492,6 +492,8 @@ public: void reloadAuxiliaryZooKeepersConfigIfChanged(const ConfigurationPtr & config); /// Has ready or expired ZooKeeper bool hasZooKeeper() const; + /// Has ready or expired auxiliary ZooKeeper + bool hasAuxiliaryZooKeeper(const String & name) const; /// Reset current zookeeper session. Do not create a new one. void resetZooKeeper() const; // Reload Zookeeper diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index cf8c32db804..b3cb7c92def 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -83,7 +83,7 @@ void ReplicatedMergeTreeRestartingThread::run() { try { - storage.setZooKeeper(storage.global_context.getZooKeeper()); + storage.setZooKeeper(); } catch (const Coordination::Exception &) { diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index dc254ae536e..6cbd7bac7a5 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -139,12 +139,18 @@ static const auto MERGE_SELECTING_SLEEP_MS = 5 * 1000; static const auto MUTATIONS_FINALIZING_SLEEP_MS = 1 * 1000; static const auto MUTATIONS_FINALIZING_IDLE_SLEEP_MS = 5 * 1000; -void StorageReplicatedMergeTree::setZooKeeper(zkutil::ZooKeeperPtr zookeeper) +void StorageReplicatedMergeTree::setZooKeeper() { std::lock_guard lock(current_zookeeper_mutex); - current_zookeeper = zookeeper; + if (zookeeper_name == default_zookeeper_name) + { + current_zookeeper = global_context.getZooKeeper(); + } + else + { + current_zookeeper = global_context.getAuxiliaryZooKeeper(zookeeper_name); + } } - zkutil::ZooKeeperPtr StorageReplicatedMergeTree::tryGetZooKeeper() const { std::lock_guard lock(current_zookeeper_mutex); @@ -195,9 +201,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( true, /// require_part_metadata attach, [this] (const std::string & name) { enqueuePartForCheck(name); }) - , zookeeper_path(normalizeZooKeeperPath(zookeeper_path_)) , replica_name(replica_name_) - , replica_path(zookeeper_path + "/replicas/" + replica_name) , reader(*this) , writer(*this) , merger_mutator(*this, global_context.getSettingsRef().background_pool_size) @@ -212,6 +216,22 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( , allow_renaming(allow_renaming_) , replicated_fetches_pool_size(global_context.getSettingsRef().background_fetches_pool_size) { + if (zookeeper_path_.empty()) + throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + auto pos = zookeeper_path_.find(':'); + if (pos != String::npos) + { + zookeeper_name = zookeeper_path_.substr(0, pos); + if (zookeeper_name.empty()) + throw Exception("Zookeeper path should start with '/' or ':/'", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + zookeeper_path = normalizeZooKeeperPath(zookeeper_path_.substr(pos + 1, String::npos)); + } + else + { + zookeeper_name = default_zookeeper_name; + zookeeper_path = normalizeZooKeeperPath(zookeeper_path_); + } + replica_path = zookeeper_path + "/replicas/" + replica_name; queue_updating_task = global_context.getSchedulePool().createTask( getStorageID().getFullTableName() + " (StorageReplicatedMergeTree::queueUpdatingTask)", [this]{ queueUpdatingTask(); }); @@ -227,7 +247,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( mutations_finalizing_task = global_context.getSchedulePool().createTask( getStorageID().getFullTableName() + " (StorageReplicatedMergeTree::mutationsFinalizingTask)", [this] { mutationsFinalizingTask(); }); - if (global_context.hasZooKeeper()) + if (global_context.hasZooKeeper() || global_context.hasAuxiliaryZooKeeper(zookeeper_name)) { /// It's possible for getZooKeeper() to timeout if zookeeper host(s) can't /// be reached. In such cases Poco::Exception is thrown after a connection @@ -244,7 +264,14 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( /// to be manually deleted before retrying the CreateQuery. try { - current_zookeeper = global_context.getZooKeeper(); + if (zookeeper_name == default_zookeeper_name) + { + current_zookeeper = global_context.getZooKeeper(); + } + else + { + current_zookeeper = global_context.getAuxiliaryZooKeeper(zookeeper_name); + } } catch (...) { diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 3b4a80bd3bf..78dd2d39fef 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -237,13 +237,15 @@ private: zkutil::ZooKeeperPtr tryGetZooKeeper() const; zkutil::ZooKeeperPtr getZooKeeper() const; - void setZooKeeper(zkutil::ZooKeeperPtr zookeeper); + void setZooKeeper(); /// If true, the table is offline and can not be written to it. std::atomic_bool is_readonly {false}; /// If false - ZooKeeper is available, but there is no table metadata. It's safe to drop table in this case. bool has_metadata_in_zookeeper = true; + static constexpr auto default_zookeeper_name = "default"; + String zookeeper_name; String zookeeper_path; String replica_name; String replica_path; From d8ae52118bd14464d4aae56343975e5e11d42a88 Mon Sep 17 00:00:00 2001 From: Peng Jian Date: Mon, 16 Nov 2020 17:50:54 +0800 Subject: [PATCH 2/7] add test --- .../__init__.py | 0 .../configs/config.xml | 42 ++++++++++++ .../test.py | 68 +++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/__init__.py create mode 100644 tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/config.xml create mode 100644 tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py diff --git a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/__init__.py b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/config.xml b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/config.xml new file mode 100644 index 00000000000..331bfede220 --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/config.xml @@ -0,0 +1,42 @@ + + + + zoo1 + 2181 + + + zoo2 + 2181 + + + zoo3 + 2181 + + + + + + zoo1 + 2181 + + + zoo2 + 2181 + + + + + + + + node1 + 9000 + + + node2 + 9000 + + + + + diff --git a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py new file mode 100644 index 00000000000..3a6deaef3ec --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py @@ -0,0 +1,68 @@ +import time + +import helpers.client as client +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + +cluster = ClickHouseCluster(__file__) +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance("node1", main_configs=["configs/config.xml"], with_zookeeper=True) +node2 = cluster.add_instance("node2", main_configs=["configs/config.xml"], with_zookeeper=True) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + + except Exception as ex: + print(ex) + + finally: + cluster.shutdown() + + +def drop_table(nodes, table_name): + for node in nodes: + node.query("DROP TABLE IF EXISTS {} NO DELAY".format(table_name)) + +# Create table with default zookeeper. +def test_create_replicated_merge_tree_with_default_zookeeper(started_cluster): + drop_table([node1, node2], "test_default_zookeeper") + for node in [node1, node2]: + node.query( + ''' + CREATE TABLE test_default_zookeeper(a Int32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_default_zookeeper', '{replica}') + ORDER BY a; + '''.format(replica=node.name)) + + # Insert data into node1, and query it from node2. + node1.query("INSERT INTO test_default_zookeeper VALUES (1)") + time.sleep(5) + + expected = "1\n" + assert TSV(node1.query("SELECT a FROM test_default_zookeeper")) == TSV(expected) + assert TSV(node2.query("SELECT a FROM test_default_zookeeper")) == TSV(expected) + +# Create table with auxiliary zookeeper. +def test_create_replicated_merge_tree_with_auxiliary_zookeeper(started_cluster): + drop_table([node1, node2], "test_auxiliary_zookeeper") + for node in [node1, node2]: + node.query( + ''' + CREATE TABLE test_auxiliary_zookeeper(a Int32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_auxiliary_zookeeper', '{replica}') + ORDER BY a; + '''.format(replica=node.name)) + + # Insert data into node1, and query it from node2. + node1.query("INSERT INTO test_auxiliary_zookeeper VALUES (1)") + time.sleep(5) + + expected = "1\n" + assert TSV(node1.query("SELECT a FROM test_auxiliary_zookeeper")) == TSV(expected) + assert TSV(node2.query("SELECT a FROM test_auxiliary_zookeeper")) == TSV(expected) From 2df0c3e0b043cbba1ae24edba4da678cd0674b77 Mon Sep 17 00:00:00 2001 From: Peng Jian Date: Mon, 16 Nov 2020 20:30:54 +0800 Subject: [PATCH 3/7] Add helper function for extracting zookeeper name and path. --- src/Storages/StorageReplicatedMergeTree.cpp | 47 ++++++++++----------- src/Storages/StorageReplicatedMergeTree.h | 1 + 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 6cbd7bac7a5..3be8286e3d0 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -165,7 +165,6 @@ zkutil::ZooKeeperPtr StorageReplicatedMergeTree::getZooKeeper() const return res; } - static std::string normalizeZooKeeperPath(std::string zookeeper_path) { if (!zookeeper_path.empty() && zookeeper_path.back() == '/') @@ -177,6 +176,24 @@ static std::string normalizeZooKeeperPath(std::string zookeeper_path) return zookeeper_path; } +void StorageReplicatedMergeTree::extractZooKeeperNameAndPath(const String & path, String & zookeeper_name_, String & zookeeper_path_) const +{ + if (path.empty()) + throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + auto pos = path.find(':'); + if (pos != String::npos) + { + zookeeper_name_ = path.substr(0, pos); + if (zookeeper_name_.empty()) + throw Exception("Zookeeper path should start with '/' or ':/'", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + zookeeper_path_ = normalizeZooKeeperPath(path.substr(pos + 1, String::npos)); + } + else + { + zookeeper_name_ = default_zookeeper_name; + zookeeper_path_ = normalizeZooKeeperPath(path); + } +} StorageReplicatedMergeTree::StorageReplicatedMergeTree( const String & zookeeper_path_, @@ -216,21 +233,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( , allow_renaming(allow_renaming_) , replicated_fetches_pool_size(global_context.getSettingsRef().background_fetches_pool_size) { - if (zookeeper_path_.empty()) - throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - auto pos = zookeeper_path_.find(':'); - if (pos != String::npos) - { - zookeeper_name = zookeeper_path_.substr(0, pos); - if (zookeeper_name.empty()) - throw Exception("Zookeeper path should start with '/' or ':/'", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - zookeeper_path = normalizeZooKeeperPath(zookeeper_path_.substr(pos + 1, String::npos)); - } - else - { - zookeeper_name = default_zookeeper_name; - zookeeper_path = normalizeZooKeeperPath(zookeeper_path_); - } + extractZooKeeperNameAndPath(zookeeper_path_, zookeeper_name, zookeeper_path); replica_path = zookeeper_path + "/replicas/" + replica_name; queue_updating_task = global_context.getSchedulePool().createTask( getStorageID().getFullTableName() + " (StorageReplicatedMergeTree::queueUpdatingTask)", [this]{ queueUpdatingTask(); }); @@ -4871,22 +4874,16 @@ void StorageReplicatedMergeTree::fetchPartition( const String & from_, const Context & query_context) { + String auxiliary_zookeeper_name; String from = from_; + extractZooKeeperNameAndPath(from, auxiliary_zookeeper_name, from); if (from.empty()) throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); String partition_id = getPartitionIDFromQuery(partition, query_context); - zkutil::ZooKeeperPtr zookeeper; - if (from[0] != '/') + if (auxiliary_zookeeper_name != default_zookeeper_name) { - auto delimiter = from.find(':'); - if (delimiter == String::npos) - throw Exception("Zookeeper path should start with '/' or ':/'", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - auto auxiliary_zookeeper_name = from.substr(0, delimiter); - from = from.substr(delimiter + 1, String::npos); - if (from.empty()) - throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); zookeeper = global_context.getAuxiliaryZooKeeper(auxiliary_zookeeper_name); LOG_INFO(log, "Will fetch partition {} from shard {} (auxiliary zookeeper '{}')", partition_id, from_, auxiliary_zookeeper_name); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 78dd2d39fef..1e20dab1f09 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -238,6 +238,7 @@ private: zkutil::ZooKeeperPtr tryGetZooKeeper() const; zkutil::ZooKeeperPtr getZooKeeper() const; void setZooKeeper(); + void extractZooKeeperNameAndPath(const String & path, String & zookeeper_name_, String & zookeeper_path_) const; /// If true, the table is offline and can not be written to it. std::atomic_bool is_readonly {false}; From 091f7065cd7e6beec6a6615895ebf92f6d2d061a Mon Sep 17 00:00:00 2001 From: Peng Jian Date: Tue, 17 Nov 2020 09:55:25 +0800 Subject: [PATCH 4/7] fix build --- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 3be8286e3d0..7fded676ecd 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -176,7 +176,7 @@ static std::string normalizeZooKeeperPath(std::string zookeeper_path) return zookeeper_path; } -void StorageReplicatedMergeTree::extractZooKeeperNameAndPath(const String & path, String & zookeeper_name_, String & zookeeper_path_) const +void StorageReplicatedMergeTree::extractZooKeeperNameAndPath(const String & path, String & zookeeper_name_, String & zookeeper_path_) { if (path.empty()) throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 1e20dab1f09..b64b9e1f25e 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -238,7 +238,7 @@ private: zkutil::ZooKeeperPtr tryGetZooKeeper() const; zkutil::ZooKeeperPtr getZooKeeper() const; void setZooKeeper(); - void extractZooKeeperNameAndPath(const String & path, String & zookeeper_name_, String & zookeeper_path_) const; + static void extractZooKeeperNameAndPath(const String & path, String & zookeeper_name_, String & zookeeper_path_); /// If true, the table is offline and can not be written to it. std::atomic_bool is_readonly {false}; From 3c86c8b3c9ac19a980f74a5017788ddca8a27c6a Mon Sep 17 00:00:00 2001 From: Peng Jian Date: Tue, 17 Nov 2020 20:27:19 +0800 Subject: [PATCH 5/7] fix test cases --- src/Storages/StorageReplicatedMergeTree.cpp | 32 ++++++++++++------- src/Storages/StorageReplicatedMergeTree.h | 1 - .../configs/remote_servers.xml | 16 ++++++++++ .../{config.xml => zookeeper_config.xml} | 14 -------- .../test.py | 18 +++++++++-- 5 files changed, 52 insertions(+), 29 deletions(-) create mode 100644 tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/remote_servers.xml rename tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/{config.xml => zookeeper_config.xml} (63%) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 7fded676ecd..77f4ed9304c 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -151,6 +151,7 @@ void StorageReplicatedMergeTree::setZooKeeper() current_zookeeper = global_context.getAuxiliaryZooKeeper(zookeeper_name); } } + zkutil::ZooKeeperPtr StorageReplicatedMergeTree::tryGetZooKeeper() const { std::lock_guard lock(current_zookeeper_mutex); @@ -176,23 +177,32 @@ static std::string normalizeZooKeeperPath(std::string zookeeper_path) return zookeeper_path; } -void StorageReplicatedMergeTree::extractZooKeeperNameAndPath(const String & path, String & zookeeper_name_, String & zookeeper_path_) +static String extractZooKeeperName(const String & path) { if (path.empty()) throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); auto pos = path.find(':'); if (pos != String::npos) { - zookeeper_name_ = path.substr(0, pos); + auto zookeeper_name_ = path.substr(0, pos); if (zookeeper_name_.empty()) throw Exception("Zookeeper path should start with '/' or ':/'", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - zookeeper_path_ = normalizeZooKeeperPath(path.substr(pos + 1, String::npos)); + return zookeeper_name_; } - else + static constexpr auto default_zookeeper_name = "default"; + return default_zookeeper_name; +} + +static String extractZooKeeperPath(const String & path) +{ + if (path.empty()) + throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + auto pos = path.find(':'); + if (pos != String::npos) { - zookeeper_name_ = default_zookeeper_name; - zookeeper_path_ = normalizeZooKeeperPath(path); + return normalizeZooKeeperPath(path.substr(pos + 1, String::npos)); } + return normalizeZooKeeperPath(path); } StorageReplicatedMergeTree::StorageReplicatedMergeTree( @@ -218,7 +228,10 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( true, /// require_part_metadata attach, [this] (const std::string & name) { enqueuePartForCheck(name); }) + , zookeeper_name(extractZooKeeperName(zookeeper_path_)) + , zookeeper_path(extractZooKeeperPath(zookeeper_path_)) , replica_name(replica_name_) + , replica_path(zookeeper_path + "/replicas/" + replica_name_) , reader(*this) , writer(*this) , merger_mutator(*this, global_context.getSettingsRef().background_pool_size) @@ -233,8 +246,6 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( , allow_renaming(allow_renaming_) , replicated_fetches_pool_size(global_context.getSettingsRef().background_fetches_pool_size) { - extractZooKeeperNameAndPath(zookeeper_path_, zookeeper_name, zookeeper_path); - replica_path = zookeeper_path + "/replicas/" + replica_name; queue_updating_task = global_context.getSchedulePool().createTask( getStorageID().getFullTableName() + " (StorageReplicatedMergeTree::queueUpdatingTask)", [this]{ queueUpdatingTask(); }); @@ -4874,9 +4885,8 @@ void StorageReplicatedMergeTree::fetchPartition( const String & from_, const Context & query_context) { - String auxiliary_zookeeper_name; - String from = from_; - extractZooKeeperNameAndPath(from, auxiliary_zookeeper_name, from); + String auxiliary_zookeeper_name = extractZooKeeperName(from_); + String from = extractZooKeeperPath(from_); if (from.empty()) throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index b64b9e1f25e..78dd2d39fef 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -238,7 +238,6 @@ private: zkutil::ZooKeeperPtr tryGetZooKeeper() const; zkutil::ZooKeeperPtr getZooKeeper() const; void setZooKeeper(); - static void extractZooKeeperNameAndPath(const String & path, String & zookeeper_name_, String & zookeeper_path_); /// If true, the table is offline and can not be written to it. std::atomic_bool is_readonly {false}; diff --git a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/remote_servers.xml b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/remote_servers.xml new file mode 100644 index 00000000000..3593cbd7f36 --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/remote_servers.xml @@ -0,0 +1,16 @@ + + + + + + node1 + 9000 + + + node2 + 9000 + + + + + diff --git a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/config.xml b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/zookeeper_config.xml similarity index 63% rename from tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/config.xml rename to tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/zookeeper_config.xml index 331bfede220..b2b0667ebbf 100644 --- a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/config.xml +++ b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/configs/zookeeper_config.xml @@ -25,18 +25,4 @@ - - - - - node1 - 9000 - - - node2 - 9000 - - - - diff --git a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py index 3a6deaef3ec..91a25ec8d8a 100644 --- a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py +++ b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py @@ -3,12 +3,13 @@ import time import helpers.client as client import pytest from helpers.cluster import ClickHouseCluster +from helpers.client import QueryRuntimeException from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", main_configs=["configs/config.xml"], with_zookeeper=True) -node2 = cluster.add_instance("node2", main_configs=["configs/config.xml"], with_zookeeper=True) +node1 = cluster.add_instance("node1", main_configs=["configs/zookeeper_config.xml", "configs/remote_servers.xml"], with_zookeeper=True) +node2 = cluster.add_instance("node2", main_configs=["configs/zookeeper_config.xml", "configs/remote_servers.xml"], with_zookeeper=True) @pytest.fixture(scope="module") @@ -55,7 +56,7 @@ def test_create_replicated_merge_tree_with_auxiliary_zookeeper(started_cluster): node.query( ''' CREATE TABLE test_auxiliary_zookeeper(a Int32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_auxiliary_zookeeper', '{replica}') + ENGINE = ReplicatedMergeTree('zookeeper2:/clickhouse/tables/test/test_auxiliary_zookeeper', '{replica}') ORDER BY a; '''.format(replica=node.name)) @@ -66,3 +67,14 @@ def test_create_replicated_merge_tree_with_auxiliary_zookeeper(started_cluster): expected = "1\n" assert TSV(node1.query("SELECT a FROM test_auxiliary_zookeeper")) == TSV(expected) assert TSV(node2.query("SELECT a FROM test_auxiliary_zookeeper")) == TSV(expected) + +# Create table with auxiliary zookeeper. +def test_create_replicated_merge_tree_with_not_exists_auxiliary_zookeeper(started_cluster): + drop_table([node1], "test_auxiliary_zookeeper") + with pytest.raises(QueryRuntimeException): + node1.query( + ''' + CREATE TABLE test_auxiliary_zookeeper(a Int32) + ENGINE = ReplicatedMergeTree('zookeeper_not_exits:/clickhouse/tables/test/test_auxiliary_zookeeper', '{replica}') + ORDER BY a; + '''.format(replica=node1.name)) From 10cefe4f513dcec2d94028081ec8f29c4c2cefbb Mon Sep 17 00:00:00 2001 From: Peng Jian Date: Tue, 17 Nov 2020 20:45:21 +0800 Subject: [PATCH 6/7] fix code style --- src/Storages/StorageReplicatedMergeTree.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 77f4ed9304c..9159275aed1 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -184,10 +184,10 @@ static String extractZooKeeperName(const String & path) auto pos = path.find(':'); if (pos != String::npos) { - auto zookeeper_name_ = path.substr(0, pos); - if (zookeeper_name_.empty()) + auto zookeeper_name = path.substr(0, pos); + if (zookeeper_name.empty()) throw Exception("Zookeeper path should start with '/' or ':/'", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - return zookeeper_name_; + return zookeeper_name; } static constexpr auto default_zookeeper_name = "default"; return default_zookeeper_name; @@ -228,8 +228,8 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( true, /// require_part_metadata attach, [this] (const std::string & name) { enqueuePartForCheck(name); }) - , zookeeper_name(extractZooKeeperName(zookeeper_path_)) - , zookeeper_path(extractZooKeeperPath(zookeeper_path_)) + , zookeeper_name(extractZooKeeperName(zookeeper_path_)) + , zookeeper_path(extractZooKeeperPath(zookeeper_path_)) , replica_name(replica_name_) , replica_path(zookeeper_path + "/replicas/" + replica_name_) , reader(*this) From fd0705c93db0f9d87ad62224a6120eb79a3ac855 Mon Sep 17 00:00:00 2001 From: Peng Jian Date: Wed, 18 Nov 2020 10:54:48 +0800 Subject: [PATCH 7/7] add doc --- .../mergetree-family/replication.md | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 932facc9ddc..7c628b8d8cc 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -53,6 +53,42 @@ Example of setting the addresses of the ZooKeeper cluster: ``` +ClickHouse also supports to store replicas meta information in the auxiliary ZooKeeper cluster by providing ZooKeeper cluster name and path as engine arguments. +In other word, it supports to store the metadata of differnt tables in different ZooKeeper clusters. + +Example of setting the addresses of the auxiliary ZooKeeper cluster: + +``` xml + + + + example_2_1 + 2181 + + + example_2_2 + 2181 + + + example_2_3 + 2181 + + + + + example_3_1 + 2181 + + + +``` + +To store table datameta in a auxiliary ZooKeeper cluster instead of default ZooKeeper cluster, we can use the SQL to create table with +ReplicatedMergeTree engine as follow: + +``` +CREATE TABLE table_name ( ... ) ENGINE = ReplicatedMergeTree('zookeeper_name_configurated_in_auxiliary_zookeepers:path', 'replica_name') ... +``` You can specify any existing ZooKeeper cluster and the system will use a directory on it for its own data (the directory is specified when creating a replicatable table). If ZooKeeper isn’t set in the config file, you can’t create replicated tables, and any existing replicated tables will be read-only.