From 3c11d444940e8b7c59d93a24366c9b2fb096ec2c Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 26 Feb 2021 14:06:24 +0300 Subject: [PATCH] Add description for getUniqueId method, fix typos --- src/Storages/MergeTree/IMergeTreeDataPart.h | 2 ++ src/Storages/MergeTree/MergeTreeData.h | 6 +++--- src/Storages/StorageReplicatedMergeTree.cpp | 6 ++++-- src/Storages/StorageReplicatedMergeTree.h | 3 +++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index f9b5f616f70..83f8c672001 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -367,6 +367,8 @@ public: /// part creation (using alter query with materialize_ttl setting). bool checkAllTTLCalculated(const StorageMetadataPtr & metadata_snapshot) const; + /// Return some uniq string for file + /// Required for distinguish different copies of the same part on S3 String getUniqueId() const; protected: diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 81e977d7ecc..3f41cc04217 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -739,16 +739,16 @@ public: bool areBackgroundMovesNeeded() const; /// Lock part in zookeeper for use common S3 data in several nodes - /// Overrided in StorageReplicatedMergeTree + /// Overridden in StorageReplicatedMergeTree virtual void lockSharedData(const IMergeTreeDataPart &) const {} /// Unlock common S3 data part in zookeeper - /// Overrided in StorageReplicatedMergeTree + /// Overridden in StorageReplicatedMergeTree virtual bool unlockSharedData(const IMergeTreeDataPart &) const { return true; } virtual bool unlockSharedData(const IMergeTreeDataPart &, const String &) const { return true; } /// Fetch part only if some replica has it on shared storage like S3 - /// Overrided in StorageReplicatedMergeTree + /// Overridden in StorageReplicatedMergeTree virtual bool tryToFetchIfShared(const IMergeTreeDataPart &, const DiskPtr &, const String &) const { return false; } protected: diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index b94d3dd3f89..8c5636d2aa0 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -142,6 +142,10 @@ static const auto MERGE_SELECTING_SLEEP_MS = 5 * 1000; static const auto MUTATIONS_FINALIZING_SLEEP_MS = 1 * 1000; static const auto MUTATIONS_FINALIZING_IDLE_SLEEP_MS = 5 * 1000; + +std::atomic_uint StorageReplicatedMergeTree::total_fetches {0}; + + void StorageReplicatedMergeTree::setZooKeeper() { std::lock_guard lock(current_zookeeper_mutex); @@ -1730,7 +1734,6 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) const auto storage_settings_ptr = getSettings(); auto metadata_snapshot = getInMemoryMetadataPtr(); - static std::atomic_uint total_fetches {0}; if (storage_settings_ptr->replicated_max_parallel_fetches && total_fetches >= storage_settings_ptr->replicated_max_parallel_fetches) { throw Exception("Too many total fetches from replicas, maximum: " + storage_settings_ptr->replicated_max_parallel_fetches.toString(), @@ -1934,7 +1937,6 @@ bool StorageReplicatedMergeTree::executeFetchShared(ReplicatedMergeTreeLogEntry const auto storage_settings_ptr = getSettings(); auto metadata_snapshot = getInMemoryMetadataPtr(); - static std::atomic_uint total_fetches {0}; if (storage_settings_ptr->replicated_max_parallel_fetches && total_fetches >= storage_settings_ptr->replicated_max_parallel_fetches) { throw Exception("Too many total fetches from replicas, maximum: " + storage_settings_ptr->replicated_max_parallel_fetches.toString(), diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 70a83145da6..58bedfc0408 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -307,6 +307,9 @@ private: /// Event that is signalled (and is reset) by the restarting_thread when the ZooKeeper session expires. Poco::Event partial_shutdown_event {false}; /// Poco::Event::EVENT_MANUALRESET + /// Limiting parallel fetches per node + static std::atomic_uint total_fetches; + /// Limiting parallel fetches per one table std::atomic_uint current_table_fetches {0};