From 6dc45af5ae08cbda447a89f3133b1de30cd28565 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 6 Apr 2018 22:43:37 +0300
Subject: [PATCH] Avoid herd effect in ReplicatedMergeTreeCleanupThread
 [#CLICKHOUSE-2]

---
 dbms/src/Storages/MergeTree/MergeTreeSettings.h              | 3 +++
 .../Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp  | 5 ++++-
 .../Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h    | 3 +++
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
index 424c634afaf..aa29dccc195 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
@@ -104,6 +104,9 @@ struct MergeTreeSettings
                                                                                                               \
     /** Period to clean old queue logs, blocks hashes and parts */                                            \
     M(SettingUInt64, cleanup_delay_period, 30)                                                                \
+    /** Add uniformly distributed value from 0 to x seconds to cleanup_delay_period                           \
+        to avoid thundering herd effect and subsequent DoS of ZooKeeper in case of very large number of tables */ \
+    M(SettingUInt64, cleanup_delay_period_random_add, 10)                                                     \
                                                                                                               \
     /** Minimal delay from other replicas to yield leadership. Here and further 0 means unlimited. */         \
     M(SettingUInt64, min_relative_delay_to_yield_leadership, 120)                                             \
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp
index fb3276fcfab..e3d090f4212 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp
@@ -3,6 +3,8 @@
 #include <Common/setThreadName.h>
 #include <Poco/Timestamp.h>
 
+#include <random>
+
 
 namespace DB
 {
@@ -25,7 +27,8 @@ void ReplicatedMergeTreeCleanupThread::run()
 {
     setThreadName("ReplMTCleanup");
 
-    const auto CLEANUP_SLEEP_MS = storage.data.settings.cleanup_delay_period * 1000;
+    const auto CLEANUP_SLEEP_MS = (storage.data.settings.cleanup_delay_period
+        + std::uniform_int_distribution<UInt64>(0, storage.data.settings.cleanup_delay_period_random_add)(rng)) * 1000;
 
     while (!storage.shutdown_called)
     {
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h
index c717e84bfd5..ccbb564fa96 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h
@@ -7,6 +7,8 @@
 #include <thread>
 #include <map>
 
+#include <pcg_random.hpp>
+
 
 namespace DB
 {
@@ -27,6 +29,7 @@ private:
     StorageReplicatedMergeTree & storage;
     Logger * log;
     std::thread thread;
+    pcg64 rng;
 
     void run();
     void iterate();