Stop ThreadFuzzer before hung check (#29167)

* stop ThreadFuzzer before hung check

* fix

* fix

docker/test/stress/stress

@@ -70,6 +70,9 @@ def compress_stress_logs(output_path, files_prefix):
 def prepare_for_hung_check(drop_databases):
     # FIXME this function should not exist, but...
 
+    # ThreadFuzzer significantly slows down server and causes false-positive hung check failures
+    call("clickhouse client -q 'SYSTEM STOP THREAD FUZZER'", shell=True, stderr=STDOUT)
+
     # We attach gdb to clickhouse-server before running tests
     # to print stacktraces of all crashes even if clickhouse cannot print it for some reason.
     # However, it obstruct checking for hung queries.

src/Common/ThreadFuzzer.cpp

@@ -128,6 +128,9 @@ void ThreadFuzzer::initConfiguration()
 
 bool ThreadFuzzer::isEffective() const
 {
+    if (!isStarted())
+        return false;
+
 #if THREAD_FUZZER_WRAP_PTHREAD
 #    define CHECK_WRAPPER_PARAMS(RET, NAME, ...) \
         if (NAME##_before_yield_probability.load(std::memory_order_relaxed)) \
@@ -159,6 +162,20 @@ bool ThreadFuzzer::isEffective() const
             || (sleep_probability > 0 && sleep_time_us > 0));
 }
 
+void ThreadFuzzer::stop()
+{
+    started.store(false, std::memory_order_relaxed);
+}
+
+void ThreadFuzzer::start()
+{
+    started.store(true, std::memory_order_relaxed);
+}
+
+bool ThreadFuzzer::isStarted()
+{
+    return started.load(std::memory_order_relaxed);
+}
 
 static void injection(
     double yield_probability,
@@ -166,6 +183,10 @@ static void injection(
     double sleep_probability,
     double sleep_time_us [[maybe_unused]])
 {
+    DENY_ALLOCATIONS_IN_SCOPE;
+    if (!ThreadFuzzer::isStarted())
+        return;
+
     if (yield_probability > 0
         && std::bernoulli_distribution(yield_probability)(thread_local_rng))
     {

src/Common/ThreadFuzzer.h

@@ -1,6 +1,6 @@
 #pragma once
 #include <cstdint>
-
+#include <atomic>
 
 namespace DB
 {
@@ -54,6 +54,10 @@ public:
 
     bool isEffective() const;
 
+    static void stop();
+    static void start();
+    static bool isStarted();
+
 private:
     uint64_t cpu_time_period_us = 0;
     double yield_probability = 0;
@@ -61,6 +65,8 @@ private:
     double sleep_probability = 0;
     double sleep_time_us = 0;
 
+    inline static std::atomic<bool> started{true};
+
     ThreadFuzzer();
 
     void initConfiguration();

src/Interpreters/InterpreterSystemQuery.cpp

@@ -40,6 +40,7 @@
 #include <Parsers/ASTSystemQuery.h>
 #include <Parsers/ASTDropQuery.h>
 #include <Parsers/ASTCreateQuery.h>
+#include <Common/ThreadFuzzer.h>
 #include <csignal>
 #include <algorithm>
 
@@ -445,6 +446,12 @@ BlockIO InterpreterSystemQuery::execute()
         case Type::STOP_LISTEN_QUERIES:
         case Type::START_LISTEN_QUERIES:
             throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} is not supported yet", query.type);
+        case Type::STOP_THREAD_FUZZER:
+            ThreadFuzzer::stop();
+            break;
+        case Type::START_THREAD_FUZZER:
+            ThreadFuzzer::start();
+            break;
         default:
             throw Exception("Unknown type of SYSTEM query", ErrorCodes::BAD_ARGUMENTS);
     }
@@ -877,6 +884,8 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster()
         }
         case Type::STOP_LISTEN_QUERIES: break;
         case Type::START_LISTEN_QUERIES: break;
+        case Type::STOP_THREAD_FUZZER: break;
+        case Type::START_THREAD_FUZZER: break;
         case Type::UNKNOWN: break;
         case Type::END: break;
     }

src/Parsers/ASTSystemQuery.h

@@ -61,6 +61,8 @@ public:
         FLUSH_DISTRIBUTED,
         STOP_DISTRIBUTED_SENDS,
         START_DISTRIBUTED_SENDS,
+        START_THREAD_FUZZER,
+        STOP_THREAD_FUZZER,
         END
     };