diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index dc181339786..cfa3c806212 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -77,7 +77,7 @@ fi if [ "$NUM_TRIES" -gt "1" ]; then export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000 export THREAD_FUZZER_SLEEP_PROBABILITY=0.1 - export THREAD_FUZZER_SLEEP_TIME_US=100000 + export THREAD_FUZZER_SLEEP_TIME_US_MAX=100000 export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1 export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1 @@ -88,10 +88,10 @@ if [ "$NUM_TRIES" -gt "1" ]; then export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001 - export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000 - export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000 - export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000 - export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000 + export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000 + export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000 + export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX=10000 + export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX=10000 mkdir -p /var/run/clickhouse-server # simplest way to forward env variables to server diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 621a6ced7f6..ea7e3aece1d 100644 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -27,7 +27,7 @@ install_packages package_folder # and find more potential issues. export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000 export THREAD_FUZZER_SLEEP_PROBABILITY=0.1 -export THREAD_FUZZER_SLEEP_TIME_US=100000 +export THREAD_FUZZER_SLEEP_TIME_US_MAX=100000 export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1 export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1 @@ -38,11 +38,11 @@ export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001 -export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000 +export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000 -export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000 -export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000 -export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000 +export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000 +export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX=10000 +export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX=10000 export THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY=0.01 export THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY=0.01 diff --git a/src/Common/ThreadFuzzer.cpp b/src/Common/ThreadFuzzer.cpp index 1d944f4a458..9f9ec4fa356 100644 --- a/src/Common/ThreadFuzzer.cpp +++ b/src/Common/ThreadFuzzer.cpp @@ -86,12 +86,12 @@ static std::atomic num_cpus = 0; static std::atomic NAME##_before_yield_probability = 0; \ static std::atomic NAME##_before_migrate_probability = 0; \ static std::atomic NAME##_before_sleep_probability = 0; \ - static std::atomic NAME##_before_sleep_time_us = 0; \ + static std::atomic NAME##_before_sleep_time_us_max = 0; \ \ static std::atomic NAME##_after_yield_probability = 0; \ static std::atomic NAME##_after_migrate_probability = 0; \ static std::atomic NAME##_after_sleep_probability = 0; \ - static std::atomic NAME##_after_sleep_time_us = 0; + static std::atomic NAME##_after_sleep_time_us_max = 0; FOR_EACH_WRAPPED_FUNCTION(DEFINE_WRAPPER_PARAMS) @@ -110,7 +110,7 @@ void ThreadFuzzer::initConfiguration() initFromEnv(yield_probability, "THREAD_FUZZER_YIELD_PROBABILITY"); initFromEnv(migrate_probability, "THREAD_FUZZER_MIGRATE_PROBABILITY"); initFromEnv(sleep_probability, "THREAD_FUZZER_SLEEP_PROBABILITY"); - initFromEnv(sleep_time_us, "THREAD_FUZZER_SLEEP_TIME_US"); + initFromEnv(sleep_time_us_max, "THREAD_FUZZER_SLEEP_TIME_US_MAX"); initFromEnv(explicit_sleep_probability, "THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY"); initFromEnv(explicit_memory_exception_probability, "THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY"); @@ -119,13 +119,12 @@ void ThreadFuzzer::initConfiguration() initFromEnv(NAME##_before_yield_probability, "THREAD_FUZZER_" #NAME "_BEFORE_YIELD_PROBABILITY"); \ initFromEnv(NAME##_before_migrate_probability, "THREAD_FUZZER_" #NAME "_BEFORE_MIGRATE_PROBABILITY"); \ initFromEnv(NAME##_before_sleep_probability, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_PROBABILITY"); \ - initFromEnv(NAME##_before_sleep_time_us, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_TIME_US"); \ + initFromEnv(NAME##_before_sleep_time_us_max, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_TIME_US_MAX"); \ \ initFromEnv(NAME##_after_yield_probability, "THREAD_FUZZER_" #NAME "_AFTER_YIELD_PROBABILITY"); \ initFromEnv(NAME##_after_migrate_probability, "THREAD_FUZZER_" #NAME "_AFTER_MIGRATE_PROBABILITY"); \ initFromEnv(NAME##_after_sleep_probability, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_PROBABILITY"); \ - initFromEnv(NAME##_after_sleep_time_us, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US"); - + initFromEnv(NAME##_after_sleep_time_us_max, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US_MAX"); FOR_EACH_WRAPPED_FUNCTION(INIT_WRAPPER_PARAMS) # undef INIT_WRAPPER_PARAMS @@ -146,7 +145,7 @@ bool ThreadFuzzer::isEffective() const return true; \ if (NAME##_before_sleep_probability.load(std::memory_order_relaxed) > 0.0) \ return true; \ - if (NAME##_before_sleep_time_us.load(std::memory_order_relaxed) > 0.0) \ + if (NAME##_before_sleep_time_us_max.load(std::memory_order_relaxed) > 0.0) \ return true; \ \ if (NAME##_after_yield_probability.load(std::memory_order_relaxed) > 0.0) \ @@ -155,7 +154,7 @@ bool ThreadFuzzer::isEffective() const return true; \ if (NAME##_after_sleep_probability.load(std::memory_order_relaxed) > 0.0) \ return true; \ - if (NAME##_after_sleep_time_us.load(std::memory_order_relaxed) > 0.0) \ + if (NAME##_after_sleep_time_us_max.load(std::memory_order_relaxed) > 0.0) \ return true; FOR_EACH_WRAPPED_FUNCTION(CHECK_WRAPPER_PARAMS) @@ -166,7 +165,7 @@ bool ThreadFuzzer::isEffective() const return cpu_time_period_us != 0 && (yield_probability > 0 || migrate_probability > 0 - || (sleep_probability > 0 && sleep_time_us > 0)); + || (sleep_probability > 0 && sleep_time_us_max > 0)); } void ThreadFuzzer::stop() @@ -190,7 +189,7 @@ static void injectionImpl( double yield_probability, double migrate_probability, double sleep_probability, - double sleep_time_us) + double sleep_time_us_max) { DENY_ALLOCATIONS_IN_SCOPE; if (!ThreadFuzzer::isStarted()) @@ -221,10 +220,10 @@ static void injectionImpl( #endif if (sleep_probability > 0 - && sleep_time_us > 0 + && sleep_time_us_max > 0 && std::bernoulli_distribution(sleep_probability)(thread_local_rng)) { - sleepForNanoseconds(static_cast(sleep_time_us * 1000)); + sleepForNanoseconds((thread_local_rng() % static_cast(sleep_time_us_max)) * 1000); /*may sleep(0)*/ } } @@ -232,19 +231,19 @@ static ALWAYS_INLINE void injection( double yield_probability, double migrate_probability, double sleep_probability, - double sleep_time_us) + double sleep_time_us_max) { DENY_ALLOCATIONS_IN_SCOPE; if (!ThreadFuzzer::isStarted()) return; - injectionImpl(yield_probability, migrate_probability, sleep_probability, sleep_time_us); + injectionImpl(yield_probability, migrate_probability, sleep_probability, sleep_time_us_max); } void ThreadFuzzer::maybeInjectSleep() { auto & fuzzer = ThreadFuzzer::instance(); - injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.explicit_sleep_probability, fuzzer.sleep_time_us); + injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.explicit_sleep_probability, fuzzer.sleep_time_us_max); } /// Sometimes maybeInjectSleep() is not enough and we need to inject an exception. @@ -265,7 +264,7 @@ void ThreadFuzzer::signalHandler(int) DENY_ALLOCATIONS_IN_SCOPE; auto saved_errno = errno; auto & fuzzer = ThreadFuzzer::instance(); - injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.sleep_probability, fuzzer.sleep_time_us); + injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.sleep_probability, fuzzer.sleep_time_us_max); errno = saved_errno; } @@ -309,13 +308,13 @@ void ThreadFuzzer::setup() const NAME##_before_yield_probability.load(std::memory_order_relaxed), \ NAME##_before_migrate_probability.load(std::memory_order_relaxed), \ NAME##_before_sleep_probability.load(std::memory_order_relaxed), \ - NAME##_before_sleep_time_us.load(std::memory_order_relaxed)); + NAME##_before_sleep_time_us_max.load(std::memory_order_relaxed)); #define INJECTION_AFTER(NAME) \ injectionImpl( \ NAME##_after_yield_probability.load(std::memory_order_relaxed), \ NAME##_after_migrate_probability.load(std::memory_order_relaxed), \ NAME##_after_sleep_probability.load(std::memory_order_relaxed), \ - NAME##_after_sleep_time_us.load(std::memory_order_relaxed)); + NAME##_after_sleep_time_us_max.load(std::memory_order_relaxed)); /// ThreadFuzzer intercepts pthread_mutex_lock()/pthread_mutex_unlock(). /// diff --git a/src/Common/ThreadFuzzer.h b/src/Common/ThreadFuzzer.h index 1cff27a7588..8956538e7d1 100644 --- a/src/Common/ThreadFuzzer.h +++ b/src/Common/ThreadFuzzer.h @@ -16,7 +16,7 @@ namespace DB * THREAD_FUZZER_YIELD_PROBABILITY - probability to do 'sched_yield'. * THREAD_FUZZER_MIGRATE_PROBABILITY - probability to set CPU affinity to random CPU core. * THREAD_FUZZER_SLEEP_PROBABILITY - probability to sleep. - * THREAD_FUZZER_SLEEP_TIME_US - amount of time to sleep in microseconds. + * THREAD_FUZZER_SLEEP_TIME_US_MAX - max amount of time to sleep in microseconds, actual sleep time is randomized. * * ThreadFuzzer will do nothing if environment variables are not set accordingly. * @@ -33,16 +33,15 @@ namespace DB * * Notes: * - it can be also implemented with instrumentation (example: LLVM Xray) instead of signals. - * - we should also make the sleep time random. * - sleep and migration obviously helps, but the effect of yield is unclear. * * In addition, we allow to inject glitches around thread synchronization functions. * Example: * * THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001 - * THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000 + * THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000 * THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001 - * THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000 + * THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000 */ class ThreadFuzzer { @@ -67,7 +66,8 @@ private: double yield_probability = 0; double migrate_probability = 0; double sleep_probability = 0; - double sleep_time_us = 0; + double sleep_time_us_max = 0; + double explicit_sleep_probability = 0; double explicit_memory_exception_probability = 0; diff --git a/src/Common/examples/chaos_sanitizer.cpp b/src/Common/examples/chaos_sanitizer.cpp index 76e22411a2e..175c3e8d767 100644 --- a/src/Common/examples/chaos_sanitizer.cpp +++ b/src/Common/examples/chaos_sanitizer.cpp @@ -12,7 +12,7 @@ /** Proves that ThreadFuzzer helps to find concurrency bugs. * * for i in {1..10}; do ./chaos_sanitizer 1000000; done - * for i in {1..10}; do THREAD_FUZZER_CPU_TIME_PERIOD_US=1000 THREAD_FUZZER_SLEEP_PROBABILITY=0.1 THREAD_FUZZER_SLEEP_TIME_US=100000 ./chaos_sanitizer 1000000; done + * for i in {1..10}; do THREAD_FUZZER_CPU_TIME_PERIOD_US=1000 THREAD_FUZZER_SLEEP_PROBABILITY=0.1 THREAD_FUZZER_SLEEP_TIME_US_MAX=100000 ./chaos_sanitizer 1000000; done */ int main(int argc, char ** argv) { diff --git a/utils/keeper-overload/keeper-overload.py b/utils/keeper-overload/keeper-overload.py index 0a059b10588..1032ea656bc 100755 --- a/utils/keeper-overload/keeper-overload.py +++ b/utils/keeper-overload/keeper-overload.py @@ -102,7 +102,7 @@ class Keeper: if self.with_thread_fuzzer: env["THREAD_FUZZER_CPU_TIME_PERIOD_US"] = "1000" env["THREAD_FUZZER_SLEEP_PROBABILITY"] = "0.1" - env["THREAD_FUZZER_SLEEP_TIME_US"] = "100000" + env["THREAD_FUZZER_SLEEP_TIME_US_MAX"] = "100000" env["THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY"] = "1" env["THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY"] = "1" env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_MIGRATE_PROBABILITY"] = "1" @@ -112,10 +112,10 @@ class Keeper: env["THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY"] = "0.001" env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY"] = "0.001" env["THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY"] = "0.001" - env["THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US"] = "10000" - env["THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US"] = "10000" - env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US"] = "10000" - env["THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US"] = "10000" + env["THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX"] = "10000" + env["THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX"] = "10000" + env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX"] = "10000" + env["THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX"] = "10000" self.process = subprocess.Popen( [