thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

thread-fuzzer: randomize sleep time

Browse Source
This commit is contained in:
tomershafir 2024-03-15 00:02:09 +02:00
parent 48ec419b9d
commit 23169117ae
6 changed files with 38 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docker/test/stateless/run.sh
View File

@ -77,7 +77,7 @@ fi
if [ "$NUM_TRIES" -gt "1" ]; then
export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
export THREAD_FUZZER_SLEEP_TIME_US=100000
export THREAD_FUZZER_SLEEP_TIME_US_MAX=100000
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
@ -88,10 +88,10 @@ if [ "$NUM_TRIES" -gt "1" ]; then
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX=10000
mkdir -p /var/run/clickhouse-server
# simplest way to forward env variables to server

10
docker/test/stress/run.sh
View File

@ -27,7 +27,7 @@ install_packages package_folder
# and find more potential issues.
export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
export THREAD_FUZZER_SLEEP_TIME_US=100000
export THREAD_FUZZER_SLEEP_TIME_US_MAX=100000
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
@ -38,11 +38,11 @@ export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY=0.01
export THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY=0.01

35
src/Common/ThreadFuzzer.cpp
View File

@ -86,12 +86,12 @@ static std::atomic<int> num_cpus = 0;
static std::atomic<double> NAME##_before_yield_probability = 0; \
static std::atomic<double> NAME##_before_migrate_probability = 0; \
static std::atomic<double> NAME##_before_sleep_probability = 0; \
static std::atomic<double> NAME##_before_sleep_time_us = 0; \
static std::atomic<double> NAME##_before_sleep_time_us_max = 0; \
\
static std::atomic<double> NAME##_after_yield_probability = 0; \
static std::atomic<double> NAME##_after_migrate_probability = 0; \
static std::atomic<double> NAME##_after_sleep_probability = 0; \
static std::atomic<double> NAME##_after_sleep_time_us = 0;
static std::atomic<double> NAME##_after_sleep_time_us_max = 0;
FOR_EACH_WRAPPED_FUNCTION(DEFINE_WRAPPER_PARAMS)
@ -110,7 +110,7 @@ void ThreadFuzzer::initConfiguration()
initFromEnv(yield_probability, "THREAD_FUZZER_YIELD_PROBABILITY");
initFromEnv(migrate_probability, "THREAD_FUZZER_MIGRATE_PROBABILITY");
initFromEnv(sleep_probability, "THREAD_FUZZER_SLEEP_PROBABILITY");
initFromEnv(sleep_time_us, "THREAD_FUZZER_SLEEP_TIME_US");
initFromEnv(sleep_time_us_max, "THREAD_FUZZER_SLEEP_TIME_US_MAX");
initFromEnv(explicit_sleep_probability, "THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY");
initFromEnv(explicit_memory_exception_probability, "THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY");
@ -119,13 +119,12 @@ void ThreadFuzzer::initConfiguration()
initFromEnv(NAME##_before_yield_probability, "THREAD_FUZZER_" #NAME "_BEFORE_YIELD_PROBABILITY"); \
initFromEnv(NAME##_before_migrate_probability, "THREAD_FUZZER_" #NAME "_BEFORE_MIGRATE_PROBABILITY"); \
initFromEnv(NAME##_before_sleep_probability, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_PROBABILITY"); \
initFromEnv(NAME##_before_sleep_time_us, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_TIME_US"); \
initFromEnv(NAME##_before_sleep_time_us_max, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_TIME_US_MAX"); \
\
initFromEnv(NAME##_after_yield_probability, "THREAD_FUZZER_" #NAME "_AFTER_YIELD_PROBABILITY"); \
initFromEnv(NAME##_after_migrate_probability, "THREAD_FUZZER_" #NAME "_AFTER_MIGRATE_PROBABILITY"); \
initFromEnv(NAME##_after_sleep_probability, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_PROBABILITY"); \
initFromEnv(NAME##_after_sleep_time_us, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US");
initFromEnv(NAME##_after_sleep_time_us_max, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US_MAX");
FOR_EACH_WRAPPED_FUNCTION(INIT_WRAPPER_PARAMS)
# undef INIT_WRAPPER_PARAMS
@ -146,7 +145,7 @@ bool ThreadFuzzer::isEffective() const
return true; \
if (NAME##_before_sleep_probability.load(std::memory_order_relaxed) > 0.0) \
return true; \
if (NAME##_before_sleep_time_us.load(std::memory_order_relaxed) > 0.0) \
if (NAME##_before_sleep_time_us_max.load(std::memory_order_relaxed) > 0.0) \
return true; \
\
if (NAME##_after_yield_probability.load(std::memory_order_relaxed) > 0.0) \
@ -155,7 +154,7 @@ bool ThreadFuzzer::isEffective() const
return true; \
if (NAME##_after_sleep_probability.load(std::memory_order_relaxed) > 0.0) \
return true; \
if (NAME##_after_sleep_time_us.load(std::memory_order_relaxed) > 0.0) \
if (NAME##_after_sleep_time_us_max.load(std::memory_order_relaxed) > 0.0) \
return true;
FOR_EACH_WRAPPED_FUNCTION(CHECK_WRAPPER_PARAMS)
@ -166,7 +165,7 @@ bool ThreadFuzzer::isEffective() const
return cpu_time_period_us != 0
&& (yield_probability > 0
|| migrate_probability > 0
|| (sleep_probability > 0 && sleep_time_us > 0));
|| (sleep_probability > 0 && sleep_time_us_max > 0));
}
void ThreadFuzzer::stop()
@ -190,7 +189,7 @@ static void injectionImpl(
double yield_probability,
double migrate_probability,
double sleep_probability,
double sleep_time_us)
double sleep_time_us_max)
{
DENY_ALLOCATIONS_IN_SCOPE;
if (!ThreadFuzzer::isStarted())
@ -221,10 +220,10 @@ static void injectionImpl(
#endif
if (sleep_probability > 0
&& sleep_time_us > 0
&& sleep_time_us_max > 0
&& std::bernoulli_distribution(sleep_probability)(thread_local_rng))
{
sleepForNanoseconds(static_cast<uint64_t>(sleep_time_us * 1000));
sleepForNanoseconds((thread_local_rng() % static_cast<uint64_t>(sleep_time_us_max)) * 1000); /*may sleep(0)*/
}
}
@ -232,19 +231,19 @@ static ALWAYS_INLINE void injection(
double yield_probability,
double migrate_probability,
double sleep_probability,
double sleep_time_us)
double sleep_time_us_max)
{
DENY_ALLOCATIONS_IN_SCOPE;
if (!ThreadFuzzer::isStarted())
return;
injectionImpl(yield_probability, migrate_probability, sleep_probability, sleep_time_us);
injectionImpl(yield_probability, migrate_probability, sleep_probability, sleep_time_us_max);
}
void ThreadFuzzer::maybeInjectSleep()
{
auto & fuzzer = ThreadFuzzer::instance();
injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.explicit_sleep_probability, fuzzer.sleep_time_us);
injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.explicit_sleep_probability, fuzzer.sleep_time_us_max);
}
/// Sometimes maybeInjectSleep() is not enough and we need to inject an exception.
@ -265,7 +264,7 @@ void ThreadFuzzer::signalHandler(int)
DENY_ALLOCATIONS_IN_SCOPE;
auto saved_errno = errno;
auto & fuzzer = ThreadFuzzer::instance();
injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.sleep_probability, fuzzer.sleep_time_us);
injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.sleep_probability, fuzzer.sleep_time_us_max);
errno = saved_errno;
}
@ -309,13 +308,13 @@ void ThreadFuzzer::setup() const
NAME##_before_yield_probability.load(std::memory_order_relaxed), \
NAME##_before_migrate_probability.load(std::memory_order_relaxed), \
NAME##_before_sleep_probability.load(std::memory_order_relaxed), \
NAME##_before_sleep_time_us.load(std::memory_order_relaxed));
NAME##_before_sleep_time_us_max.load(std::memory_order_relaxed));
#define INJECTION_AFTER(NAME) \
injectionImpl( \
NAME##_after_yield_probability.load(std::memory_order_relaxed), \
NAME##_after_migrate_probability.load(std::memory_order_relaxed), \
NAME##_after_sleep_probability.load(std::memory_order_relaxed), \
NAME##_after_sleep_time_us.load(std::memory_order_relaxed));
NAME##_after_sleep_time_us_max.load(std::memory_order_relaxed));
/// ThreadFuzzer intercepts pthread_mutex_lock()/pthread_mutex_unlock().
///

10
src/Common/ThreadFuzzer.h
View File

@ -16,7 +16,7 @@ namespace DB
* THREAD_FUZZER_YIELD_PROBABILITY - probability to do 'sched_yield'.
* THREAD_FUZZER_MIGRATE_PROBABILITY - probability to set CPU affinity to random CPU core.
* THREAD_FUZZER_SLEEP_PROBABILITY - probability to sleep.
* THREAD_FUZZER_SLEEP_TIME_US - amount of time to sleep in microseconds.
* THREAD_FUZZER_SLEEP_TIME_US_MAX - max amount of time to sleep in microseconds, actual sleep time is randomized.
*
* ThreadFuzzer will do nothing if environment variables are not set accordingly.
*
@ -33,16 +33,15 @@ namespace DB
*
* Notes:
* - it can be also implemented with instrumentation (example: LLVM Xray) instead of signals.
* - we should also make the sleep time random.
* - sleep and migration obviously helps, but the effect of yield is unclear.
*
* In addition, we allow to inject glitches around thread synchronization functions.
* Example:
*
* THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
* THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
* THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000
* THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
* THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
* THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000
*/
class ThreadFuzzer
{
@ -67,7 +66,8 @@ private:
double yield_probability = 0;
double migrate_probability = 0;
double sleep_probability = 0;
double sleep_time_us = 0;
double sleep_time_us_max = 0;
double explicit_sleep_probability = 0;
double explicit_memory_exception_probability = 0;

2
src/Common/examples/chaos_sanitizer.cpp
View File

@ -12,7 +12,7 @@
/** Proves that ThreadFuzzer helps to find concurrency bugs.
*
* for i in {1..10}; do ./chaos_sanitizer 1000000; done
* for i in {1..10}; do THREAD_FUZZER_CPU_TIME_PERIOD_US=1000 THREAD_FUZZER_SLEEP_PROBABILITY=0.1 THREAD_FUZZER_SLEEP_TIME_US=100000 ./chaos_sanitizer 1000000; done
* for i in {1..10}; do THREAD_FUZZER_CPU_TIME_PERIOD_US=1000 THREAD_FUZZER_SLEEP_PROBABILITY=0.1 THREAD_FUZZER_SLEEP_TIME_US_MAX=100000 ./chaos_sanitizer 1000000; done
*/
int main(int argc, char ** argv)
{

10
utils/keeper-overload/keeper-overload.py
View File

@ -102,7 +102,7 @@ class Keeper:
if self.with_thread_fuzzer:
env["THREAD_FUZZER_CPU_TIME_PERIOD_US"] = "1000"
env["THREAD_FUZZER_SLEEP_PROBABILITY"] = "0.1"
env["THREAD_FUZZER_SLEEP_TIME_US"] = "100000"
env["THREAD_FUZZER_SLEEP_TIME_US_MAX"] = "100000"
env["THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY"] = "1"
env["THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY"] = "1"
env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_MIGRATE_PROBABILITY"] = "1"
@ -112,10 +112,10 @@ class Keeper:
env["THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY"] = "0.001"
env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY"] = "0.001"
env["THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY"] = "0.001"
env["THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US"] = "10000"
env["THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US"] = "10000"
env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US"] = "10000"
env["THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US"] = "10000"
env["THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX"] = "10000"
env["THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX"] = "10000"
env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX"] = "10000"
env["THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX"] = "10000"
self.process = subprocess.Popen(
[