thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

thread-fuzzer: randomize sleep time

Browse Source
This commit is contained in:
tomershafir 2024-03-15 00:02:09 +02:00
parent 48ec419b9d
commit 23169117ae
6 changed files with 38 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docker/test/stateless/run.sh
View File

@ -77,7 +77,7 @@ fi
if [ "$NUM_TRIES" -gt "1" ]; then if [ "$NUM_TRIES" -gt "1" ]; then
export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000 export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
export THREAD_FUZZER_SLEEP_PROBABILITY=0.1 export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
export THREAD_FUZZER_SLEEP_TIME_US=100000 export THREAD_FUZZER_SLEEP_TIME_US_MAX=100000
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1 export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1 export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
@ -88,10 +88,10 @@ if [ "$NUM_TRIES" -gt "1" ]; then
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000 export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000 export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000 export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000 export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX=10000
mkdir -p /var/run/clickhouse-server mkdir -p /var/run/clickhouse-server
# simplest way to forward env variables to server # simplest way to forward env variables to server

10
docker/test/stress/run.sh
View File

@ -27,7 +27,7 @@ install_packages package_folder
# and find more potential issues. # and find more potential issues.
export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000 export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
export THREAD_FUZZER_SLEEP_PROBABILITY=0.1 export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
export THREAD_FUZZER_SLEEP_TIME_US=100000 export THREAD_FUZZER_SLEEP_TIME_US_MAX=100000
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1 export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1 export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
@ -38,11 +38,11 @@ export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000 export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000 export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000 export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000 export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX=10000
export THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY=0.01 export THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY=0.01
export THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY=0.01 export THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY=0.01

35
src/Common/ThreadFuzzer.cpp
View File

@ -86,12 +86,12 @@ static std::atomic<int> num_cpus = 0;
static std::atomic<double> NAME##_before_yield_probability = 0; \ static std::atomic<double> NAME##_before_yield_probability = 0; \
static std::atomic<double> NAME##_before_migrate_probability = 0; \ static std::atomic<double> NAME##_before_migrate_probability = 0; \
static std::atomic<double> NAME##_before_sleep_probability = 0; \ static std::atomic<double> NAME##_before_sleep_probability = 0; \
static std::atomic<double> NAME##_before_sleep_time_us = 0; \ static std::atomic<double> NAME##_before_sleep_time_us_max = 0; \
\ \
static std::atomic<double> NAME##_after_yield_probability = 0; \ static std::atomic<double> NAME##_after_yield_probability = 0; \
static std::atomic<double> NAME##_after_migrate_probability = 0; \ static std::atomic<double> NAME##_after_migrate_probability = 0; \
static std::atomic<double> NAME##_after_sleep_probability = 0; \ static std::atomic<double> NAME##_after_sleep_probability = 0; \
static std::atomic<double> NAME##_after_sleep_time_us = 0; static std::atomic<double> NAME##_after_sleep_time_us_max = 0;
FOR_EACH_WRAPPED_FUNCTION(DEFINE_WRAPPER_PARAMS) FOR_EACH_WRAPPED_FUNCTION(DEFINE_WRAPPER_PARAMS)
@ -110,7 +110,7 @@ void ThreadFuzzer::initConfiguration()
initFromEnv(yield_probability, "THREAD_FUZZER_YIELD_PROBABILITY"); initFromEnv(yield_probability, "THREAD_FUZZER_YIELD_PROBABILITY");
initFromEnv(migrate_probability, "THREAD_FUZZER_MIGRATE_PROBABILITY"); initFromEnv(migrate_probability, "THREAD_FUZZER_MIGRATE_PROBABILITY");
initFromEnv(sleep_probability, "THREAD_FUZZER_SLEEP_PROBABILITY"); initFromEnv(sleep_probability, "THREAD_FUZZER_SLEEP_PROBABILITY");
initFromEnv(sleep_time_us, "THREAD_FUZZER_SLEEP_TIME_US"); initFromEnv(sleep_time_us_max, "THREAD_FUZZER_SLEEP_TIME_US_MAX");
initFromEnv(explicit_sleep_probability, "THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY"); initFromEnv(explicit_sleep_probability, "THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY");
initFromEnv(explicit_memory_exception_probability, "THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY"); initFromEnv(explicit_memory_exception_probability, "THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY");
@ -119,13 +119,12 @@ void ThreadFuzzer::initConfiguration()
initFromEnv(NAME##_before_yield_probability, "THREAD_FUZZER_" #NAME "_BEFORE_YIELD_PROBABILITY"); \ initFromEnv(NAME##_before_yield_probability, "THREAD_FUZZER_" #NAME "_BEFORE_YIELD_PROBABILITY"); \
initFromEnv(NAME##_before_migrate_probability, "THREAD_FUZZER_" #NAME "_BEFORE_MIGRATE_PROBABILITY"); \ initFromEnv(NAME##_before_migrate_probability, "THREAD_FUZZER_" #NAME "_BEFORE_MIGRATE_PROBABILITY"); \
initFromEnv(NAME##_before_sleep_probability, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_PROBABILITY"); \ initFromEnv(NAME##_before_sleep_probability, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_PROBABILITY"); \
initFromEnv(NAME##_before_sleep_time_us, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_TIME_US"); \ initFromEnv(NAME##_before_sleep_time_us_max, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_TIME_US_MAX"); \
\ \
initFromEnv(NAME##_after_yield_probability, "THREAD_FUZZER_" #NAME "_AFTER_YIELD_PROBABILITY"); \ initFromEnv(NAME##_after_yield_probability, "THREAD_FUZZER_" #NAME "_AFTER_YIELD_PROBABILITY"); \
initFromEnv(NAME##_after_migrate_probability, "THREAD_FUZZER_" #NAME "_AFTER_MIGRATE_PROBABILITY"); \ initFromEnv(NAME##_after_migrate_probability, "THREAD_FUZZER_" #NAME "_AFTER_MIGRATE_PROBABILITY"); \
initFromEnv(NAME##_after_sleep_probability, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_PROBABILITY"); \ initFromEnv(NAME##_after_sleep_probability, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_PROBABILITY"); \
initFromEnv(NAME##_after_sleep_time_us, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US"); initFromEnv(NAME##_after_sleep_time_us_max, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US_MAX");
FOR_EACH_WRAPPED_FUNCTION(INIT_WRAPPER_PARAMS) FOR_EACH_WRAPPED_FUNCTION(INIT_WRAPPER_PARAMS)
# undef INIT_WRAPPER_PARAMS # undef INIT_WRAPPER_PARAMS
@ -146,7 +145,7 @@ bool ThreadFuzzer::isEffective() const
return true; \ return true; \
if (NAME##_before_sleep_probability.load(std::memory_order_relaxed) > 0.0) \ if (NAME##_before_sleep_probability.load(std::memory_order_relaxed) > 0.0) \
return true; \ return true; \
if (NAME##_before_sleep_time_us.load(std::memory_order_relaxed) > 0.0) \ if (NAME##_before_sleep_time_us_max.load(std::memory_order_relaxed) > 0.0) \
return true; \ return true; \
\ \
if (NAME##_after_yield_probability.load(std::memory_order_relaxed) > 0.0) \ if (NAME##_after_yield_probability.load(std::memory_order_relaxed) > 0.0) \
@ -155,7 +154,7 @@ bool ThreadFuzzer::isEffective() const
return true; \ return true; \
if (NAME##_after_sleep_probability.load(std::memory_order_relaxed) > 0.0) \ if (NAME##_after_sleep_probability.load(std::memory_order_relaxed) > 0.0) \
return true; \ return true; \
if (NAME##_after_sleep_time_us.load(std::memory_order_relaxed) > 0.0) \ if (NAME##_after_sleep_time_us_max.load(std::memory_order_relaxed) > 0.0) \
return true; return true;
FOR_EACH_WRAPPED_FUNCTION(CHECK_WRAPPER_PARAMS) FOR_EACH_WRAPPED_FUNCTION(CHECK_WRAPPER_PARAMS)
@ -166,7 +165,7 @@ bool ThreadFuzzer::isEffective() const
return cpu_time_period_us != 0 return cpu_time_period_us != 0
&& (yield_probability > 0 && (yield_probability > 0
|| migrate_probability > 0 || migrate_probability > 0
|| (sleep_probability > 0 && sleep_time_us > 0)); || (sleep_probability > 0 && sleep_time_us_max > 0));
} }
void ThreadFuzzer::stop() void ThreadFuzzer::stop()
@ -190,7 +189,7 @@ static void injectionImpl(
double yield_probability, double yield_probability,
double migrate_probability, double migrate_probability,
double sleep_probability, double sleep_probability,
double sleep_time_us) double sleep_time_us_max)
{ {
DENY_ALLOCATIONS_IN_SCOPE; DENY_ALLOCATIONS_IN_SCOPE;
if (!ThreadFuzzer::isStarted()) if (!ThreadFuzzer::isStarted())
@ -221,10 +220,10 @@ static void injectionImpl(
#endif #endif
if (sleep_probability > 0 if (sleep_probability > 0
&& sleep_time_us > 0 && sleep_time_us_max > 0
&& std::bernoulli_distribution(sleep_probability)(thread_local_rng)) && std::bernoulli_distribution(sleep_probability)(thread_local_rng))
{ {
sleepForNanoseconds(static_cast<uint64_t>(sleep_time_us * 1000)); sleepForNanoseconds((thread_local_rng() % static_cast<uint64_t>(sleep_time_us_max)) * 1000); /*may sleep(0)*/
} }
} }
@ -232,19 +231,19 @@ static ALWAYS_INLINE void injection(
double yield_probability, double yield_probability,
double migrate_probability, double migrate_probability,
double sleep_probability, double sleep_probability,
double sleep_time_us) double sleep_time_us_max)
{ {
DENY_ALLOCATIONS_IN_SCOPE; DENY_ALLOCATIONS_IN_SCOPE;
if (!ThreadFuzzer::isStarted()) if (!ThreadFuzzer::isStarted())
return; return;
injectionImpl(yield_probability, migrate_probability, sleep_probability, sleep_time_us); injectionImpl(yield_probability, migrate_probability, sleep_probability, sleep_time_us_max);
} }
void ThreadFuzzer::maybeInjectSleep() void ThreadFuzzer::maybeInjectSleep()
{ {
auto & fuzzer = ThreadFuzzer::instance(); auto & fuzzer = ThreadFuzzer::instance();
injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.explicit_sleep_probability, fuzzer.sleep_time_us); injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.explicit_sleep_probability, fuzzer.sleep_time_us_max);
} }
/// Sometimes maybeInjectSleep() is not enough and we need to inject an exception. /// Sometimes maybeInjectSleep() is not enough and we need to inject an exception.
@ -265,7 +264,7 @@ void ThreadFuzzer::signalHandler(int)
DENY_ALLOCATIONS_IN_SCOPE; DENY_ALLOCATIONS_IN_SCOPE;
auto saved_errno = errno; auto saved_errno = errno;
auto & fuzzer = ThreadFuzzer::instance(); auto & fuzzer = ThreadFuzzer::instance();
injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.sleep_probability, fuzzer.sleep_time_us); injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.sleep_probability, fuzzer.sleep_time_us_max);
errno = saved_errno; errno = saved_errno;
} }
@ -309,13 +308,13 @@ void ThreadFuzzer::setup() const
NAME##_before_yield_probability.load(std::memory_order_relaxed), \ NAME##_before_yield_probability.load(std::memory_order_relaxed), \
NAME##_before_migrate_probability.load(std::memory_order_relaxed), \ NAME##_before_migrate_probability.load(std::memory_order_relaxed), \
NAME##_before_sleep_probability.load(std::memory_order_relaxed), \ NAME##_before_sleep_probability.load(std::memory_order_relaxed), \
NAME##_before_sleep_time_us.load(std::memory_order_relaxed)); NAME##_before_sleep_time_us_max.load(std::memory_order_relaxed));
#define INJECTION_AFTER(NAME) \ #define INJECTION_AFTER(NAME) \
injectionImpl( \ injectionImpl( \
NAME##_after_yield_probability.load(std::memory_order_relaxed), \ NAME##_after_yield_probability.load(std::memory_order_relaxed), \
NAME##_after_migrate_probability.load(std::memory_order_relaxed), \ NAME##_after_migrate_probability.load(std::memory_order_relaxed), \
NAME##_after_sleep_probability.load(std::memory_order_relaxed), \ NAME##_after_sleep_probability.load(std::memory_order_relaxed), \
NAME##_after_sleep_time_us.load(std::memory_order_relaxed)); NAME##_after_sleep_time_us_max.load(std::memory_order_relaxed));
/// ThreadFuzzer intercepts pthread_mutex_lock()/pthread_mutex_unlock(). /// ThreadFuzzer intercepts pthread_mutex_lock()/pthread_mutex_unlock().
/// ///

10
src/Common/ThreadFuzzer.h
View File

@ -16,7 +16,7 @@ namespace DB
* THREAD_FUZZER_YIELD_PROBABILITY - probability to do 'sched_yield'. * THREAD_FUZZER_YIELD_PROBABILITY - probability to do 'sched_yield'.
* THREAD_FUZZER_MIGRATE_PROBABILITY - probability to set CPU affinity to random CPU core. * THREAD_FUZZER_MIGRATE_PROBABILITY - probability to set CPU affinity to random CPU core.
* THREAD_FUZZER_SLEEP_PROBABILITY - probability to sleep. * THREAD_FUZZER_SLEEP_PROBABILITY - probability to sleep.
* THREAD_FUZZER_SLEEP_TIME_US - amount of time to sleep in microseconds. * THREAD_FUZZER_SLEEP_TIME_US_MAX - max amount of time to sleep in microseconds, actual sleep time is randomized.
* *
* ThreadFuzzer will do nothing if environment variables are not set accordingly. * ThreadFuzzer will do nothing if environment variables are not set accordingly.
* *
@ -33,16 +33,15 @@ namespace DB
* *
* Notes: * Notes:
* - it can be also implemented with instrumentation (example: LLVM Xray) instead of signals. * - it can be also implemented with instrumentation (example: LLVM Xray) instead of signals.
* - we should also make the sleep time random.
* - sleep and migration obviously helps, but the effect of yield is unclear. * - sleep and migration obviously helps, but the effect of yield is unclear.
* *
* In addition, we allow to inject glitches around thread synchronization functions. * In addition, we allow to inject glitches around thread synchronization functions.
* Example: * Example:
* *
* THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001 * THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
* THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000 * THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000
* THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001 * THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
* THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000 * THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000
*/ */
class ThreadFuzzer class ThreadFuzzer
{ {
@ -67,7 +66,8 @@ private:
double yield_probability = 0; double yield_probability = 0;
double migrate_probability = 0; double migrate_probability = 0;
double sleep_probability = 0; double sleep_probability = 0;
double sleep_time_us = 0; double sleep_time_us_max = 0;
double explicit_sleep_probability = 0; double explicit_sleep_probability = 0;
double explicit_memory_exception_probability = 0; double explicit_memory_exception_probability = 0;

2
src/Common/examples/chaos_sanitizer.cpp
View File

@ -12,7 +12,7 @@
/** Proves that ThreadFuzzer helps to find concurrency bugs. /** Proves that ThreadFuzzer helps to find concurrency bugs.
* *
* for i in {1..10}; do ./chaos_sanitizer 1000000; done * for i in {1..10}; do ./chaos_sanitizer 1000000; done
* for i in {1..10}; do THREAD_FUZZER_CPU_TIME_PERIOD_US=1000 THREAD_FUZZER_SLEEP_PROBABILITY=0.1 THREAD_FUZZER_SLEEP_TIME_US=100000 ./chaos_sanitizer 1000000; done * for i in {1..10}; do THREAD_FUZZER_CPU_TIME_PERIOD_US=1000 THREAD_FUZZER_SLEEP_PROBABILITY=0.1 THREAD_FUZZER_SLEEP_TIME_US_MAX=100000 ./chaos_sanitizer 1000000; done
*/ */
int main(int argc, char ** argv) int main(int argc, char ** argv)
{ {

10
utils/keeper-overload/keeper-overload.py
View File

@ -102,7 +102,7 @@ class Keeper:
if self.with_thread_fuzzer: if self.with_thread_fuzzer:
env["THREAD_FUZZER_CPU_TIME_PERIOD_US"] = "1000" env["THREAD_FUZZER_CPU_TIME_PERIOD_US"] = "1000"
env["THREAD_FUZZER_SLEEP_PROBABILITY"] = "0.1" env["THREAD_FUZZER_SLEEP_PROBABILITY"] = "0.1"
env["THREAD_FUZZER_SLEEP_TIME_US"] = "100000" env["THREAD_FUZZER_SLEEP_TIME_US_MAX"] = "100000"
env["THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY"] = "1" env["THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY"] = "1"
env["THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY"] = "1" env["THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY"] = "1"
env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_MIGRATE_PROBABILITY"] = "1" env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_MIGRATE_PROBABILITY"] = "1"
@ -112,10 +112,10 @@ class Keeper:
env["THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY"] = "0.001" env["THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY"] = "0.001"
env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY"] = "0.001" env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY"] = "0.001"
env["THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY"] = "0.001" env["THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY"] = "0.001"
env["THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US"] = "10000" env["THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX"] = "10000"
env["THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US"] = "10000" env["THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX"] = "10000"
env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US"] = "10000" env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX"] = "10000"
env["THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US"] = "10000" env["THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX"] = "10000"
self.process = subprocess.Popen( self.process = subprocess.Popen(
[ [