This commit is contained in:
Alexander Tokmakov 2024-04-03 20:57:12 +02:00
parent 98ac8031e0
commit c53b20a770
7 changed files with 33 additions and 1 deletions

View File

@ -1569,7 +1569,8 @@ try
new_server_settings.http_connections_store_limit,
});
CannotAllocateThreadFaultInjector::setFaultProbability(new_server_settings.cannot_allocate_thread_fault_injection_probability);
if (global_context->isServerCompletelyStarted())
CannotAllocateThreadFaultInjector::setFaultProbability(new_server_settings.cannot_allocate_thread_fault_injection_probability);
ProfileEvents::increment(ProfileEvents::MainConfigLoads);

View File

@ -873,6 +873,7 @@ void AsyncLoader::spawn(Pool & pool, std::unique_lock<std::mutex> & lock)
ALLOW_ALLOCATIONS_IN_SCOPE;
if (log_events)
LOG_DEBUG(log, "Spawn loader worker #{} in {}", pool.workers, pool.name);
auto blocker = CannotAllocateThreadFaultInjector::blockFaultInjections();
pool.thread_pool->scheduleOrThrowOnError([this, &pool] { worker(pool); });
});
}

View File

@ -568,6 +568,21 @@ bool CannotAllocateThreadFaultInjector::injectFault()
if (!ins.enabled.load(std::memory_order_relaxed))
return false;
if (ins.block_fault_injections)
return false;
std::lock_guard lock(ins.mutex);
return ins.random && (*ins.random)(ins.rndgen);
}
thread_local bool CannotAllocateThreadFaultInjector::block_fault_injections = false;
scope_guard CannotAllocateThreadFaultInjector::blockFaultInjections()
{
auto & ins = instance();
if (!ins.enabled.load(std::memory_order_relaxed))
return {};
ins.block_fault_injections = true;
return [&ins](){ ins.block_fault_injections = false; };
}

View File

@ -334,8 +334,13 @@ class CannotAllocateThreadFaultInjector
std::mutex mutex;
pcg64_fast rndgen;
std::optional<std::bernoulli_distribution> random;
static thread_local bool block_fault_injections;
static CannotAllocateThreadFaultInjector & instance();
public:
static void setFaultProbability(double probability);
static bool injectFault();
static scope_guard blockFaultInjections();
};

View File

@ -1909,6 +1909,8 @@ try
auto runner = threadPoolCallbackRunner<void>(getOutdatedPartsLoadingThreadPool().get(), "OutdatedParts");
std::vector<std::future<void>> parts_futures;
auto blocker = CannotAllocateThreadFaultInjector::blockFaultInjections();
while (true)
{
ThreadFuzzer::maybeInjectSleep();

View File

@ -0,0 +1,7 @@
0 BBB
1 BBB
2 BBB
3 BBB
4 AAA
5 BBB
6 AAA
1 0 BBB
2 1 BBB
3 2 BBB
4 3 BBB
5 4 AAA
6 5 BBB
7 6 AAA

View File

@ -907,6 +907,7 @@ class MergeTreeSettingsRandomizer:
1, 32 * 1024 * 1024
),
"cache_populated_by_fetch": lambda: random.randint(0, 1),
"concurrent_part_removal_threshold": threshold_generator(0.2, 0.3, 0, 100)
}
@staticmethod