diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 41245013a4a..dc2b184dc5a 100644 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -53,6 +53,7 @@ function configure() local total_mem total_mem=$(awk '/MemTotal/ { print $(NF-1) }' /proc/meminfo) # KiB total_mem=$(( total_mem*1024 )) # bytes + # Set maximum memory usage as half of total memory (less chance of OOM). # # But not via max_server_memory_usage but via max_memory_usage_for_user, @@ -65,16 +66,17 @@ function configure() # max_server_memory_usage will be hard limit, and queries that should be # executed regardless memory limits will use max_memory_usage_for_user=0, # instead of relying on max_untracked_memory - local max_server_mem - max_server_mem=$((total_mem*75/100)) # 75% - echo "Setting max_server_memory_usage=$max_server_mem" + + max_server_memory_usage_to_ram_ratio=0.5 + echo "Setting max_server_memory_usage_to_ram_ratio to ${max_server_memory_usage_to_ram_ratio}" cat > /etc/clickhouse-server/config.d/max_server_memory_usage.xml < - ${max_server_mem} + ${max_server_memory_usage_to_ram_ratio} EOL + local max_users_mem - max_users_mem=$((total_mem*50/100)) # 50% + max_users_mem=$((total_mem*30/100)) # 30% echo "Setting max_memory_usage_for_user=$max_users_mem" cat > /etc/clickhouse-server/users.d/max_memory_usage_for_user.xml < @@ -97,6 +99,13 @@ EOL --> $PWD +EOL + + # Let OOM killer terminate other processes before clickhouse-server: + cat > /etc/clickhouse-server/config.d/oom_score.xml < + -1000 + EOL # Analyzer is not yet ready for testing diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index b43c9a2a021..a4ddd8553c6 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -420,6 +420,33 @@ void Server::createServer( } } + +#if defined(OS_LINUX) +namespace +{ + +void setOOMScore(int value, Poco::Logger * log) +{ + try + { + std::string value_string = std::to_string(value); + DB::WriteBufferFromFile buf("/proc/self/oom_score_adj"); + buf.write(value_string.c_str(), value_string.size()); + buf.next(); + buf.close(); + } + catch (const Poco::Exception & e) + { + LOG_WARNING(log, "Failed to adjust OOM score: '{}'.", e.displayText()); + return; + } + LOG_INFO(log, "Set OOM score adjustment to {}", value); +} + +} +#endif + + void Server::uninitialize() { logger().information("shutting down"); @@ -881,6 +908,21 @@ try } } } + + int default_oom_score = 0; + +#if !defined(NDEBUG) + /// In debug version on Linux, increase oom score so that clickhouse is killed + /// first, instead of some service. Use a carefully chosen random score of 555: + /// the maximum is 1000, and chromium uses 300 for its tab processes. Ignore + /// whatever errors that occur, because it's just a debugging aid and we don't + /// care if it breaks. + default_oom_score = 555; +#endif + + int oom_score = config().getInt("oom_score", default_oom_score); + if (oom_score) + setOOMScore(oom_score, log); #endif global_context->setRemoteHostFilter(config()); diff --git a/programs/server/config.xml b/programs/server/config.xml index 0cbc3d9339e..5f4a9fb983f 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -1464,4 +1464,8 @@ I don't recommend to change this setting. false --> + + diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index 5a08c8ffcd0..d4e76a0cad8 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -602,34 +602,6 @@ void BaseDaemon::closeFDs() } } -namespace -{ -/// In debug version on Linux, increase oom score so that clickhouse is killed -/// first, instead of some service. Use a carefully chosen random score of 555: -/// the maximum is 1000, and chromium uses 300 for its tab processes. Ignore -/// whatever errors that occur, because it's just a debugging aid and we don't -/// care if it breaks. -#if defined(OS_LINUX) && !defined(NDEBUG) -void debugIncreaseOOMScore() -{ - const std::string new_score = "555"; - try - { - DB::WriteBufferFromFile buf("/proc/self/oom_score_adj"); - buf.write(new_score.c_str(), new_score.size()); - buf.close(); - } - catch (const Poco::Exception & e) - { - LOG_WARNING(&Poco::Logger::root(), "Failed to adjust OOM score: '{}'.", e.displayText()); - return; - } - LOG_INFO(&Poco::Logger::root(), "Set OOM score adjustment to {}", new_score); -} -#else -void debugIncreaseOOMScore() {} -#endif -} void BaseDaemon::initialize(Application & self) { @@ -796,7 +768,6 @@ void BaseDaemon::initialize(Application & self) initializeTerminationAndSignalProcessing(); logRevision(); - debugIncreaseOOMScore(); for (const auto & key : DB::getMultipleKeysFromConfig(config(), "", "graphite")) {