From d68ecdc84c121a628a18945dac4e3b35f2b643dd Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 7 Jul 2020 01:43:39 +0300 Subject: [PATCH] Cap max_memory_usage* limits to the process resident memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are still some issues with memory tracking, but now with per-user tracking: executeQuery: Code: 241, e.displayText() = DB::Exception: Memory limit (for user) exceeded: would use 437.72 GiB (attempt to allocate chunk of 4200926 bytes), maximum: 437.72 GiB (version 20.6.1.1) (from 10.7.140.7:31318) Although the server is mostly idle: SELECT formatReadableSize(memory_usage) FROM system.processes ┌─formatReadableSize(memory_usage)─┐ │ 289.28 MiB │ │ 155.75 MiB │ │ 0.00 B │ └──────────────────────────────────┘ Refs: https://github.com/ClickHouse/ClickHouse/pull/10496/files#r450206865 Cc: @alexey-milovidov --- src/Common/MemoryTracker.cpp | 11 +++++++++++ src/Interpreters/AsynchronousMetrics.cpp | 1 - 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 03bd8be94f3..b8ac09f9449 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -77,6 +77,17 @@ void MemoryTracker::alloc(Int64 size) Int64 current_hard_limit = hard_limit.load(std::memory_order_relaxed); Int64 current_profiler_limit = profiler_limit.load(std::memory_order_relaxed); + /// Cap the limit to the total_memory_tracker, since it may include some drift. + /// + /// And since total_memory_tracker is reseted to the process resident + /// memory peridically (in AsynchronousMetrics::update()), any limit can be + /// capped to it, to avoid possible drift. + if (unlikely(current_hard_limit && will_be > current_hard_limit)) + { + set(total_memory_tracker.amount); + will_be = size + amount.fetch_add(size, std::memory_order_relaxed); + } + std::bernoulli_distribution fault(fault_probability); if (unlikely(fault_probability && fault(thread_local_rng))) { diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index ac71a88dc00..5c734f4da2a 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -208,7 +208,6 @@ void AsynchronousMetrics::update() /// Otherwise it might be calculated incorrectly - it can include a "drift" of memory amount. /// See https://github.com/ClickHouse/ClickHouse/issues/10293 total_memory_tracker.set(data.resident); - CurrentMetrics::set(CurrentMetrics::MemoryTracking, data.resident); } #endif