ClickHouse/src/Common/OvercommitTracker.cpp

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

215 lines
6.7 KiB
C++
Raw Normal View History

2021-10-22 12:56:09 +00:00
#include "OvercommitTracker.h"
2021-10-22 15:15:33 +00:00
#include <chrono>
2021-12-06 22:16:09 +00:00
#include <mutex>
#include <Common/ProfileEvents.h>
2022-12-14 16:56:19 +00:00
#include <Common/CurrentMetrics.h>
2021-10-22 12:56:09 +00:00
#include <Interpreters/ProcessList.h>
2022-12-14 16:56:19 +00:00
namespace CurrentMetrics
{
extern const Metric ThreadsInOvercommitTracker;
}
2022-10-17 02:21:08 +00:00
namespace ProfileEvents
{
extern const Event MemoryOvercommitWaitTimeMicroseconds;
}
2021-10-22 15:15:33 +00:00
using namespace std::chrono_literals;
constexpr std::chrono::microseconds ZERO_MICROSEC = 0us;
2022-08-29 18:24:35 +00:00
OvercommitTracker::OvercommitTracker(DB::ProcessList * process_list_)
: picked_tracker(nullptr)
2022-08-29 18:24:35 +00:00
, process_list(process_list_)
, cancellation_state(QueryCancellationState::NONE)
, freed_memory(0)
, required_memory(0)
, next_id(0)
, id_to_release(0)
, allow_release(true)
2021-10-22 15:15:33 +00:00
{}
OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount)
2021-10-22 12:56:09 +00:00
{
DENY_ALLOCATIONS_IN_SCOPE;
if (OvercommitTrackerBlockerInThread::isBlocked())
return OvercommitResult::NONE;
2022-12-14 18:05:07 +00:00
CurrentMetrics::Increment metric_increment(CurrentMetrics::ThreadsInOvercommitTracker);
// NOTE: Do not change the order of locks
2022-02-21 15:45:17 +00:00
//
2022-08-29 18:24:35 +00:00
// global mutex must be acquired before overcommit_m, because
// method OvercommitTracker::onQueryStop(MemoryTracker *) is
2022-08-29 18:24:35 +00:00
// always called with already acquired global mutex in
2022-02-21 15:45:17 +00:00
// ProcessListEntry::~ProcessListEntry().
2022-08-29 18:24:35 +00:00
auto global_lock = process_list->unsafeLock();
2021-10-22 12:56:09 +00:00
std::unique_lock<std::mutex> lk(overcommit_m);
size_t id = next_id++;
auto max_wait_time = tracker->getOvercommitWaitingTime();
if (max_wait_time == ZERO_MICROSEC)
return OvercommitResult::DISABLED;
2021-10-22 12:56:09 +00:00
pickQueryToExclude();
assert(cancellation_state != QueryCancellationState::NONE);
2022-02-16 20:02:14 +00:00
global_lock.unlock();
2022-01-18 12:21:59 +00:00
// If no query was chosen we need to stop current query.
// This may happen if no soft limit is set.
if (picked_tracker == nullptr)
{
2022-05-02 23:02:43 +00:00
// Here state can not be RUNNING, because it requires
// picked_tracker to be not null pointer.
assert(cancellation_state == QueryCancellationState::SELECTED);
cancellation_state = QueryCancellationState::NONE;
return OvercommitResult::DISABLED;
}
if (picked_tracker == tracker)
{
2022-05-02 23:02:43 +00:00
// Query of the provided as an argument memory tracker was chosen.
// It may happen even when current state is RUNNING, because
// ThreadStatus::~ThreadStatus may call MemoryTracker::alloc.
cancellation_state = QueryCancellationState::RUNNING;
return OvercommitResult::SELECTED;
}
allow_release = true;
required_memory += amount;
auto wait_start_time = std::chrono::system_clock::now();
2022-12-14 18:05:07 +00:00
bool timeout = !cv.wait_for(lk, max_wait_time, [this, id]()
2021-10-22 12:56:09 +00:00
{
2022-12-14 18:05:07 +00:00
return id < id_to_release || cancellation_state == QueryCancellationState::NONE;
});
auto wait_end_time = std::chrono::system_clock::now();
ProfileEvents::increment(ProfileEvents::MemoryOvercommitWaitTimeMicroseconds, (wait_end_time - wait_start_time) / 1us);
required_memory -= amount;
bool still_need = !(id < id_to_release); // True if thread wasn't released
// If threads where not released since last call of this method,
// we can release them now.
if (allow_release && required_memory <= freed_memory && still_need)
releaseThreads();
// All required amount of memory is free now and selected query to stop doesn't know about it.
// As we don't need to free memory, we can continue execution of the selected query.
if (required_memory == 0 && cancellation_state == QueryCancellationState::SELECTED)
reset();
if (timeout)
return OvercommitResult::TIMEOUTED;
if (still_need)
return OvercommitResult::NOT_ENOUGH_FREED;
else
return OvercommitResult::MEMORY_FREED;
}
void OvercommitTracker::tryContinueQueryExecutionAfterFree(Int64 amount)
{
DENY_ALLOCATIONS_IN_SCOPE;
if (OvercommitTrackerBlockerInThread::isBlocked())
return;
std::lock_guard guard(overcommit_m);
if (cancellation_state != QueryCancellationState::NONE)
{
freed_memory += amount;
if (freed_memory >= required_memory)
releaseThreads();
}
}
void OvercommitTracker::onQueryStop(MemoryTracker * tracker)
{
DENY_ALLOCATIONS_IN_SCOPE;
std::lock_guard lk(overcommit_m);
if (picked_tracker == tracker)
{
reset();
cv.notify_all();
}
}
void OvercommitTracker::releaseThreads()
{
id_to_release = next_id;
freed_memory = 0;
allow_release = false; // To avoid repeating call of this method in OvercommitTracker::needToStopQuery
cv.notify_all();
}
2022-08-29 18:24:35 +00:00
UserOvercommitTracker::UserOvercommitTracker(DB::ProcessList * process_list_, DB::ProcessListForUser * user_process_list_)
: OvercommitTracker(process_list_)
2022-02-16 20:02:14 +00:00
, user_process_list(user_process_list_)
2021-10-22 12:56:09 +00:00
{}
void UserOvercommitTracker::pickQueryToExcludeImpl()
{
2022-01-18 12:21:59 +00:00
MemoryTracker * query_tracker = nullptr;
2021-10-22 12:56:09 +00:00
OvercommitRatio current_ratio{0, 0};
2022-01-18 12:21:59 +00:00
// At this moment query list must be read only.
2022-02-21 15:45:17 +00:00
// This is guaranteed by locking global_mutex in OvercommitTracker::needToStopQuery.
auto & queries = user_process_list->queries;
for (auto const & query : queries)
2021-10-22 12:56:09 +00:00
{
if (query.second->isKilled())
continue;
2021-10-22 12:56:09 +00:00
auto * memory_tracker = query.second->getMemoryTracker();
if (!memory_tracker)
continue;
2021-10-22 12:56:09 +00:00
auto ratio = memory_tracker->getOvercommitRatio();
if (ratio.soft_limit != 0 && current_ratio < ratio)
2021-10-22 12:56:09 +00:00
{
2022-01-18 12:21:59 +00:00
query_tracker = memory_tracker;
2021-10-22 12:56:09 +00:00
current_ratio = ratio;
}
}
2022-01-18 12:21:59 +00:00
picked_tracker = query_tracker;
2021-10-22 12:56:09 +00:00
}
2022-02-16 20:02:14 +00:00
GlobalOvercommitTracker::GlobalOvercommitTracker(DB::ProcessList * process_list_)
2022-08-29 18:24:35 +00:00
: OvercommitTracker(process_list_)
2022-10-17 02:21:08 +00:00
{
}
2022-02-16 20:02:14 +00:00
2021-10-22 12:56:09 +00:00
void GlobalOvercommitTracker::pickQueryToExcludeImpl()
{
2022-01-18 12:21:59 +00:00
MemoryTracker * query_tracker = nullptr;
2021-10-22 12:56:09 +00:00
OvercommitRatio current_ratio{0, 0};
2022-02-15 15:04:13 +00:00
// At this moment query list must be read only.
2022-02-21 15:45:17 +00:00
// This is guaranteed by locking global_mutex in OvercommitTracker::needToStopQuery.
for (auto const & query : process_list->processes)
2021-10-22 12:56:09 +00:00
{
2022-10-17 02:21:08 +00:00
if (query->isKilled())
2022-05-04 00:41:15 +00:00
continue;
2021-12-06 22:06:00 +00:00
Int64 user_soft_limit = 0;
2022-10-17 02:21:08 +00:00
if (auto const * user_process_list = query->getUserProcessList())
user_soft_limit = user_process_list->user_memory_tracker.getSoftLimit();
2021-12-06 22:06:00 +00:00
if (user_soft_limit == 0)
2022-05-04 00:41:15 +00:00
continue;
2022-10-17 02:21:08 +00:00
auto * memory_tracker = query->getMemoryTracker();
if (!memory_tracker)
2022-05-04 00:41:15 +00:00
continue;
auto ratio = memory_tracker->getOvercommitRatio(user_soft_limit);
2021-10-22 12:56:09 +00:00
if (current_ratio < ratio)
{
2022-01-18 12:21:59 +00:00
query_tracker = memory_tracker;
2021-10-22 12:56:09 +00:00
current_ratio = ratio;
}
2022-02-21 15:45:17 +00:00
}
2022-01-18 12:21:59 +00:00
picked_tracker = query_tracker;
2021-10-22 12:56:09 +00:00
}
thread_local size_t OvercommitTrackerBlockerInThread::counter = 0;