2021-10-22 12:56:09 +00:00
|
|
|
#pragma once
|
|
|
|
|
2022-04-27 15:05:45 +00:00
|
|
|
#include <Common/logger_useful.h>
|
2021-10-22 12:56:09 +00:00
|
|
|
#include <base/types.h>
|
|
|
|
#include <boost/core/noncopyable.hpp>
|
2022-01-18 12:21:59 +00:00
|
|
|
#include <Poco/Logger.h>
|
2021-10-22 12:56:09 +00:00
|
|
|
#include <cassert>
|
2021-10-22 15:15:33 +00:00
|
|
|
#include <chrono>
|
2021-10-22 12:56:09 +00:00
|
|
|
#include <condition_variable>
|
|
|
|
#include <mutex>
|
|
|
|
#include <unordered_map>
|
|
|
|
|
2022-01-18 12:21:59 +00:00
|
|
|
// This struct is used for the comparison of query memory usage.
|
2021-10-22 12:56:09 +00:00
|
|
|
struct OvercommitRatio
|
|
|
|
{
|
2021-11-09 13:40:23 +00:00
|
|
|
OvercommitRatio(Int64 committed_, Int64 soft_limit_)
|
|
|
|
: committed(committed_)
|
2021-10-22 12:56:09 +00:00
|
|
|
, soft_limit(soft_limit_)
|
|
|
|
{}
|
|
|
|
|
2022-01-18 12:21:59 +00:00
|
|
|
friend bool operator<(OvercommitRatio const & lhs, OvercommitRatio const & rhs) noexcept
|
2021-10-22 12:56:09 +00:00
|
|
|
{
|
2022-05-13 15:39:05 +00:00
|
|
|
Int128 lhs_committed = lhs.committed, lhs_soft_limit = lhs.soft_limit;
|
|
|
|
Int128 rhs_committed = rhs.committed, rhs_soft_limit = rhs.soft_limit;
|
2021-10-26 13:55:57 +00:00
|
|
|
// (a / b < c / d) <=> (a * d < c * b)
|
2022-05-13 15:39:05 +00:00
|
|
|
return (lhs_committed * rhs_soft_limit) < (rhs_committed * lhs_soft_limit)
|
|
|
|
|| (lhs_soft_limit == 0 && rhs_soft_limit > 0)
|
|
|
|
|| (lhs_committed == 0 && rhs_committed == 0 && lhs_soft_limit > rhs_soft_limit);
|
2021-10-22 12:56:09 +00:00
|
|
|
}
|
|
|
|
|
2022-01-18 12:21:59 +00:00
|
|
|
// actual query memory usage
|
2021-11-09 13:40:23 +00:00
|
|
|
Int64 committed;
|
2022-01-18 12:21:59 +00:00
|
|
|
// guaranteed amount of memory query can use
|
2021-10-22 12:56:09 +00:00
|
|
|
Int64 soft_limit;
|
|
|
|
};
|
|
|
|
|
|
|
|
class MemoryTracker;
|
|
|
|
|
2022-05-23 14:35:09 +00:00
|
|
|
enum class OvercommitResult
|
|
|
|
{
|
|
|
|
NONE,
|
|
|
|
DISABLED,
|
|
|
|
MEMORY_FREED,
|
|
|
|
SELECTED,
|
|
|
|
TIMEOUTED,
|
|
|
|
NOT_ENOUGH_FREED,
|
|
|
|
};
|
|
|
|
|
2022-05-02 22:45:13 +00:00
|
|
|
enum class QueryCancellationState
|
|
|
|
{
|
|
|
|
NONE = 0, // Hard limit is not reached, there is no selected query to kill.
|
|
|
|
SELECTED = 1, // Hard limit is reached, query to stop was chosen but it still is not aware of cancellation.
|
|
|
|
RUNNING = 2, // Hard limit is reached, selected query has started the process of cancellation.
|
|
|
|
};
|
|
|
|
|
2022-01-18 12:21:59 +00:00
|
|
|
// Usually it's hard to set some reasonable hard memory limit
|
|
|
|
// (especially, the default value). This class introduces new
|
|
|
|
// mechanisim for the limiting of memory usage.
|
|
|
|
// Soft limit represents guaranteed amount of memory query/user
|
|
|
|
// may use. It's allowed to exceed this limit. But if hard limit
|
|
|
|
// is reached, query with the biggest overcommit ratio
|
|
|
|
// is killed to free memory.
|
2021-10-22 12:56:09 +00:00
|
|
|
struct OvercommitTracker : boost::noncopyable
|
|
|
|
{
|
2022-05-23 14:35:09 +00:00
|
|
|
OvercommitResult needToStopQuery(MemoryTracker * tracker, Int64 amount);
|
2022-05-02 22:45:13 +00:00
|
|
|
|
|
|
|
void tryContinueQueryExecutionAfterFree(Int64 amount);
|
2021-10-22 12:56:09 +00:00
|
|
|
|
2022-05-02 22:45:13 +00:00
|
|
|
void onQueryStop(MemoryTracker * tracker);
|
2021-10-22 12:56:09 +00:00
|
|
|
|
|
|
|
virtual ~OvercommitTracker() = default;
|
|
|
|
|
|
|
|
protected:
|
2022-02-21 15:45:17 +00:00
|
|
|
explicit OvercommitTracker(std::mutex & global_mutex_);
|
|
|
|
|
2021-10-22 12:56:09 +00:00
|
|
|
virtual void pickQueryToExcludeImpl() = 0;
|
|
|
|
|
2022-02-21 15:45:17 +00:00
|
|
|
// This mutex is used to disallow concurrent access
|
|
|
|
// to picked_tracker and cancelation_state variables.
|
2022-05-02 22:45:13 +00:00
|
|
|
std::mutex overcommit_m;
|
|
|
|
std::condition_variable cv;
|
2021-10-22 12:56:09 +00:00
|
|
|
|
2021-11-09 13:40:23 +00:00
|
|
|
// Specifies memory tracker of the chosen to stop query.
|
2021-10-26 13:21:58 +00:00
|
|
|
// If soft limit is not set, all the queries which reach hard limit must stop.
|
|
|
|
// This case is represented as picked tracker pointer is set to nullptr and
|
2022-05-02 22:45:13 +00:00
|
|
|
// overcommit tracker is in SELECTED state.
|
2021-10-22 12:56:09 +00:00
|
|
|
MemoryTracker * picked_tracker;
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
|
|
|
void pickQueryToExclude()
|
|
|
|
{
|
2022-05-02 22:45:13 +00:00
|
|
|
if (cancellation_state == QueryCancellationState::NONE)
|
2021-10-22 12:56:09 +00:00
|
|
|
{
|
|
|
|
pickQueryToExcludeImpl();
|
2022-05-02 22:45:13 +00:00
|
|
|
cancellation_state = QueryCancellationState::SELECTED;
|
2021-10-22 12:56:09 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-02 22:45:13 +00:00
|
|
|
void reset() noexcept
|
|
|
|
{
|
|
|
|
picked_tracker = nullptr;
|
|
|
|
cancellation_state = QueryCancellationState::NONE;
|
|
|
|
freed_memory = 0;
|
2022-06-21 10:15:33 +00:00
|
|
|
|
|
|
|
next_id = 0;
|
|
|
|
id_to_release = 0;
|
|
|
|
|
2022-05-02 22:45:13 +00:00
|
|
|
allow_release = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
void releaseThreads();
|
|
|
|
|
|
|
|
QueryCancellationState cancellation_state;
|
|
|
|
|
2022-02-21 15:45:17 +00:00
|
|
|
// Global mutex which is used in ProcessList to synchronize
|
|
|
|
// insertion and deletion of queries.
|
|
|
|
// OvercommitTracker::pickQueryToExcludeImpl() implementations
|
|
|
|
// require this mutex to be locked, because they read list (or sublist)
|
|
|
|
// of queries.
|
2022-02-16 20:02:14 +00:00
|
|
|
std::mutex & global_mutex;
|
2022-05-02 22:45:13 +00:00
|
|
|
Int64 freed_memory;
|
|
|
|
Int64 required_memory;
|
|
|
|
|
2022-06-21 10:15:33 +00:00
|
|
|
size_t next_id; // Id provided to the next thread to come in OvercommitTracker
|
|
|
|
size_t id_to_release; // We can release all threads with id smaller than this
|
|
|
|
|
2022-05-02 22:45:13 +00:00
|
|
|
bool allow_release;
|
2021-10-22 12:56:09 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
class ProcessList;
|
|
|
|
struct ProcessListForUser;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct UserOvercommitTracker : OvercommitTracker
|
|
|
|
{
|
2022-02-16 20:02:14 +00:00
|
|
|
explicit UserOvercommitTracker(DB::ProcessList * process_list, DB::ProcessListForUser * user_process_list_);
|
2021-10-22 12:56:09 +00:00
|
|
|
|
|
|
|
~UserOvercommitTracker() override = default;
|
|
|
|
|
|
|
|
protected:
|
2022-05-02 22:45:13 +00:00
|
|
|
void pickQueryToExcludeImpl() override;
|
2021-10-22 12:56:09 +00:00
|
|
|
|
|
|
|
private:
|
|
|
|
DB::ProcessListForUser * user_process_list;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct GlobalOvercommitTracker : OvercommitTracker
|
|
|
|
{
|
2022-02-16 20:02:14 +00:00
|
|
|
explicit GlobalOvercommitTracker(DB::ProcessList * process_list_);
|
2021-10-22 12:56:09 +00:00
|
|
|
|
|
|
|
~GlobalOvercommitTracker() override = default;
|
|
|
|
|
|
|
|
protected:
|
2022-05-02 22:45:13 +00:00
|
|
|
void pickQueryToExcludeImpl() override;
|
2021-10-22 12:56:09 +00:00
|
|
|
|
|
|
|
private:
|
|
|
|
DB::ProcessList * process_list;
|
|
|
|
};
|
2022-05-17 18:07:52 +00:00
|
|
|
|
2022-05-18 01:42:13 +00:00
|
|
|
// This class is used to disallow tracking during logging to avoid deadlocks.
|
2022-05-17 18:07:52 +00:00
|
|
|
struct OvercommitTrackerBlockerInThread
|
|
|
|
{
|
|
|
|
OvercommitTrackerBlockerInThread() { ++counter; }
|
|
|
|
~OvercommitTrackerBlockerInThread() { --counter; }
|
|
|
|
|
2022-05-18 01:42:13 +00:00
|
|
|
OvercommitTrackerBlockerInThread(OvercommitTrackerBlockerInThread const &) = delete;
|
|
|
|
OvercommitTrackerBlockerInThread & operator=(OvercommitTrackerBlockerInThread const &) = delete;
|
|
|
|
|
2022-05-17 18:07:52 +00:00
|
|
|
static bool isBlocked() { return counter > 0; }
|
|
|
|
|
|
|
|
private:
|
|
|
|
static thread_local size_t counter;
|
|
|
|
};
|