mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-24 02:30:51 +00:00
Use thread pool (background_distributed_schedule_pool_size) for distributed sends
After #8756 the problem with 1 thread for each (distributed table, disk) for distributed sends became even worse (since there can be multiple disks), so use predefined thread pool for this tasks, that can be controlled with background_distributed_schedule_pool_size knob.
This commit is contained in:
parent
201d5d5b25
commit
5d11118cc9
@ -82,6 +82,7 @@ struct Settings : public SettingsCollection<Settings>
|
|||||||
M(SettingUInt64, background_pool_size, 16, "Number of threads performing background work for tables (for example, merging in merge tree). Only has meaning at server startup.", 0) \
|
M(SettingUInt64, background_pool_size, 16, "Number of threads performing background work for tables (for example, merging in merge tree). Only has meaning at server startup.", 0) \
|
||||||
M(SettingUInt64, background_move_pool_size, 8, "Number of threads performing background moves for tables. Only has meaning at server startup.", 0) \
|
M(SettingUInt64, background_move_pool_size, 8, "Number of threads performing background moves for tables. Only has meaning at server startup.", 0) \
|
||||||
M(SettingUInt64, background_schedule_pool_size, 16, "Number of threads performing background tasks for replicated tables, kafka streaming, dns cache updates. Only has meaning at server startup.", 0) \
|
M(SettingUInt64, background_schedule_pool_size, 16, "Number of threads performing background tasks for replicated tables, kafka streaming, dns cache updates. Only has meaning at server startup.", 0) \
|
||||||
|
M(SettingUInt64, background_distributed_schedule_pool_size, 16, "Number of threads performing background tasks for distributed sends. Only has meaning at server startup.", 0) \
|
||||||
\
|
\
|
||||||
M(SettingMilliseconds, distributed_directory_monitor_sleep_time_ms, 100, "Sleep time for StorageDistributed DirectoryMonitors, in case of any errors delay grows exponentially.", 0) \
|
M(SettingMilliseconds, distributed_directory_monitor_sleep_time_ms, 100, "Sleep time for StorageDistributed DirectoryMonitors, in case of any errors delay grows exponentially.", 0) \
|
||||||
M(SettingMilliseconds, distributed_directory_monitor_max_sleep_time_ms, 30000, "Maximum sleep time for StorageDistributed DirectoryMonitors, it limits exponential growth too.", 0) \
|
M(SettingMilliseconds, distributed_directory_monitor_max_sleep_time_ms, 30000, "Maximum sleep time for StorageDistributed DirectoryMonitors, it limits exponential growth too.", 0) \
|
||||||
|
@ -321,6 +321,7 @@ struct ContextShared
|
|||||||
std::optional<BackgroundProcessingPool> background_pool; /// The thread pool for the background work performed by the tables.
|
std::optional<BackgroundProcessingPool> background_pool; /// The thread pool for the background work performed by the tables.
|
||||||
std::optional<BackgroundProcessingPool> background_move_pool; /// The thread pool for the background moves performed by the tables.
|
std::optional<BackgroundProcessingPool> background_move_pool; /// The thread pool for the background moves performed by the tables.
|
||||||
std::optional<BackgroundSchedulePool> schedule_pool; /// A thread pool that can run different jobs in background (used in replicated tables)
|
std::optional<BackgroundSchedulePool> schedule_pool; /// A thread pool that can run different jobs in background (used in replicated tables)
|
||||||
|
std::optional<BackgroundSchedulePool> distributed_schedule_pool; /// A thread pool that can run different jobs in background (used for distributed sends)
|
||||||
MultiVersion<Macros> macros; /// Substitutions extracted from config.
|
MultiVersion<Macros> macros; /// Substitutions extracted from config.
|
||||||
std::unique_ptr<DDLWorker> ddl_worker; /// Process ddl commands from zk.
|
std::unique_ptr<DDLWorker> ddl_worker; /// Process ddl commands from zk.
|
||||||
/// Rules for selecting the compression settings, depending on the size of the part.
|
/// Rules for selecting the compression settings, depending on the size of the part.
|
||||||
@ -418,6 +419,7 @@ struct ContextShared
|
|||||||
background_pool.reset();
|
background_pool.reset();
|
||||||
background_move_pool.reset();
|
background_move_pool.reset();
|
||||||
schedule_pool.reset();
|
schedule_pool.reset();
|
||||||
|
distributed_schedule_pool.reset();
|
||||||
ddl_worker.reset();
|
ddl_worker.reset();
|
||||||
|
|
||||||
/// Stop trace collector if any
|
/// Stop trace collector if any
|
||||||
@ -1348,6 +1350,14 @@ BackgroundSchedulePool & Context::getSchedulePool()
|
|||||||
return *shared->schedule_pool;
|
return *shared->schedule_pool;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
BackgroundSchedulePool & Context::getDistributedSchedulePool()
|
||||||
|
{
|
||||||
|
auto lock = getLock();
|
||||||
|
if (!shared->distributed_schedule_pool)
|
||||||
|
shared->distributed_schedule_pool.emplace(settings.background_distributed_schedule_pool_size);
|
||||||
|
return *shared->distributed_schedule_pool;
|
||||||
|
}
|
||||||
|
|
||||||
void Context::setDDLWorker(std::unique_ptr<DDLWorker> ddl_worker)
|
void Context::setDDLWorker(std::unique_ptr<DDLWorker> ddl_worker)
|
||||||
{
|
{
|
||||||
auto lock = getLock();
|
auto lock = getLock();
|
||||||
|
@ -475,6 +475,7 @@ public:
|
|||||||
BackgroundProcessingPool & getBackgroundPool();
|
BackgroundProcessingPool & getBackgroundPool();
|
||||||
BackgroundProcessingPool & getBackgroundMovePool();
|
BackgroundProcessingPool & getBackgroundMovePool();
|
||||||
BackgroundSchedulePool & getSchedulePool();
|
BackgroundSchedulePool & getSchedulePool();
|
||||||
|
BackgroundSchedulePool & getDistributedSchedulePool();
|
||||||
|
|
||||||
void setDDLWorker(std::unique_ptr<DDLWorker> ddl_worker);
|
void setDDLWorker(std::unique_ptr<DDLWorker> ddl_worker);
|
||||||
DDLWorker & getDDLWorker() const;
|
DDLWorker & getDDLWorker() const;
|
||||||
|
@ -1,7 +1,6 @@
|
|||||||
#include <DataStreams/RemoteBlockOutputStream.h>
|
#include <DataStreams/RemoteBlockOutputStream.h>
|
||||||
#include <DataStreams/NativeBlockInputStream.h>
|
#include <DataStreams/NativeBlockInputStream.h>
|
||||||
#include <Common/escapeForFileName.h>
|
#include <Common/escapeForFileName.h>
|
||||||
#include <Common/setThreadName.h>
|
|
||||||
#include <Common/CurrentMetrics.h>
|
#include <Common/CurrentMetrics.h>
|
||||||
#include <Common/StringUtils/StringUtils.h>
|
#include <Common/StringUtils/StringUtils.h>
|
||||||
#include <Common/ClickHouseRevision.h>
|
#include <Common/ClickHouseRevision.h>
|
||||||
@ -78,7 +77,7 @@ namespace
|
|||||||
|
|
||||||
|
|
||||||
StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor(
|
StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor(
|
||||||
StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_)
|
StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_, BackgroundSchedulePool & bg_pool_)
|
||||||
/// It's important to initialize members before `thread` to avoid race.
|
/// It's important to initialize members before `thread` to avoid race.
|
||||||
: storage(storage_)
|
: storage(storage_)
|
||||||
, pool(std::move(pool_))
|
, pool(std::move(pool_))
|
||||||
@ -92,7 +91,10 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor(
|
|||||||
, max_sleep_time{storage.global_context.getSettingsRef().distributed_directory_monitor_max_sleep_time_ms.totalMilliseconds()}
|
, max_sleep_time{storage.global_context.getSettingsRef().distributed_directory_monitor_max_sleep_time_ms.totalMilliseconds()}
|
||||||
, log{&Logger::get(getLoggerName())}
|
, log{&Logger::get(getLoggerName())}
|
||||||
, monitor_blocker(monitor_blocker_)
|
, monitor_blocker(monitor_blocker_)
|
||||||
|
, bg_pool(bg_pool_)
|
||||||
{
|
{
|
||||||
|
task_handle = bg_pool.createTask(getLoggerName() + "/Bg", [this]{ run(); });
|
||||||
|
task_handle->activateAndSchedule();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -102,7 +104,7 @@ StorageDistributedDirectoryMonitor::~StorageDistributedDirectoryMonitor()
|
|||||||
{
|
{
|
||||||
quit = true;
|
quit = true;
|
||||||
cond.notify_one();
|
cond.notify_one();
|
||||||
thread.join();
|
task_handle->deactivate();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -121,7 +123,7 @@ void StorageDistributedDirectoryMonitor::shutdownAndDropAllData()
|
|||||||
{
|
{
|
||||||
quit = true;
|
quit = true;
|
||||||
cond.notify_one();
|
cond.notify_one();
|
||||||
thread.join();
|
task_handle->deactivate();
|
||||||
}
|
}
|
||||||
|
|
||||||
Poco::File(path).remove(true);
|
Poco::File(path).remove(true);
|
||||||
@ -130,16 +132,11 @@ void StorageDistributedDirectoryMonitor::shutdownAndDropAllData()
|
|||||||
|
|
||||||
void StorageDistributedDirectoryMonitor::run()
|
void StorageDistributedDirectoryMonitor::run()
|
||||||
{
|
{
|
||||||
setThreadName("DistrDirMonitor");
|
|
||||||
|
|
||||||
std::unique_lock lock{mutex};
|
std::unique_lock lock{mutex};
|
||||||
|
|
||||||
const auto quit_requested = [this] { return quit.load(std::memory_order_relaxed); };
|
while (!quit)
|
||||||
|
|
||||||
while (!quit_requested())
|
|
||||||
{
|
{
|
||||||
auto do_sleep = true;
|
bool do_sleep = true;
|
||||||
|
|
||||||
if (!monitor_blocker.isCancelled())
|
if (!monitor_blocker.isCancelled())
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
@ -161,15 +158,25 @@ void StorageDistributedDirectoryMonitor::run()
|
|||||||
LOG_DEBUG(log, "Skipping send data over distributed table.");
|
LOG_DEBUG(log, "Skipping send data over distributed table.");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (do_sleep)
|
|
||||||
cond.wait_for(lock, sleep_time, quit_requested);
|
|
||||||
|
|
||||||
const auto now = std::chrono::system_clock::now();
|
const auto now = std::chrono::system_clock::now();
|
||||||
if (now - last_decrease_time > decrease_error_count_period)
|
if (now - last_decrease_time > decrease_error_count_period)
|
||||||
{
|
{
|
||||||
error_count /= 2;
|
error_count /= 2;
|
||||||
last_decrease_time = now;
|
last_decrease_time = now;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (do_sleep)
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!quit)
|
||||||
|
{
|
||||||
|
/// If there is no error, then it will be scheduled by the DistributedBlockOutputStream,
|
||||||
|
/// so this is just in case, hence it is distributed_directory_monitor_max_sleep_time_ms
|
||||||
|
if (error_count)
|
||||||
|
task_handle->scheduleAfter(sleep_time.count());
|
||||||
|
else
|
||||||
|
task_handle->scheduleAfter(max_sleep_time.count());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -580,6 +587,13 @@ BlockInputStreamPtr StorageDistributedDirectoryMonitor::createStreamFromFile(con
|
|||||||
return std::make_shared<DirectoryMonitorBlockInputStream>(file_name);
|
return std::make_shared<DirectoryMonitorBlockInputStream>(file_name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool StorageDistributedDirectoryMonitor::scheduleAfter(size_t ms)
|
||||||
|
{
|
||||||
|
if (quit)
|
||||||
|
return false;
|
||||||
|
return task_handle->scheduleAfter(ms);
|
||||||
|
}
|
||||||
|
|
||||||
void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map<UInt64, std::string> & files)
|
void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map<UInt64, std::string> & files)
|
||||||
{
|
{
|
||||||
std::unordered_set<UInt64> file_indices_to_skip;
|
std::unordered_set<UInt64> file_indices_to_skip;
|
||||||
@ -708,8 +722,13 @@ std::string StorageDistributedDirectoryMonitor::getLoggerName() const
|
|||||||
void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_path)
|
void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_path)
|
||||||
{
|
{
|
||||||
std::lock_guard lock{mutex};
|
std::lock_guard lock{mutex};
|
||||||
|
|
||||||
|
task_handle->deactivate();
|
||||||
|
|
||||||
path = new_path;
|
path = new_path;
|
||||||
current_batch_file_path = path + "current_batch.txt";
|
current_batch_file_path = path + "current_batch.txt";
|
||||||
|
|
||||||
|
task_handle->activateAndSchedule();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -1,10 +1,9 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <Storages/StorageDistributed.h>
|
#include <Storages/StorageDistributed.h>
|
||||||
#include <Common/ThreadPool.h>
|
#include <Core/BackgroundSchedulePool.h>
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
#include <thread>
|
|
||||||
#include <mutex>
|
#include <mutex>
|
||||||
#include <condition_variable>
|
#include <condition_variable>
|
||||||
#include <IO/ReadBufferFromFile.h>
|
#include <IO/ReadBufferFromFile.h>
|
||||||
@ -20,7 +19,7 @@ class StorageDistributedDirectoryMonitor
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
StorageDistributedDirectoryMonitor(
|
StorageDistributedDirectoryMonitor(
|
||||||
StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_);
|
StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_, BackgroundSchedulePool & bg_pool_);
|
||||||
|
|
||||||
~StorageDistributedDirectoryMonitor();
|
~StorageDistributedDirectoryMonitor();
|
||||||
|
|
||||||
@ -33,6 +32,9 @@ public:
|
|||||||
void shutdownAndDropAllData();
|
void shutdownAndDropAllData();
|
||||||
|
|
||||||
static BlockInputStreamPtr createStreamFromFile(const String & file_name);
|
static BlockInputStreamPtr createStreamFromFile(const String & file_name);
|
||||||
|
|
||||||
|
/// For scheduling via DistributedBlockOutputStream
|
||||||
|
bool scheduleAfter(size_t ms);
|
||||||
private:
|
private:
|
||||||
void run();
|
void run();
|
||||||
bool processFiles();
|
bool processFiles();
|
||||||
@ -67,7 +69,9 @@ private:
|
|||||||
std::condition_variable cond;
|
std::condition_variable cond;
|
||||||
Logger * log;
|
Logger * log;
|
||||||
ActionBlocker & monitor_blocker;
|
ActionBlocker & monitor_blocker;
|
||||||
ThreadFromGlobalPool thread{&StorageDistributedDirectoryMonitor::run, this};
|
|
||||||
|
BackgroundSchedulePool & bg_pool;
|
||||||
|
BackgroundSchedulePoolTaskHolder task_handle;
|
||||||
|
|
||||||
/// Read insert query and insert settings for backward compatible.
|
/// Read insert query and insert settings for backward compatible.
|
||||||
static void readHeader(ReadBuffer & in, Settings & insert_settings, std::string & insert_query, ClientInfo & client_info, Logger * log);
|
static void readHeader(ReadBuffer & in, Settings & insert_settings, std::string & insert_query, ClientInfo & client_info, Logger * log);
|
||||||
|
@ -589,8 +589,8 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std::
|
|||||||
const std::string path(disk + data_path + dir_name + '/');
|
const std::string path(disk + data_path + dir_name + '/');
|
||||||
|
|
||||||
/// ensure shard subdirectory creation and notify storage
|
/// ensure shard subdirectory creation and notify storage
|
||||||
if (Poco::File(path).createDirectory())
|
Poco::File(path).createDirectory();
|
||||||
storage.requireDirectoryMonitor(disk, dir_name);
|
auto & directory_monitor = storage.requireDirectoryMonitor(disk, dir_name);
|
||||||
|
|
||||||
const auto & file_name = toString(storage.file_names_increment.get()) + ".bin";
|
const auto & file_name = toString(storage.file_names_increment.get()) + ".bin";
|
||||||
const auto & block_file_path = path + file_name;
|
const auto & block_file_path = path + file_name;
|
||||||
@ -632,6 +632,9 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std::
|
|||||||
stream.writePrefix();
|
stream.writePrefix();
|
||||||
stream.write(block);
|
stream.write(block);
|
||||||
stream.writeSuffix();
|
stream.writeSuffix();
|
||||||
|
|
||||||
|
auto sleep_ms = context.getSettingsRef().distributed_directory_monitor_sleep_time_ms;
|
||||||
|
directory_monitor.scheduleAfter(sleep_ms.totalMilliseconds());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (link(first_file_tmp_path.data(), block_file_path.data()))
|
if (link(first_file_tmp_path.data(), block_file_path.data()))
|
||||||
|
@ -577,15 +577,20 @@ void StorageDistributed::createDirectoryMonitors(const std::string & disk)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void StorageDistributed::requireDirectoryMonitor(const std::string & disk, const std::string & name)
|
StorageDistributedDirectoryMonitor& StorageDistributed::requireDirectoryMonitor(const std::string & disk, const std::string & name)
|
||||||
{
|
{
|
||||||
const std::string path(disk + relative_data_path + name);
|
const std::string path(disk + relative_data_path + name);
|
||||||
const std::string key(disk + name);
|
const std::string key(disk + name);
|
||||||
|
|
||||||
std::lock_guard lock(cluster_nodes_mutex);
|
std::lock_guard lock(cluster_nodes_mutex);
|
||||||
auto & node_data = cluster_nodes_data[key];
|
auto & node_data = cluster_nodes_data[key];
|
||||||
|
if (!node_data.directory_monitor)
|
||||||
|
{
|
||||||
node_data.conneciton_pool = StorageDistributedDirectoryMonitor::createPool(name, *this);
|
node_data.conneciton_pool = StorageDistributedDirectoryMonitor::createPool(name, *this);
|
||||||
node_data.directory_monitor = std::make_unique<StorageDistributedDirectoryMonitor>(*this, path, node_data.conneciton_pool, monitors_blocker);
|
node_data.directory_monitor = std::make_unique<StorageDistributedDirectoryMonitor>(
|
||||||
|
*this, path, node_data.conneciton_pool, monitors_blocker, global_context.getDistributedSchedulePool());
|
||||||
|
}
|
||||||
|
return *node_data.directory_monitor;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t StorageDistributed::getShardCount() const
|
size_t StorageDistributed::getShardCount() const
|
||||||
|
@ -109,7 +109,7 @@ public:
|
|||||||
/// create directory monitors for each existing subdirectory
|
/// create directory monitors for each existing subdirectory
|
||||||
void createDirectoryMonitors(const std::string & disk);
|
void createDirectoryMonitors(const std::string & disk);
|
||||||
/// ensure directory monitor thread and connectoin pool creation by disk and subdirectory name
|
/// ensure directory monitor thread and connectoin pool creation by disk and subdirectory name
|
||||||
void requireDirectoryMonitor(const std::string & disk, const std::string & name);
|
StorageDistributedDirectoryMonitor & requireDirectoryMonitor(const std::string & disk, const std::string & name);
|
||||||
|
|
||||||
void flushClusterNodesAllData();
|
void flushClusterNodesAllData();
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user