thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 08:02:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16729 from azat/dist-cleanup-empty-dirs-at-start

Browse Source 
Remove empty directories for async INSERT at start of Distributed engine
This commit is contained in:
alexey-milovidov 2020-11-06 12:39:31 +03:00 committed by GitHub
commit 0da1127440
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/Storages/Distributed/DirectoryMonitor.cpp
View File

@ -79,7 +79,7 @@ namespace
StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor( StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor(
StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_, BackgroundSchedulePool & bg_pool_) StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_, BackgroundSchedulePool & bg_pool)
: storage(storage_) : storage(storage_)
, pool(std::move(pool_)) , pool(std::move(pool_))
, path{path_ + '/'} , path{path_ + '/'}
@ -92,7 +92,6 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor(
, max_sleep_time{storage.global_context->getSettingsRef().distributed_directory_monitor_max_sleep_time_ms.totalMilliseconds()} , max_sleep_time{storage.global_context->getSettingsRef().distributed_directory_monitor_max_sleep_time_ms.totalMilliseconds()}
, log{&Poco::Logger::get(getLoggerName())} , log{&Poco::Logger::get(getLoggerName())}
, monitor_blocker(monitor_blocker_) , monitor_blocker(monitor_blocker_)
, bg_pool(bg_pool_)
, metric_pending_files(CurrentMetrics::DistributedFilesToInsert, 0) , metric_pending_files(CurrentMetrics::DistributedFilesToInsert, 0)
{ {
task_handle = bg_pool.createTask(getLoggerName() + "/Bg", [this]{ run(); }); task_handle = bg_pool.createTask(getLoggerName() + "/Bg", [this]{ run(); });

3
src/Storages/Distributed/DirectoryMonitor.h
View File

@ -25,7 +25,7 @@ class StorageDistributedDirectoryMonitor
{ {
public: public:
StorageDistributedDirectoryMonitor( StorageDistributedDirectoryMonitor(
StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_, BackgroundSchedulePool & bg_pool_); StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_, BackgroundSchedulePool & bg_pool);
~StorageDistributedDirectoryMonitor(); ~StorageDistributedDirectoryMonitor();
@ -95,7 +95,6 @@ private:
Poco::Logger * log; Poco::Logger * log;
ActionBlocker & monitor_blocker; ActionBlocker & monitor_blocker;
BackgroundSchedulePool & bg_pool;
BackgroundSchedulePoolTaskHolder task_handle; BackgroundSchedulePoolTaskHolder task_handle;
CurrentMetrics::Increment metric_pending_files; CurrentMetrics::Increment metric_pending_files;

24
src/Storages/StorageDistributed.cpp
View File

@ -670,8 +670,28 @@ void StorageDistributed::createDirectoryMonitors(const std::string & disk)
std::filesystem::directory_iterator begin(path); std::filesystem::directory_iterator begin(path);
std::filesystem::directory_iterator end; std::filesystem::directory_iterator end;
for (auto it = begin; it != end; ++it) for (auto it = begin; it != end; ++it)
if (std::filesystem::is_directory(*it)) {
requireDirectoryMonitor(disk, it->path().filename().string()); const auto & dir_path = it->path();
if (std::filesystem::is_directory(dir_path))
{
const auto & tmp_path = dir_path / "tmp";
/// "tmp" created by DistributedBlockOutputStream
if (std::filesystem::is_directory(tmp_path) && std::filesystem::is_empty(tmp_path))
std::filesystem::remove(tmp_path);
if (std::filesystem::is_empty(dir_path))
{
LOG_DEBUG(log, "Removing {} (used for async INSERT into Distributed)", dir_path);
/// Will be created by DistributedBlockOutputStream on demand.
std::filesystem::remove(dir_path);
}
else
{
requireDirectoryMonitor(disk, dir_path.filename().string());
}
}
}
} }