Merge pull request #26313 from fastio/control_execution_period_of_clearOldTemporaryDirectories

Control the execution period of clear old temporary directories by parameter

docs/en/operations/settings/settings.md

@@ -768,6 +768,26 @@ Possible value:
 
 Default value: 2013265920.
 
+## merge_tree_clear_old_temporary_directories_interval_seconds {#setting-merge-tree-clear-old-temporary-directories-interval-seconds}
+
+The interval in seconds for ClickHouse to execute the cleanup old temporary directories.
+
+Possible value:
+
+-   Any positive integer.
+
+Default value: 60.
+
+## merge_tree_clear_old_parts_interval_seconds {#setting-merge-tree-clear-old-parts-interval-seconds}
+
+The interval in seconds for ClickHouse to execute the cleanup old parts, WALs, and mutations.
+
+Possible value:
+
+-   Any positive integer.
+
+Default value: 1.
+
 ## min_bytes_to_use_direct_io {#settings-min-bytes-to-use-direct-io}
 
 The minimum data volume required for using direct I/O access to the storage disk.

src/Core/Settings.h

@@ -144,6 +144,8 @@ class IColumn;
     M(UInt64, merge_tree_coarse_index_granularity, 8, "If the index segment can contain the required keys, divide it into as many parts and recursively check them.", 0) \
     M(UInt64, merge_tree_max_rows_to_use_cache, (128 * 8192), "The maximum number of rows per request, to use the cache of uncompressed data. If the request is large, the cache is not used. (For large queries not to flush out the cache.)", 0) \
     M(UInt64, merge_tree_max_bytes_to_use_cache, (192 * 10 * 1024 * 1024), "The maximum number of bytes per request, to use the cache of uncompressed data. If the request is large, the cache is not used. (For large queries not to flush out the cache.)", 0) \
+    M(UInt64, merge_tree_clear_old_temporary_directories_interval_seconds, 60, "The period of executing the clear old temporary directories operation in background.", 0) \
+    M(UInt64, merge_tree_clear_old_parts_interval_seconds, 1, "The period of executing the clear old parts operation in background.", 0) \
     M(Bool, do_not_merge_across_partitions_select_final, false, "Merge parts only in one partition in select final", 0) \
     \
     M(UInt64, mysql_max_rows_to_insert, 65536, "The maximum number of rows in MySQL batch insertion of the MySQL storage engine", 0) \

src/Storages/StorageMergeTree.cpp

@@ -109,7 +109,8 @@ void StorageMergeTree::startup()
     clearOldTemporaryDirectories(0);
 
     /// NOTE background task will also do the above cleanups periodically.
-    time_after_previous_cleanup.restart();
+    time_after_previous_cleanup_parts.restart();
+    time_after_previous_cleanup_temporary_directories.restart();
 
     try
     {
@@ -1061,22 +1062,32 @@ bool StorageMergeTree::scheduleDataProcessingJob(IBackgroundJobExecutor & execut
         }, PoolType::MERGE_MUTATE});
         return true;
     }
-    else if (auto cmp_lock = time_after_previous_cleanup.compareAndRestartDeferred(1))
+    bool executed = false;
+    if (time_after_previous_cleanup_temporary_directories.compareAndRestartDeferred(getContext()->getSettingsRef().merge_tree_clear_old_temporary_directories_interval_seconds))
+    {
+        executor.execute({[this, share_lock] ()
+        {
+            clearOldTemporaryDirectories(getSettings()->temporary_directories_lifetime.totalSeconds());
+            return true;
+        }, PoolType::MERGE_MUTATE});
+        executed = true;
+    }
+    if (time_after_previous_cleanup_parts.compareAndRestartDeferred(getContext()->getSettingsRef().merge_tree_clear_old_parts_interval_seconds))
     {
         executor.execute({[this, share_lock] ()
         {
             /// All use relative_data_path which changes during rename
             /// so execute under share lock.
             clearOldPartsFromFilesystem();
-            clearOldTemporaryDirectories(getSettings()->temporary_directories_lifetime.totalSeconds());
             clearOldWriteAheadLogs();
             clearOldMutations();
             clearEmptyParts();
             return true;
         }, PoolType::MERGE_MUTATE});
-        return true;
-    }
-    return false;
+        executed = true;
+     }
+
+    return executed;
 }
 
 Int64 StorageMergeTree::getCurrentMutationVersion(

src/Storages/StorageMergeTree.h

@@ -114,8 +114,10 @@ private:
     /// For block numbers.
     SimpleIncrement increment;
 
-    /// For clearOldParts, clearOldTemporaryDirectories.
-    AtomicStopwatch time_after_previous_cleanup;
+    /// For clearOldParts
+    AtomicStopwatch time_after_previous_cleanup_parts;
+    /// For clearOldTemporaryDirectories.
+    AtomicStopwatch time_after_previous_cleanup_temporary_directories;
 
     /// Mutex for parts currently processing in background
     /// merging (also with TTL), mutating or moving.