Add a separate setting for background download max size

src/Interpreters/Cache/FileCache.cpp

@@ -103,7 +103,11 @@ FileCache::FileCache(const std::string & cache_name, const FileCacheSettings & s
     , keep_current_elements_to_max_ratio(1 - settings.keep_free_space_elements_ratio)
     , keep_up_free_space_remove_batch(settings.keep_free_space_remove_batch)
     , log(getLogger("FileCache(" + cache_name + ")"))
-    , metadata(settings.base_path, settings.background_download_queue_size_limit, settings.background_download_threads, write_cache_per_user_directory)
+    , metadata(settings.base_path,
+               settings.background_download_queue_size_limit,
+               settings.background_download_threads,
+               settings.background_download_max_file_segment_size,
+               write_cache_per_user_directory)
 {
     if (settings.cache_policy == "LRU")
     {
@@ -1600,6 +1604,17 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings,
         }
     }
 
+    if (new_settings.background_download_max_file_segment_size != actual_settings.background_download_max_file_segment_size)
+    {
+        metadata.setBackgroundDownloadMaxFileSegmentSize(new_settings.background_download_max_file_segment_size);
+
+        LOG_INFO(log, "Changed background_download_max_file_segment_size from {} to {}",
+                actual_settings.background_download_max_file_segment_size,
+                new_settings.background_download_max_file_segment_size);
+
+        actual_settings.background_download_max_file_segment_size = new_settings.background_download_max_file_segment_size;
+    }
+
     if (new_settings.max_size != actual_settings.max_size
         || new_settings.max_elements != actual_settings.max_elements)
     {

src/Interpreters/Cache/FileCache.h

@@ -161,6 +161,8 @@ public:
 
     size_t getMaxFileSegmentSize() const { return max_file_segment_size; }
 
+    size_t getBackgroundDownloadMaxFileSegmentSize() const { return metadata.getBackgroundDownloadMaxFileSegmentSize(); }
+
     bool tryReserve(
         FileSegment & file_segment,
         size_t size,

src/Interpreters/Cache/FileCacheSettings.cpp

@@ -62,6 +62,9 @@ void FileCacheSettings::loadImpl(FuncHas has, FuncGetUInt get_uint, FuncGetStrin
     if (has("background_download_queue_size_limit"))
         background_download_queue_size_limit = get_uint("background_download_queue_size_limit");
 
+    if (has("background_download_max_file_segment_size"))
+        background_download_threads = get_uint("background_download_max_file_segment_size");
+
     if (has("load_metadata_threads"))
         load_metadata_threads = get_uint("load_metadata_threads");
 

src/Interpreters/Cache/FileCacheSettings.h

@@ -43,6 +43,8 @@ struct FileCacheSettings
     double keep_free_space_elements_ratio = FILECACHE_DEFAULT_FREE_SPACE_ELEMENTS_RATIO;
     size_t keep_free_space_remove_batch = FILECACHE_DEFAULT_FREE_SPACE_REMOVE_BATCH;
 
+    size_t background_download_max_file_segment_size = FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE_WITH_BACKGROUND_DOWLOAD;
+
     void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix);
     void loadFromCollection(const NamedCollection & collection);
 

src/Interpreters/Cache/FileCache_fwd.h

@@ -6,6 +6,7 @@ namespace DB
 
 static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 32 * 1024 * 1024; /// 32Mi
 static constexpr int FILECACHE_DEFAULT_FILE_SEGMENT_ALIGNMENT = 4 * 1024 * 1024; /// 4Mi
+static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE_WITH_BACKGROUND_DOWLOAD = 4 * 1024 * 1024; /// 4Mi
 static constexpr int FILECACHE_DEFAULT_BACKGROUND_DOWNLOAD_THREADS = 5;
 static constexpr int FILECACHE_DEFAULT_BACKGROUND_DOWNLOAD_QUEUE_SIZE_LIMIT = 5000;
 static constexpr int FILECACHE_DEFAULT_LOAD_METADATA_THREADS = 16;

src/Interpreters/Cache/FileSegment.cpp

@@ -708,7 +708,10 @@ void FileSegment::complete(bool allow_background_download)
             if (is_last_holder)
             {
                 bool added_to_download_queue = false;
-                if (allow_background_download && background_download_enabled && remote_file_reader)
+                if (allow_background_download
+                    && background_download_enabled
+                    && remote_file_reader
+                    && downloaded_size < cache->getBackgroundDownloadMaxFileSegmentSize())
                 {
                     ProfileEvents::increment(ProfileEvents::FilesystemCacheBackgroundDownloadQueuePush);
                     added_to_download_queue = locked_key->addToDownloadQueue(offset(), segment_lock); /// Finish download in background.

src/Interpreters/Cache/Metadata.cpp

@@ -168,6 +168,7 @@ CacheMetadata::CacheMetadata(
     const std::string & path_,
     size_t background_download_queue_size_limit_,
     size_t background_download_threads_,
+    size_t background_download_max_file_segment_size_,
     bool write_cache_per_user_directory_)
     : path(path_)
     , cleanup_queue(std::make_shared<CleanupQueue>())
@@ -175,6 +176,7 @@ CacheMetadata::CacheMetadata(
     , write_cache_per_user_directory(write_cache_per_user_directory_)
     , log(getLogger("CacheMetadata"))
     , download_threads_num(background_download_threads_)
+    , download_max_file_segment_size(background_download_max_file_segment_size_)
 {
 }
 
@@ -630,6 +632,9 @@ void CacheMetadata::downloadThreadFunc(const bool & stop_flag)
 
                 auto & file_segment = holder->front();
 
+                if (file_segment.getDownloadedSize() >= download_max_file_segment_size)
+                    continue;
+
                 if (file_segment.getOrSetDownloader() != FileSegment::getCallerId())
                     continue;
 
@@ -701,10 +706,25 @@ void CacheMetadata::downloadImpl(FileSegment & file_segment, std::optional<Memor
     if (offset != static_cast<size_t>(reader->getPosition()))
         reader->seek(offset, SEEK_SET);
 
-    while (!reader->eof())
+    bool stop = false;
+    const size_t max_file_segment_size = download_max_file_segment_size.load();
+
+    while (!stop && !reader->eof())
     {
         auto size = reader->available();
 
+        const size_t downloaded_size = file_segment.getDownloadedSize();
+        if (downloaded_size >= max_file_segment_size)
+            break;
+
+        if (download_max_file_segment_size + size > max_file_segment_size)
+        {
+            /// Do not download more than download_max_file_segment_size
+            /// because we want to leave right boundary of file segment aligned.
+            size = max_file_segment_size - downloaded_size;
+            stop = true;
+        }
+
         std::string failure_reason;
         if (!file_segment.reserve(size, reserve_space_lock_wait_timeout_milliseconds, failure_reason))
         {

src/Interpreters/Cache/Metadata.h

@@ -165,6 +165,7 @@ public:
         const std::string & path_,
         size_t background_download_queue_size_limit_,
         size_t background_download_threads_,
+        size_t background_download_max_file_segment_size_,
         bool write_cache_per_user_directory_);
 
     void startup();
@@ -210,6 +211,10 @@ public:
 
     bool setBackgroundDownloadThreads(size_t threads_num);
     size_t getBackgroundDownloadThreads() const { return download_threads.size(); }
+
+    void setBackgroundDownloadMaxFileSegmentSize(size_t max_file_segment_size) { download_max_file_segment_size = max_file_segment_size; }
+    size_t getBackgroundDownloadMaxFileSegmentSize() const { return download_max_file_segment_size; }
+
     bool setBackgroundDownloadQueueSizeLimit(size_t size);
 
     bool isBackgroundDownloadEnabled();
@@ -241,6 +246,7 @@ private:
     };
 
     std::atomic<size_t> download_threads_num;
+    std::atomic<size_t> download_max_file_segment_size;
     std::vector<std::shared_ptr<DownloadThread>> download_threads;
     std::unique_ptr<ThreadFromGlobalPool> cleanup_thread;