Merge 25be6bb68f into 44b4bd38b9

src/Interpreters/Cache/FileCache.cpp

@@ -10,6 +10,7 @@
 #include <Interpreters/Cache/SLRUFileCachePriority.h>
 #include <Interpreters/Cache/FileCacheUtils.h>
 #include <Interpreters/Cache/EvictionCandidates.h>
+#include <Interpreters/Cache/FileCacheUtils.h>
 #include <Interpreters/Context.h>
 #include <base/hex.h>
 #include <Common/callOnce.h>

src/Interpreters/Cache/FileSegment.cpp

@@ -86,7 +86,6 @@ FileSegment::FileSegment(
             break;
         }
         /// DOWNLOADED is used either on initial cache metadata load into memory on server startup
-        /// or on shrinkFileSegmentToDownloadedSize() -- when file segment object is updated.
         case (State::DOWNLOADED):
         {
             reserved_size = downloaded_size = size_;
@@ -644,6 +643,49 @@ void FileSegment::completePartAndResetDownloader()
     LOG_TEST(log, "Complete batch. ({})", getInfoForLogUnlocked(lk));
 }
 
+void FileSegment::shrinkFileSegmentToDownloadedSize(const LockedKey & locked_key, const FileSegmentGuard::Lock & lock)
+{
+    if (downloaded_size == range().size())
+    {
+        /// Nothing to resize;
+        return;
+    }
+
+    if (!locked_key.isLastOwnerOfFileSegment(offset()))
+    {
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "Shrinking of file segment can  be done only by the last holder: {}",
+            getInfoForLog());
+    }
+
+    size_t aligned_downloaded_size = FileCacheUtils::roundUpToMultiple(downloaded_size, cache->getBoundaryAlignment());
+    chassert(aligned_downloaded_size >= downloaded_size);
+
+    if (aligned_downloaded_size == range().size())
+    {
+        /// Nothing to resize;
+        return;
+    }
+    else if (aligned_downloaded_size > range().size()
+             || downloaded_size == aligned_downloaded_size)
+    {
+        /// Does not make sense to resize upwords.
+        setDownloadState(State::DOWNLOADED, lock);
+        segment_range.right = segment_range.left + downloaded_size - 1;
+    }
+    else
+    {
+        setDownloadState(State::PARTIALLY_DOWNLOADED, lock);
+        segment_range.right = segment_range.left + aligned_downloaded_size - 1;
+    }
+
+    const size_t diff = reserved_size - downloaded_size;
+    chassert(reserved_size >= downloaded_size);
+    if (diff)
+        getQueueIterator()->decrementSize(diff);
+}
+
 size_t FileSegment::getSizeForBackgroundDownload() const
 {
     auto lk = lock();
@@ -762,8 +804,7 @@ void FileSegment::complete(bool allow_background_download)
 
                 if (!added_to_download_queue)
                 {
-                    locked_key->shrinkFileSegmentToDownloadedSize(offset(), segment_lock);
-                    setDetachedState(segment_lock); /// See comment below.
+                    shrinkFileSegmentToDownloadedSize(*locked_key, segment_lock);
                 }
             }
             break;
@@ -793,11 +834,7 @@ void FileSegment::complete(bool allow_background_download)
                     /// but current file segment should remain PARRTIALLY_DOWNLOADED_NO_CONTINUATION and with detached state,
                     /// because otherwise an invariant that getOrSet() returns a contiguous range of file segments will be broken
                     /// (this will be crucial for other file segment holder, not for current one).
-                    locked_key->shrinkFileSegmentToDownloadedSize(offset(), segment_lock);
-
-                    /// We mark current file segment with state DETACHED, even though the data is still in cache
-                    /// (but a separate file segment) because is_last_holder is satisfied, so it does not matter.
-                    setDetachedState(segment_lock);
+                    shrinkFileSegmentToDownloadedSize(*locked_key, segment_lock);
                 }
             }
             break;

src/Interpreters/Cache/FileSegment.h

@@ -242,6 +242,7 @@ private:
 
     void setDownloadedUnlocked(const FileSegmentGuard::Lock &);
     void setDownloadFailedUnlocked(const FileSegmentGuard::Lock &);
+    void shrinkFileSegmentToDownloadedSize(const LockedKey &, const FileSegmentGuard::Lock &);
 
     void assertNotDetached() const;
     void assertNotDetachedUnlocked(const FileSegmentGuard::Lock &) const;

src/Interpreters/Cache/Metadata.cpp

@@ -1000,42 +1000,6 @@ KeyMetadata::iterator LockedKey::removeFileSegmentImpl(
     return key_metadata->erase(it);
 }
 
-void LockedKey::shrinkFileSegmentToDownloadedSize(
-    size_t offset,
-    const FileSegmentGuard::Lock & segment_lock)
-{
-    /**
-     * In case file was partially downloaded and it's download cannot be continued
-     * because of no space left in cache, we need to be able to cut file segment's size to downloaded_size.
-     */
-
-    auto file_segment_metadata = getByOffset(offset);
-    const auto & file_segment = file_segment_metadata->file_segment;
-    chassert(file_segment->assertCorrectnessUnlocked(segment_lock));
-
-    const size_t downloaded_size = file_segment->getDownloadedSize();
-    if (downloaded_size == file_segment->range().size())
-    {
-        throw Exception(
-            ErrorCodes::LOGICAL_ERROR,
-            "Nothing to reduce, file segment fully downloaded: {}",
-            file_segment->getInfoForLogUnlocked(segment_lock));
-    }
-
-    chassert(file_segment->reserved_size >= downloaded_size);
-    int64_t diff = file_segment->reserved_size - downloaded_size;
-
-    file_segment_metadata->file_segment = std::make_shared<FileSegment>(
-        getKey(), offset, downloaded_size, FileSegment::State::DOWNLOADED,
-        CreateFileSegmentSettings(file_segment->getKind()), false,
-        file_segment->cache, key_metadata, file_segment->queue_iterator);
-
-    if (diff)
-        file_segment_metadata->getQueueIterator()->decrementSize(diff);
-
-    chassert(file_segment_metadata->file_segment->assertCorrectnessUnlocked(segment_lock));
-}
-
 bool LockedKey::addToDownloadQueue(size_t offset, const FileSegmentGuard::Lock &)
 {
     auto it = key_metadata->find(offset);

src/Interpreters/Cache/Metadata.h

@@ -326,8 +326,6 @@ struct LockedKey : private boost::noncopyable
         bool can_be_broken = false,
         bool invalidate_queue_entry = true);
 
-    void shrinkFileSegmentToDownloadedSize(size_t offset, const FileSegmentGuard::Lock &);
-
     bool addToDownloadQueue(size_t offset, const FileSegmentGuard::Lock &);
 
     bool isLastOwnerOfFileSegment(size_t offset) const;