diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index a8900d5eebb..8e8109f8067 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/Cache/FileCacheUtils.cpp b/src/Interpreters/Cache/FileCacheUtils.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 05fe2713576..018ecb4bba6 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -86,7 +86,6 @@ FileSegment::FileSegment( break; } /// DOWNLOADED is used either on initial cache metadata load into memory on server startup - /// or on shrinkFileSegmentToDownloadedSize() -- when file segment object is updated. case (State::DOWNLOADED): { reserved_size = downloaded_size = size_; @@ -644,6 +643,49 @@ void FileSegment::completePartAndResetDownloader() LOG_TEST(log, "Complete batch. ({})", getInfoForLogUnlocked(lk)); } +void FileSegment::shrinkFileSegmentToDownloadedSize(const LockedKey & locked_key, const FileSegmentGuard::Lock & lock) +{ + if (downloaded_size == range().size()) + { + /// Nothing to resize; + return; + } + + if (!locked_key.isLastOwnerOfFileSegment(offset())) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Shrinking of file segment can be done only by the last holder: {}", + getInfoForLog()); + } + + size_t aligned_downloaded_size = FileCacheUtils::roundUpToMultiple(downloaded_size, cache->getBoundaryAlignment()); + chassert(aligned_downloaded_size >= downloaded_size); + + if (aligned_downloaded_size == range().size()) + { + /// Nothing to resize; + return; + } + else if (aligned_downloaded_size > range().size() + || downloaded_size == aligned_downloaded_size) + { + /// Does not make sense to resize upwords. + setDownloadState(State::DOWNLOADED, lock); + segment_range.right = segment_range.left + downloaded_size - 1; + } + else + { + setDownloadState(State::PARTIALLY_DOWNLOADED, lock); + segment_range.right = segment_range.left + aligned_downloaded_size - 1; + } + + const size_t diff = reserved_size - downloaded_size; + chassert(reserved_size >= downloaded_size); + if (diff) + getQueueIterator()->decrementSize(diff); +} + size_t FileSegment::getSizeForBackgroundDownload() const { auto lk = lock(); @@ -762,8 +804,7 @@ void FileSegment::complete(bool allow_background_download) if (!added_to_download_queue) { - locked_key->shrinkFileSegmentToDownloadedSize(offset(), segment_lock); - setDetachedState(segment_lock); /// See comment below. + shrinkFileSegmentToDownloadedSize(*locked_key, segment_lock); } } break; @@ -793,11 +834,7 @@ void FileSegment::complete(bool allow_background_download) /// but current file segment should remain PARRTIALLY_DOWNLOADED_NO_CONTINUATION and with detached state, /// because otherwise an invariant that getOrSet() returns a contiguous range of file segments will be broken /// (this will be crucial for other file segment holder, not for current one). - locked_key->shrinkFileSegmentToDownloadedSize(offset(), segment_lock); - - /// We mark current file segment with state DETACHED, even though the data is still in cache - /// (but a separate file segment) because is_last_holder is satisfied, so it does not matter. - setDetachedState(segment_lock); + shrinkFileSegmentToDownloadedSize(*locked_key, segment_lock); } } break; diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index a6bfb203cec..2c0e99a081f 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -242,6 +242,7 @@ private: void setDownloadedUnlocked(const FileSegmentGuard::Lock &); void setDownloadFailedUnlocked(const FileSegmentGuard::Lock &); + void shrinkFileSegmentToDownloadedSize(const LockedKey &, const FileSegmentGuard::Lock &); void assertNotDetached() const; void assertNotDetachedUnlocked(const FileSegmentGuard::Lock &) const; diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 4d3033191dc..8b15b08249d 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -1000,42 +1000,6 @@ KeyMetadata::iterator LockedKey::removeFileSegmentImpl( return key_metadata->erase(it); } -void LockedKey::shrinkFileSegmentToDownloadedSize( - size_t offset, - const FileSegmentGuard::Lock & segment_lock) -{ - /** - * In case file was partially downloaded and it's download cannot be continued - * because of no space left in cache, we need to be able to cut file segment's size to downloaded_size. - */ - - auto file_segment_metadata = getByOffset(offset); - const auto & file_segment = file_segment_metadata->file_segment; - chassert(file_segment->assertCorrectnessUnlocked(segment_lock)); - - const size_t downloaded_size = file_segment->getDownloadedSize(); - if (downloaded_size == file_segment->range().size()) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Nothing to reduce, file segment fully downloaded: {}", - file_segment->getInfoForLogUnlocked(segment_lock)); - } - - chassert(file_segment->reserved_size >= downloaded_size); - int64_t diff = file_segment->reserved_size - downloaded_size; - - file_segment_metadata->file_segment = std::make_shared( - getKey(), offset, downloaded_size, FileSegment::State::DOWNLOADED, - CreateFileSegmentSettings(file_segment->getKind()), false, - file_segment->cache, key_metadata, file_segment->queue_iterator); - - if (diff) - file_segment_metadata->getQueueIterator()->decrementSize(diff); - - chassert(file_segment_metadata->file_segment->assertCorrectnessUnlocked(segment_lock)); -} - bool LockedKey::addToDownloadQueue(size_t offset, const FileSegmentGuard::Lock &) { auto it = key_metadata->find(offset); diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index 24683b2de71..a4368d6ff26 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -326,8 +326,6 @@ struct LockedKey : private boost::noncopyable bool can_be_broken = false, bool invalidate_queue_entry = true); - void shrinkFileSegmentToDownloadedSize(size_t offset, const FileSegmentGuard::Lock &); - bool addToDownloadQueue(size_t offset, const FileSegmentGuard::Lock &); bool isLastOwnerOfFileSegment(size_t offset) const;