thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

impl max_cache_download_limit

Browse Source
This commit is contained in:
KinderRiven 2022-10-18 05:05:32 +00:00
parent 26e8d0a93d
commit f94f491679
5 changed files with 76 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
src/Interpreters/Cache/FileCache.cpp
View File

@ -32,6 +32,8 @@ FileCache::FileCache(
, allow_persistent_files(cache_settings_.do_not_evict_index_and_mark_files)
, enable_cache_hits_threshold(cache_settings_.enable_cache_hits_threshold)
, enable_filesystem_query_cache_limit(cache_settings_.enable_filesystem_query_cache_limit)
, enable_limit_download_cache_size(cache_settings_.enable_limit_download_cache_size)
, max_enable_download_cache_size(cache_settings_.max_enable_download_cache_size)
, log(&Poco::Logger::get("FileCache"))
, main_priority(std::make_unique<LRUFileCachePriority>())
, stash_priority(std::make_unique<LRUFileCachePriority>())
@ -185,70 +187,83 @@ FileSegments FileCache::getImpl(
/// Given range = [left, right] and non-overlapping ordered set of file segments,
/// find list [segment1, ..., segmentN] of segments which intersect with given range.
auto it = files.find(key);
if (it == files.end())
return {};
const auto & file_segments = it->second;
if (file_segments.empty())
{
files.erase(key);
removeKeyDirectoryIfExists(key, cache_lock);
return {};
}
FileSegments result;
auto segment_it = file_segments.lower_bound(range.left);
if (segment_it == file_segments.end())
{
/// N - last cached segment for given file key, segment{N}.offset < range.left:
/// segment{N} segment{N}
/// [________ [_______]
/// [__________] OR [________]
/// ^ ^
/// range.left range.left
const auto & cell = file_segments.rbegin()->second;
if (cell.file_segment->range().right < range.left)
if (enable_limit_download_cache_size && (range.size() > max_enable_download_cache_size))
{
auto file_segment = std::make_shared<FileSegment>(
range.left, range.size(), key, this, FileSegment::State::SKIP_CACHE, CreateFileSegmentSettings{});
{
std::unique_lock segment_lock(file_segment->mutex);
file_segment->detachAssumeStateFinalized(segment_lock);
}
result.emplace_back(file_segment);
}
else
{
auto it = files.find(key);
if (it == files.end())
return {};
useCell(cell, result, cache_lock);
}
else /// segment_it <-- segmment{k}
{
if (segment_it != file_segments.begin())
const auto & file_segments = it->second;
if (file_segments.empty())
{
const auto & prev_cell = std::prev(segment_it)->second;
const auto & prev_cell_range = prev_cell.file_segment->range();
if (range.left <= prev_cell_range.right)
{
/// segment{k-1} segment{k}
/// [________] [_____
/// [___________
/// ^
/// range.left
useCell(prev_cell, result, cache_lock);
}
files.erase(key);
removeKeyDirectoryIfExists(key, cache_lock);
return {};
}
/// segment{k} ... segment{k-1} segment{k} segment{k}
/// [______ [______] [____ [________
/// [_________ OR [________ OR [______] ^
/// ^ ^ ^ segment{k}.offset
/// range.left range.left range.right
while (segment_it != file_segments.end())
auto segment_it = file_segments.lower_bound(range.left);
if (segment_it == file_segments.end())
{
const auto & cell = segment_it->second;
if (range.right < cell.file_segment->range().left)
break;
/// N - last cached segment for given file key, segment{N}.offset < range.left:
/// segment{N} segment{N}
/// [________ [_______]
/// [__________] OR [________]
/// ^ ^
/// range.left range.left
const auto & cell = file_segments.rbegin()->second;
if (cell.file_segment->range().right < range.left)
return {};
useCell(cell, result, cache_lock);
++segment_it;
}
else /// segment_it <-- segmment{k}
{
if (segment_it != file_segments.begin())
{
const auto & prev_cell = std::prev(segment_it)->second;
const auto & prev_cell_range = prev_cell.file_segment->range();
if (range.left <= prev_cell_range.right)
{
/// segment{k-1} segment{k}
/// [________] [_____
/// [___________
/// ^
/// range.left
useCell(prev_cell, result, cache_lock);
}
}
/// segment{k} ... segment{k-1} segment{k} segment{k}
/// [______ [______] [____ [________
/// [_________ OR [________ OR [______] ^
/// ^ ^ ^ segment{k}.offset
/// range.left range.left range.right
while (segment_it != file_segments.end())
{
const auto & cell = segment_it->second;
if (range.right < cell.file_segment->range().left)
break;
useCell(cell, result, cache_lock);
++segment_it;
}
}
}
return result;
}
@ -392,7 +407,6 @@ FileSegmentsHolder FileCache::getOrSet(const Key & key, size_t offset, size_t si
#endif
FileSegment::Range range(offset, offset + size - 1);
/// Get all segments which intersect with the given range.
auto file_segments = getImpl(key, range, cache_lock);
@ -404,7 +418,6 @@ FileSegmentsHolder FileCache::getOrSet(const Key & key, size_t offset, size_t si
{
fillHolesWithEmptyFileSegments(file_segments, key, range, /* fill_with_detached */false, settings, cache_lock);
}
assert(!file_segments.empty());
return FileSegmentsHolder(std::move(file_segments));
}

3
src/Interpreters/Cache/FileCache.h
View File

@ -140,6 +140,9 @@ private:
const size_t enable_cache_hits_threshold;
const bool enable_filesystem_query_cache_limit;
const bool enable_limit_download_cache_size;
const size_t max_enable_download_cache_size;
mutable std::mutex mutex;
Poco::Logger * log;

2
src/Interpreters/Cache/FileCacheSettings.cpp
View File

@ -34,6 +34,8 @@ void FileCacheSettings::loadFromConfig(const Poco::Util::AbstractConfiguration &
cache_on_write_operations = config.getUInt64(config_prefix + ".cache_on_write_operations", false);
enable_filesystem_query_cache_limit = config.getUInt64(config_prefix + ".enable_filesystem_query_cache_limit", false);
enable_cache_hits_threshold = config.getUInt64(config_prefix + ".enable_cache_hits_threshold", REMOTE_FS_OBJECTS_CACHE_ENABLE_HITS_THRESHOLD);
enable_limit_download_cache_size = config.getUInt64(config_prefix + ".enable_limit_download_cache_size", false);
max_enable_download_cache_size = config.getUInt64(config_prefix + ".max_enable_download_cache_size", REMOTE_FS_OBJECTS_CACHE_MAX_ENABLE_DOWNLOAD_SIZE);
do_not_evict_index_and_mark_files = config.getUInt64(config_prefix + ".do_not_evict_index_and_mark_files", false);
}

3
src/Interpreters/Cache/FileCacheSettings.h
View File

@ -20,6 +20,9 @@ struct FileCacheSettings
bool do_not_evict_index_and_mark_files = true;
bool enable_limit_download_cache_size = false;
size_t max_enable_download_cache_size = REMOTE_FS_OBJECTS_CACHE_MAX_ENABLE_DOWNLOAD_SIZE;
void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix);
};

1
src/Interpreters/Cache/FileCache_fwd.h
View File

@ -7,6 +7,7 @@ namespace DB
static constexpr int REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 100 * 1024 * 1024;
static constexpr int REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_ELEMENTS = 1024 * 1024;
static constexpr int REMOTE_FS_OBJECTS_CACHE_ENABLE_HITS_THRESHOLD = 0;
static constexpr size_t REMOTE_FS_OBJECTS_CACHE_MAX_ENABLE_DOWNLOAD_SIZE = 256 * 1024 * 1024;;
class FileCache;
using FileCachePtr = std::shared_ptr<FileCache>;