From c28c1ac4a2932fd652280b8bbe087a33e4d7ff2c Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 9 Nov 2023 18:22:24 +0100 Subject: [PATCH] Allow getting cache configuration from named collection --- .../Cached/registerDiskCache.cpp | 7 ++- src/Interpreters/Cache/FileCacheSettings.cpp | 43 ++++++++++++++++--- src/Interpreters/Cache/FileCacheSettings.h | 2 + tests/config/config.d/named_collection.xml | 4 ++ ...8_filesystem_cache_as_collection.reference | 2 + .../02908_filesystem_cache_as_collection.sql | 8 ++++ 6 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/02908_filesystem_cache_as_collection.reference create mode 100644 tests/queries/0_stateless/02908_filesystem_cache_as_collection.sql diff --git a/src/Disks/ObjectStorages/Cached/registerDiskCache.cpp b/src/Disks/ObjectStorages/Cached/registerDiskCache.cpp index 2f80b4c9efd..182326bbdc3 100644 --- a/src/Disks/ObjectStorages/Cached/registerDiskCache.cpp +++ b/src/Disks/ObjectStorages/Cached/registerDiskCache.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -39,7 +40,11 @@ void registerDiskCache(DiskFactory & factory, bool /* global_skip_access_check * } FileCacheSettings file_cache_settings; - file_cache_settings.loadFromConfig(config, config_prefix); + auto predefined_configuration = config.has("cache_name") ? NamedCollectionFactory::instance().tryGet(config.getString("cache_name")) : nullptr; + if (predefined_configuration) + file_cache_settings.loadFromCollection(*predefined_configuration); + else + file_cache_settings.loadFromConfig(config, config_prefix); auto config_fs_caches_dir = context->getFilesystemCachesPath(); if (config_fs_caches_dir.empty()) diff --git a/src/Interpreters/Cache/FileCacheSettings.cpp b/src/Interpreters/Cache/FileCacheSettings.cpp index 6f2f8c4b778..e81bd9ddc35 100644 --- a/src/Interpreters/Cache/FileCacheSettings.cpp +++ b/src/Interpreters/Cache/FileCacheSettings.cpp @@ -2,6 +2,7 @@ #include #include +#include #include namespace DB @@ -26,10 +27,6 @@ void FileCacheSettings::loadFromConfig(const Poco::Util::AbstractConfiguration & if (max_size == 0) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected non-zero size for cache configuration"); - auto path = config.getString(config_prefix + ".path", ""); - if (path.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Disk Cache requires non-empty `path` field (cache base path) in config"); - max_elements = config.getUInt64(config_prefix + ".max_elements", FILECACHE_DEFAULT_MAX_ELEMENTS); if (config.has(config_prefix + ".max_file_segment_size")) @@ -38,7 +35,6 @@ void FileCacheSettings::loadFromConfig(const Poco::Util::AbstractConfiguration & cache_on_write_operations = config.getUInt64(config_prefix + ".cache_on_write_operations", false); enable_filesystem_query_cache_limit = config.getUInt64(config_prefix + ".enable_filesystem_query_cache_limit", false); cache_hits_threshold = config.getUInt64(config_prefix + ".cache_hits_threshold", FILECACHE_DEFAULT_HITS_THRESHOLD); - enable_bypass_cache_with_threshold = config.getUInt64(config_prefix + ".enable_bypass_cache_with_threshold", false); if (config.has(config_prefix + ".bypass_cache_threshold")) @@ -54,4 +50,41 @@ void FileCacheSettings::loadFromConfig(const Poco::Util::AbstractConfiguration & load_metadata_threads = config.getUInt(config_prefix + ".load_metadata_threads"); } +void FileCacheSettings::loadFromCollection(const NamedCollection & collection) +{ + if (!collection.has("path")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected cache path (`path`) in configuration"); + + base_path = collection.get("path"); + + if (!collection.has("max_size")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected cache size (`max_size`) in configuration"); + + max_size = parseWithSizeSuffix(collection.get("max_size")); + if (max_size == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected non-zero size for cache configuration"); + + max_elements = collection.getOrDefault("max_elements", FILECACHE_DEFAULT_MAX_ELEMENTS); + + if (collection.has("max_file_segment_size")) + max_file_segment_size = parseWithSizeSuffix(collection.get("max_file_segment_size")); + + cache_on_write_operations = collection.getOrDefault("cache_on_write_operations", false); + enable_filesystem_query_cache_limit = collection.getOrDefault("enable_filesystem_query_cache_limit", false); + cache_hits_threshold = collection.getOrDefault("cache_hits_threshold", FILECACHE_DEFAULT_HITS_THRESHOLD); + enable_bypass_cache_with_threshold = collection.getOrDefault("enable_bypass_cache_with_threshold", false); + + if (collection.has("bypass_cache_threshold")) + bypass_cache_threshold = parseWithSizeSuffix(collection.get("bypass_cache_threshold")); + + if (collection.has("boundary_alignment")) + boundary_alignment = parseWithSizeSuffix(collection.get("boundary_alignment")); + + if (collection.has("background_download_threads")) + background_download_threads = collection.get("background_download_threads"); + + if (collection.has("load_metadata_threads")) + load_metadata_threads = collection.get("load_metadata_threads"); +} + } diff --git a/src/Interpreters/Cache/FileCacheSettings.h b/src/Interpreters/Cache/FileCacheSettings.h index 9888b814a0b..9d15a32d3bf 100644 --- a/src/Interpreters/Cache/FileCacheSettings.h +++ b/src/Interpreters/Cache/FileCacheSettings.h @@ -8,6 +8,7 @@ namespace Poco { namespace Util { class AbstractConfiguration; } } // NOLINT(cpp namespace DB { +class NamedCollection; struct FileCacheSettings { @@ -31,6 +32,7 @@ struct FileCacheSettings size_t load_metadata_threads = FILECACHE_DEFAULT_LOAD_METADATA_THREADS; void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); + void loadFromCollection(const NamedCollection & collection); }; } diff --git a/tests/config/config.d/named_collection.xml b/tests/config/config.d/named_collection.xml index 5b716a7b8da..2c4b349a183 100644 --- a/tests/config/config.d/named_collection.xml +++ b/tests/config/config.d/named_collection.xml @@ -37,5 +37,9 @@ test testtest + + 1Mi + collection + diff --git a/tests/queries/0_stateless/02908_filesystem_cache_as_collection.reference b/tests/queries/0_stateless/02908_filesystem_cache_as_collection.reference new file mode 100644 index 00000000000..f5e0af6d507 --- /dev/null +++ b/tests/queries/0_stateless/02908_filesystem_cache_as_collection.reference @@ -0,0 +1,2 @@ +1048576 10000000 33554432 4194304 0 0 0 0 /var/lib/clickhouse/filesystem_caches/collection_sql 2 0 1 +1048576 10000000 33554432 4194304 0 0 0 0 /var/lib/clickhouse/filesystem_caches/collection 2 0 1 diff --git a/tests/queries/0_stateless/02908_filesystem_cache_as_collection.sql b/tests/queries/0_stateless/02908_filesystem_cache_as_collection.sql new file mode 100644 index 00000000000..e878727359b --- /dev/null +++ b/tests/queries/0_stateless/02908_filesystem_cache_as_collection.sql @@ -0,0 +1,8 @@ +CREATE NAMED COLLECTION IF NOT EXISTS cache_collection_sql AS path = 'collection_sql', max_size = '1Mi'; +DROP TABLE IF EXISTS test; +CREATE TABLE test (a Int32, b String) +ENGINE = MergeTree() ORDER BY a SETTINGS disk = disk(type = cache, disk = 's3_disk', name = '$CLICHOUSE_TEST_UNIQUE_NAME', cache_name='cache_collection_sql'); +DESCRIBE FILESYSTEM CACHE '$CLICHOUSE_TEST_UNIQUE_NAME'; +CREATE TABLE test2 (a Int32, b String) +ENGINE = MergeTree() ORDER BY a SETTINGS disk = disk(type = cache, disk = 's3_disk', name = '$CLICHOUSE_TEST_UNIQUE_NAME_2', cache_name='cache_collection'); +DESCRIBE FILESYSTEM CACHE '$CLICHOUSE_TEST_UNIQUE_NAME_2';