From a76a35d3ce142928ec1afa40698d988dd229823c Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 25 Apr 2022 18:49:00 +0200 Subject: [PATCH 1/4] Better drop cache --- src/Common/FileCache.cpp | 5 +-- src/Common/FileCache.h | 4 +-- src/Common/FileSegment.cpp | 4 +-- src/Interpreters/InterpreterSystemQuery.cpp | 4 +-- src/Parsers/ASTSystemQuery.h | 3 ++ src/Parsers/ParserSystemQuery.cpp | 10 ++++++ tests/config/config.d/storage_conf.xml | 18 ++++++++++ .../02286_drop_filesystem_cache.reference | 30 ++++++++++++++++ .../02286_drop_filesystem_cache.sql | 34 +++++++++++++++++++ 9 files changed, 104 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/02286_drop_filesystem_cache.reference create mode 100644 tests/queries/0_stateless/02286_drop_filesystem_cache.sql diff --git a/src/Common/FileCache.cpp b/src/Common/FileCache.cpp index 2f9eec33022..e8d6c52d948 100644 --- a/src/Common/FileCache.cpp +++ b/src/Common/FileCache.cpp @@ -555,7 +555,7 @@ void LRUFileCache::remove(const Key & key) fs::remove(key_path); } -void LRUFileCache::tryRemoveAll() +void LRUFileCache::remove(bool force_remove_unreleasable) { /// Try remove all cached files by cache_base_path. /// Only releasable file segments are evicted. @@ -567,12 +567,13 @@ void LRUFileCache::tryRemoveAll() auto & [key, offset] = *it++; auto * cell = getCell(key, offset, cache_lock); - if (cell->releasable()) + if (cell->releasable() || force_remove_unreleasable) { auto file_segment = cell->file_segment; if (file_segment) { std::lock_guard segment_lock(file_segment->mutex); + file_segment->detached = true; remove(file_segment->key(), file_segment->offset(), cache_lock, segment_lock); } } diff --git a/src/Common/FileCache.h b/src/Common/FileCache.h index 983156959de..d18bdc74acf 100644 --- a/src/Common/FileCache.h +++ b/src/Common/FileCache.h @@ -42,7 +42,7 @@ public: virtual void remove(const Key & key) = 0; - virtual void tryRemoveAll() = 0; + virtual void remove(bool force_remove_unreleasable) = 0; static bool isReadOnly(); @@ -145,7 +145,7 @@ public: void remove(const Key & key) override; - void tryRemoveAll() override; + void remove(bool force_remove_unreleasable) override; std::vector tryGetCachePaths(const Key & key) override; diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index e94a54ec3f7..4de8b043903 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -452,6 +452,8 @@ void FileSegment::completeBatchAndResetDownloader() void FileSegment::complete(State state) { + assertNotDetached(); + std::lock_guard cache_lock(cache->mutex); std::lock_guard segment_lock(mutex); @@ -477,8 +479,6 @@ void FileSegment::complete(State state) download_state = state; - assertNotDetached(); - try { completeImpl(cache_lock, segment_lock); diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 28a2082d233..289aba2a695 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -306,12 +306,12 @@ BlockIO InterpreterSystemQuery::execute() { auto caches = FileCacheFactory::instance().getAll(); for (const auto & [_, cache_data] : caches) - cache_data.cache->tryRemoveAll(); + cache_data.cache->remove(query.force_removal); } else { auto cache = FileCacheFactory::instance().get(query.filesystem_cache_path); - cache->tryRemoveAll(); + cache->remove(query.force_removal); } break; } diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index 600525f9abe..a5a2c724135 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -89,7 +89,10 @@ public: String volume; String disk; UInt64 seconds{}; + + /// Values for `drop fileystem cache` system query. String filesystem_cache_path; + bool force_removal = false; String getID(char) const override { return "SYSTEM query"; } diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 61e96b9c1de..8990becaedc 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -346,6 +346,16 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & res->seconds = seconds->as()->value.get(); break; } + case Type::DROP_FILESYSTEM_CACHE: + { + ParserLiteral path_parser; + ASTPtr ast; + if (path_parser.parse(pos, ast, expected)) + res->filesystem_cache_path = ast->as()->value.safeGet(); + if (ParserKeyword{"FORCE"}.ignore(pos, expected)) + res->force_removal = true; + break; + } default: { diff --git a/tests/config/config.d/storage_conf.xml b/tests/config/config.d/storage_conf.xml index 3dd4811b1bf..474e22fa577 100644 --- a/tests/config/config.d/storage_conf.xml +++ b/tests/config/config.d/storage_conf.xml @@ -10,7 +10,18 @@ 0 22548578304 1 + ./s3_cache/ + + s3 + http://localhost:11111/test/00170_test/ + clickhouse + clickhouse + 1 + 0 + 22548578304 + 0 + @@ -20,6 +31,13 @@ + + +
+ s3_cache +
+ + diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.reference b/tests/queries/0_stateless/02286_drop_filesystem_cache.reference new file mode 100644 index 00000000000..0dabc778274 --- /dev/null +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.reference @@ -0,0 +1,30 @@ +-- { echo } + +SET enable_filesystem_cache_on_write_operations=0; +DROP TABLE IF EXISTS test; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760; +SYSTEM DROP FILESYSTEM CACHE; +SELECT count() FROM system.filesystem_cache; +0 +INSERT INTO test SELECT number, toString(number) FROM numbers(100); +SELECT * FROM test FORMAT Null; +SELECT count() FROM system.filesystem_cache; +2 +SYSTEM DROP FILESYSTEM CACHE FORCE; +SELECT count() FROM system.filesystem_cache; +0 +SELECT * FROM test FORMAT Null; +SELECT count() FROM system.filesystem_cache; +1 +SYSTEM DROP FILESYSTEM CACHE './data'; -- { serverError 36 } +SELECT count() FROM system.filesystem_cache; +1 +DROP TABLE IF EXISTS test2; +CREATE TABLE test2 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_2', min_bytes_for_wide_part = 10485760; +INSERT INTO test2 SELECT number, toString(number) FROM numbers(100); +SELECT * FROM test2 FORMAT Null; +SELECT count() FROM system.filesystem_cache; +3 +SYSTEM DROP FILESYSTEM CACHE './s3_cache/'; +SELECT count() FROM system.filesystem_cache; +2 diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.sql b/tests/queries/0_stateless/02286_drop_filesystem_cache.sql new file mode 100644 index 00000000000..4c99c248dbc --- /dev/null +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.sql @@ -0,0 +1,34 @@ +-- Tags: no-parallel, no-fasttest, no-s3-storage + +-- { echo } + +SET enable_filesystem_cache_on_write_operations=0; + +DROP TABLE IF EXISTS test; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760; + +SYSTEM DROP FILESYSTEM CACHE; + +SELECT count() FROM system.filesystem_cache; +INSERT INTO test SELECT number, toString(number) FROM numbers(100); + +SELECT * FROM test FORMAT Null; +SELECT count() FROM system.filesystem_cache; + +SYSTEM DROP FILESYSTEM CACHE FORCE; +SELECT count() FROM system.filesystem_cache; + +SELECT * FROM test FORMAT Null; +SELECT count() FROM system.filesystem_cache; + +SYSTEM DROP FILESYSTEM CACHE './data'; -- { serverError 36 } +SELECT count() FROM system.filesystem_cache; + +DROP TABLE IF EXISTS test2; +CREATE TABLE test2 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_2', min_bytes_for_wide_part = 10485760; +INSERT INTO test2 SELECT number, toString(number) FROM numbers(100); +SELECT * FROM test2 FORMAT Null; +SELECT count() FROM system.filesystem_cache; + +SYSTEM DROP FILESYSTEM CACHE './s3_cache/'; +SELECT count() FROM system.filesystem_cache; From cf34ae0f673cda51905ae27adaa6c2fd4c5047cb Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 25 Apr 2022 19:40:28 +0200 Subject: [PATCH 2/4] Update ASTSystemQuery.h --- src/Parsers/ASTSystemQuery.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index a5a2c724135..213b741f63b 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -90,7 +90,7 @@ public: String disk; UInt64 seconds{}; - /// Values for `drop fileystem cache` system query. + /// Values for `drop filesystem cache` system query. String filesystem_cache_path; bool force_removal = false; From 3215faca72368a1b29ea707148ae5d9629f46040 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 25 Apr 2022 21:15:38 +0200 Subject: [PATCH 3/4] Update storage_conf.xml --- tests/config/config.d/storage_conf.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/config.d/storage_conf.xml b/tests/config/config.d/storage_conf.xml index 474e22fa577..45fad002c88 100644 --- a/tests/config/config.d/storage_conf.xml +++ b/tests/config/config.d/storage_conf.xml @@ -34,7 +34,7 @@
- s3_cache + s3_cache_2
From 35b8d302da85472572c42e7d31932ff6825e21ea Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 25 Apr 2022 21:57:13 +0200 Subject: [PATCH 4/4] Fix --- src/Common/FileSegment.cpp | 4 ++-- src/Parsers/ASTSystemQuery.cpp | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index 4de8b043903..a14051fa9c8 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -452,11 +452,11 @@ void FileSegment::completeBatchAndResetDownloader() void FileSegment::complete(State state) { - assertNotDetached(); - std::lock_guard cache_lock(cache->mutex); std::lock_guard segment_lock(mutex); + assertNotDetached(); + bool is_downloader = isDownloaderImpl(segment_lock); if (!is_downloader) { diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index a4b0a69faaa..274d1639b4b 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -192,6 +192,13 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &, << (settings.hilite ? hilite_keyword : "") << " SECOND" << (settings.hilite ? hilite_none : ""); } + else if (type == Type::DROP_FILESYSTEM_CACHE) + { + if (!filesystem_cache_path.empty()) + settings.ostr << (settings.hilite ? hilite_none : "") << filesystem_cache_path; + if (force_removal) + settings.ostr << (settings.hilite ? hilite_keyword : "") << " FORCE"; + } }