From 4ba7530c236d80743e3c9917f493224cd2b1fc4d Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 5 May 2023 00:21:13 +0200 Subject: [PATCH 1/4] Fix thread MemoryTracker parent after detach --- src/Interpreters/ThreadStatusExt.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index fd4a6b5e996..5a37d324fc3 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -223,7 +223,8 @@ void ThreadStatus::detachFromGroup() performance_counters.setParent(&ProfileEvents::global_counters); memory_tracker.reset(); - memory_tracker.setParent(thread_group->memory_tracker.getParent()); + /// Extract MemoryTracker out from query and user context + memory_tracker.setParent(&total_memory_tracker); thread_group.reset(); From 951bef4fd219587dcabe1d7b01e9bfbf19a7c62f Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 5 May 2023 15:10:50 +0000 Subject: [PATCH 2/4] Fixmemory counting in TCPHandler --- src/Server/TCPHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index e45a26d76af..4e41fbf4230 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -505,8 +505,8 @@ void TCPHandler::runImpl() /// the MemoryTracker will be wrong for possible deallocations. /// (i.e. deallocations from the Aggregator with two-level aggregation) state.reset(); + last_sent_snapshots = {}; query_scope.reset(); - last_sent_snapshots.clear(); thread_trace_context.reset(); } catch (const Exception & e) From 2536dfad3a5ac3e99976e4fb00d8ed62e22da946 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 5 May 2023 18:21:25 +0000 Subject: [PATCH 3/4] Fix build --- src/Server/TCPHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 4e41fbf4230..0522b6d8a48 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -505,7 +505,7 @@ void TCPHandler::runImpl() /// the MemoryTracker will be wrong for possible deallocations. /// (i.e. deallocations from the Aggregator with two-level aggregation) state.reset(); - last_sent_snapshots = {}; + last_sent_snapshots = ProfileEvents::ThreadIdToCountersSnapshot{}; query_scope.reset(); thread_trace_context.reset(); } From 0598bfd79e6518c9432e341af100a31649f457e0 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Mar 2023 14:38:40 +0000 Subject: [PATCH 4/4] Mark query cache as non-experimental --- docs/en/operations/caches.md | 2 +- docs/en/operations/query-cache.md | 19 ++++--------------- src/Core/Settings.h | 1 - src/Interpreters/Cache/QueryCache.cpp | 4 +--- src/Interpreters/executeQuery.cpp | 4 +--- tests/ci/stress.py | 1 - ...494_query_cache_case_agnostic_matching.sql | 2 -- .../02494_query_cache_compression.sql | 2 -- .../02494_query_cache_drop_cache.sql | 2 -- .../02494_query_cache_eligible_queries.sql | 2 -- .../0_stateless/02494_query_cache_events.sql | 2 -- .../02494_query_cache_exception_handling.sql | 2 -- .../0_stateless/02494_query_cache_explain.sql | 1 - .../02494_query_cache_min_query_duration.sql | 2 -- .../02494_query_cache_min_query_runs.sql | 2 -- ...query_cache_nondeterministic_functions.sql | 2 -- .../02494_query_cache_normalize_ast.sql | 2 -- .../02494_query_cache_passive_usage.sql | 2 -- .../0_stateless/02494_query_cache_secrets.sql | 2 -- .../02494_query_cache_sparse_columns.sql | 1 - ...494_query_cache_squash_partial_results.sql | 2 -- .../02494_query_cache_totals_extremes.sql | 2 -- .../02494_query_cache_ttl_long.sql | 2 -- .../02494_query_cache_user_quotas.sql | 2 -- 24 files changed, 7 insertions(+), 58 deletions(-) diff --git a/docs/en/operations/caches.md b/docs/en/operations/caches.md index 86bf8065d94..5bc16b6f51d 100644 --- a/docs/en/operations/caches.md +++ b/docs/en/operations/caches.md @@ -23,6 +23,6 @@ Additional cache types: - [Dictionaries](../sql-reference/dictionaries/index.md) data cache. - Schema inference cache. - [Filesystem cache](storing-data.md) over S3, Azure, Local and other disks. -- [(Experimental) Query cache](query-cache.md). +- [Query cache](query-cache.md). To drop one of the caches, use [SYSTEM DROP ... CACHE](../sql-reference/statements/system.md#drop-mark-cache) statements. diff --git a/docs/en/operations/query-cache.md b/docs/en/operations/query-cache.md index 50f73e2c1bb..bfa51650cd8 100644 --- a/docs/en/operations/query-cache.md +++ b/docs/en/operations/query-cache.md @@ -1,10 +1,10 @@ --- slug: /en/operations/query-cache sidebar_position: 65 -sidebar_label: Query Cache [experimental] +sidebar_label: Query Cache --- -# Query Cache [experimental] +# Query Cache The query cache allows to compute `SELECT` queries just once and to serve further executions of the same query directly from the cache. Depending on the type of the queries, this can dramatically reduce latency and resource consumption of the ClickHouse server. @@ -29,21 +29,10 @@ Transactionally inconsistent caching is traditionally provided by client tools o the same caching logic and configuration is often duplicated. With ClickHouse's query cache, the caching logic moves to the server side. This reduces maintenance effort and avoids redundancy. -:::note -The query cache is an experimental feature that should not be used in production. There are known cases (e.g. in distributed query -processing) where wrong results are returned. -::: - ## Configuration Settings and Usage -As long as the result cache is experimental it must be activated using the following configuration setting: - -```sql -SET allow_experimental_query_cache = true; -``` - -Afterwards, setting [use_query_cache](settings/settings.md#use-query-cache) can be used to control whether a specific query or all queries -of the current session should utilize the query cache. For example, the first execution of query +Setting [use_query_cache](settings/settings.md#use-query-cache) can be used to control whether a specific query or all queries of the +current session should utilize the query cache. For example, the first execution of query ```sql SELECT some_expensive_calculation(column_1, column_2) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 27e2b1801b7..42ae645615a 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -720,7 +720,6 @@ class IColumn; M(Bool, allow_experimental_nlp_functions, false, "Enable experimental functions for natural language processing.", 0) \ M(Bool, allow_experimental_hash_functions, false, "Enable experimental hash functions (hashid, etc)", 0) \ M(Bool, allow_experimental_object_type, false, "Allow Object and JSON data types", 0) \ - M(Bool, allow_experimental_query_cache, false, "Enable experimental query cache", 0) \ M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \ M(String, ann_index_select_query_params, "", "Parameters passed to ANN indexes in SELECT queries, the format is 'param1=x, param2=y, ...'", 0) \ M(UInt64, max_limit_for_ann_queries, 1000000, "Maximum limit value for using ANN indexes is used to prevent memory overflow in search queries for indexes", 0) \ diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp index 796be10ccc8..aab004de4e5 100644 --- a/src/Interpreters/Cache/QueryCache.cpp +++ b/src/Interpreters/Cache/QueryCache.cpp @@ -79,9 +79,7 @@ public: auto is_query_cache_related_setting = [](const auto & change) { - return change.name == "allow_experimental_query_cache" - || change.name.starts_with("query_cache") - || change.name.ends_with("query_cache"); + return change.name.starts_with("query_cache_") || change.name.ends_with("_query_cache"); }; std::erase_if(set_clause->changes, is_query_cache_related_setting); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 763cf7ffc85..3a6b1f2124f 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -643,9 +643,7 @@ static std::tuple executeQueryImpl( } } - bool can_use_query_cache = - settings.allow_experimental_query_cache && settings.use_query_cache && !internal - && !ast->as(); + bool can_use_query_cache = settings.use_query_cache && !internal && !ast->as(); if (!async_insert) { diff --git a/tests/ci/stress.py b/tests/ci/stress.py index 953b55dbf63..b9044874071 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -45,7 +45,6 @@ def get_options(i, upgrade_check): client_options.append("max_rows_in_join=1000") if i > 0 and random.random() < 1 / 3: - client_options.append("allow_experimental_query_cache=1") client_options.append("use_query_cache=1") if i % 5 == 1: diff --git a/tests/queries/0_stateless/02494_query_cache_case_agnostic_matching.sql b/tests/queries/0_stateless/02494_query_cache_case_agnostic_matching.sql index 9440a1fd9c0..9f26d7759de 100644 --- a/tests/queries/0_stateless/02494_query_cache_case_agnostic_matching.sql +++ b/tests/queries/0_stateless/02494_query_cache_case_agnostic_matching.sql @@ -1,8 +1,6 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - -- Start with empty query cache (QC) and query log SYSTEM DROP QUERY CACHE; DROP TABLE system.query_log SYNC; diff --git a/tests/queries/0_stateless/02494_query_cache_compression.sql b/tests/queries/0_stateless/02494_query_cache_compression.sql index 619ae7d5cd8..3d17deebd94 100644 --- a/tests/queries/0_stateless/02494_query_cache_compression.sql +++ b/tests/queries/0_stateless/02494_query_cache_compression.sql @@ -1,8 +1,6 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - SYSTEM DROP QUERY CACHE; DROP TABLE IF EXISTS t; diff --git a/tests/queries/0_stateless/02494_query_cache_drop_cache.sql b/tests/queries/0_stateless/02494_query_cache_drop_cache.sql index 078057a834f..bc2e7f442fc 100644 --- a/tests/queries/0_stateless/02494_query_cache_drop_cache.sql +++ b/tests/queries/0_stateless/02494_query_cache_drop_cache.sql @@ -1,8 +1,6 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - -- (it's silly to use what will be tested below but we have to assume other tests cluttered the query cache) SYSTEM DROP QUERY CACHE; diff --git a/tests/queries/0_stateless/02494_query_cache_eligible_queries.sql b/tests/queries/0_stateless/02494_query_cache_eligible_queries.sql index b4bc9e2c258..5c45ee8aedd 100644 --- a/tests/queries/0_stateless/02494_query_cache_eligible_queries.sql +++ b/tests/queries/0_stateless/02494_query_cache_eligible_queries.sql @@ -1,8 +1,6 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - SYSTEM DROP QUERY CACHE; DROP TABLE IF EXISTS eligible_test; DROP TABLE IF EXISTS eligible_test2; diff --git a/tests/queries/0_stateless/02494_query_cache_events.sql b/tests/queries/0_stateless/02494_query_cache_events.sql index d775467d525..900b68f5eb2 100644 --- a/tests/queries/0_stateless/02494_query_cache_events.sql +++ b/tests/queries/0_stateless/02494_query_cache_events.sql @@ -1,8 +1,6 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - -- Start with empty query cache QC and query log SYSTEM DROP QUERY CACHE; DROP TABLE system.query_log SYNC; diff --git a/tests/queries/0_stateless/02494_query_cache_exception_handling.sql b/tests/queries/0_stateless/02494_query_cache_exception_handling.sql index 4d686d81ed3..70a443cc75c 100644 --- a/tests/queries/0_stateless/02494_query_cache_exception_handling.sql +++ b/tests/queries/0_stateless/02494_query_cache_exception_handling.sql @@ -1,8 +1,6 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - SYSTEM DROP QUERY CACHE; -- If an exception is thrown during query execution, no entry must be created in the query cache diff --git a/tests/queries/0_stateless/02494_query_cache_explain.sql b/tests/queries/0_stateless/02494_query_cache_explain.sql index 68b7e0005f8..d12938181c2 100644 --- a/tests/queries/0_stateless/02494_query_cache_explain.sql +++ b/tests/queries/0_stateless/02494_query_cache_explain.sql @@ -2,7 +2,6 @@ -- Tag no-parallel: Messes with internal cache SET allow_experimental_analyzer = 1; -SET allow_experimental_query_cache = true; SYSTEM DROP QUERY CACHE; diff --git a/tests/queries/0_stateless/02494_query_cache_min_query_duration.sql b/tests/queries/0_stateless/02494_query_cache_min_query_duration.sql index 7d759c86130..ee0b73f43e2 100644 --- a/tests/queries/0_stateless/02494_query_cache_min_query_duration.sql +++ b/tests/queries/0_stateless/02494_query_cache_min_query_duration.sql @@ -1,8 +1,6 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - SYSTEM DROP QUERY CACHE; -- This creates an entry in the query cache ... diff --git a/tests/queries/0_stateless/02494_query_cache_min_query_runs.sql b/tests/queries/0_stateless/02494_query_cache_min_query_runs.sql index 2401beeab13..ffdf12169e3 100644 --- a/tests/queries/0_stateless/02494_query_cache_min_query_runs.sql +++ b/tests/queries/0_stateless/02494_query_cache_min_query_runs.sql @@ -1,8 +1,6 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - SYSTEM DROP QUERY CACHE; -- Cache the query after the 1st query invocation diff --git a/tests/queries/0_stateless/02494_query_cache_nondeterministic_functions.sql b/tests/queries/0_stateless/02494_query_cache_nondeterministic_functions.sql index 534d63aa427..045b7258a34 100644 --- a/tests/queries/0_stateless/02494_query_cache_nondeterministic_functions.sql +++ b/tests/queries/0_stateless/02494_query_cache_nondeterministic_functions.sql @@ -1,8 +1,6 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - SYSTEM DROP QUERY CACHE; -- rand() is non-deterministic, with default settings no entry in the query cache should be created diff --git a/tests/queries/0_stateless/02494_query_cache_normalize_ast.sql b/tests/queries/0_stateless/02494_query_cache_normalize_ast.sql index 7e3cd273312..5fd09eb935b 100644 --- a/tests/queries/0_stateless/02494_query_cache_normalize_ast.sql +++ b/tests/queries/0_stateless/02494_query_cache_normalize_ast.sql @@ -1,8 +1,6 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - -- Start with empty query cache (QC) and query log. SYSTEM DROP QUERY CACHE; DROP TABLE system.query_log SYNC; diff --git a/tests/queries/0_stateless/02494_query_cache_passive_usage.sql b/tests/queries/0_stateless/02494_query_cache_passive_usage.sql index 8f1e3972b6d..6143b5f7083 100644 --- a/tests/queries/0_stateless/02494_query_cache_passive_usage.sql +++ b/tests/queries/0_stateless/02494_query_cache_passive_usage.sql @@ -1,8 +1,6 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - -- Start with empty query cache (QC). SYSTEM DROP QUERY CACHE; diff --git a/tests/queries/0_stateless/02494_query_cache_secrets.sql b/tests/queries/0_stateless/02494_query_cache_secrets.sql index 99a972b003c..66427df8f10 100644 --- a/tests/queries/0_stateless/02494_query_cache_secrets.sql +++ b/tests/queries/0_stateless/02494_query_cache_secrets.sql @@ -2,8 +2,6 @@ -- Tag no-fasttest: Depends on OpenSSL -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - SYSTEM DROP QUERY CACHE; -- Cache a result of a query with secret in the query cache diff --git a/tests/queries/0_stateless/02494_query_cache_sparse_columns.sql b/tests/queries/0_stateless/02494_query_cache_sparse_columns.sql index 952c3227852..4344d139d60 100644 --- a/tests/queries/0_stateless/02494_query_cache_sparse_columns.sql +++ b/tests/queries/0_stateless/02494_query_cache_sparse_columns.sql @@ -12,7 +12,6 @@ SYSTEM STOP MERGES t_cache_sparse; INSERT INTO t_cache_sparse SELECT number, number FROM numbers(10000); INSERT INTO t_cache_sparse SELECT number, 0 FROM numbers(10000); -SET allow_experimental_query_cache = 1; SET use_query_cache = 1; SET max_threads = 1; diff --git a/tests/queries/0_stateless/02494_query_cache_squash_partial_results.sql b/tests/queries/0_stateless/02494_query_cache_squash_partial_results.sql index 0dd5fcb2977..391cc7f7e1a 100644 --- a/tests/queries/0_stateless/02494_query_cache_squash_partial_results.sql +++ b/tests/queries/0_stateless/02494_query_cache_squash_partial_results.sql @@ -1,8 +1,6 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - SYSTEM DROP QUERY CACHE; DROP TABLE IF EXISTS t; diff --git a/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql b/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql index 0d05adca6c8..8fde4a668c8 100644 --- a/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql +++ b/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql @@ -1,8 +1,6 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - SYSTEM DROP QUERY CACHE; DROP TABLE IF EXISTS tbl; diff --git a/tests/queries/0_stateless/02494_query_cache_ttl_long.sql b/tests/queries/0_stateless/02494_query_cache_ttl_long.sql index 135ddf2195c..acaf34ee81d 100644 --- a/tests/queries/0_stateless/02494_query_cache_ttl_long.sql +++ b/tests/queries/0_stateless/02494_query_cache_ttl_long.sql @@ -3,8 +3,6 @@ -- Tag long: Test runtime is > 6 sec -- Tag no-parallel: Messes with internal cache -SET allow_experimental_query_cache = true; - SYSTEM DROP QUERY CACHE; -- Cache query result into query cache with a TTL of 3 sec diff --git a/tests/queries/0_stateless/02494_query_cache_user_quotas.sql b/tests/queries/0_stateless/02494_query_cache_user_quotas.sql index 9890cdf7933..123c9d21101 100644 --- a/tests/queries/0_stateless/02494_query_cache_user_quotas.sql +++ b/tests/queries/0_stateless/02494_query_cache_user_quotas.sql @@ -5,8 +5,6 @@ -- be used in a settings profile, together with a readonly constraint. For simplicity, test both settings stand-alone in a stateless test -- instead of an integration test - the relevant logic will still be covered by that. -SET allow_experimental_query_cache = true; - SYSTEM DROP QUERY CACHE; SET query_cache_max_size_in_bytes = 1;