From 4e9d894e24041f7c462a486394f19d602403ae15 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 23 Oct 2022 01:32:35 +0200
Subject: [PATCH 001/689] Compress marks and primary key by default

---
 src/Storages/MergeTree/MergeTreeSettings.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h
index a0db39a97f1..210846b9bb2 100644
--- a/src/Storages/MergeTree/MergeTreeSettings.h
+++ b/src/Storages/MergeTree/MergeTreeSettings.h
@@ -147,8 +147,8 @@ struct Settings;
     M(Bool, remote_fs_zero_copy_path_compatible_mode, false, "Run zero-copy in compatible mode during conversion process.", 0) \
     \
     /** Compress marks and primary key. */ \
-    M(Bool, compress_marks, false, "Marks support compression, reduce mark file size and speed up network transmission.", 0) \
-    M(Bool, compress_primary_key, false, "Primary key support compression, reduce primary key file size and speed up network transmission.", 0) \
+    M(Bool, compress_marks, true, "Marks support compression, reduce mark file size and speed up network transmission.", 0) \
+    M(Bool, compress_primary_key, true, "Primary key support compression, reduce primary key file size and speed up network transmission.", 0) \
     M(String, marks_compression_codec, "ZSTD(3)", "Compression encoding used by marks, marks are small enough and cached, so the default compression is ZSTD(3).", 0) \
     M(String, primary_key_compression_codec, "ZSTD(3)", "Compression encoding used by primary, primary key is small enough and cached, so the default compression is ZSTD(3).", 0) \
     M(UInt64, marks_compress_block_size, 65536, "Mark compress block size, the actual size of the block to compress.", 0) \

From 462e7f76bf20b2872e3605a17080a97d1d6e8e33 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 27 Dec 2022 13:28:25 +0100
Subject: [PATCH 002/689] Attempt to fix the "system.stack_trace" test

---
 src/Core/Settings.h                                    | 1 +
 src/Storages/System/StorageSystemStackTrace.cpp        | 6 +++++-
 tests/queries/0_stateless/01051_system_stack_trace.sql | 2 +-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index b10760c8277..f26d6a2e12a 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -655,6 +655,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
     M(String, additional_result_filter, "", "Additional filter expression which would be applied to query result", 0) \
     \
     M(String, workload, "default", "Name of workload to be used to access resources", 0) \
+    M(Milliseconds, storage_system_stack_trace_pipe_read_timeout_ms, 100, "Maximum time to read from a pipe for receiving information from the threads when querying the `system.stack_trace` table. This setting is used for testing purposes and not meant to be changed by users.", 0) \
     \
     /** Experimental functions */ \
     M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \
diff --git a/src/Storages/System/StorageSystemStackTrace.cpp b/src/Storages/System/StorageSystemStackTrace.cpp
index df3d8b74e6e..8f3f9744725 100644
--- a/src/Storages/System/StorageSystemStackTrace.cpp
+++ b/src/Storages/System/StorageSystemStackTrace.cpp
@@ -22,6 +22,7 @@
 #include <Common/CurrentThread.h>
 #include <Common/HashTable/Hash.h>
 #include <Common/logger_useful.h>
+#include <Interpreters/Context.h>
 #include <Processors/Sources/SourceFromSingleChunk.h>
 #include <QueryPipeline/Pipe.h>
 #include <base/getThreadId.h>
@@ -262,6 +263,9 @@ Pipe StorageSystemStackTrace::read(
 {
     storage_snapshot->check(column_names);
 
+    int pipe_read_timeout_ms = static_cast<int>(
+        context->getSettingsRef().storage_system_stack_trace_pipe_read_timeout_ms.totalMilliseconds());
+
     /// It shouldn't be possible to do concurrent reads from this table.
     std::lock_guard lock(mutex);
 
@@ -334,7 +338,7 @@ Pipe StorageSystemStackTrace::read(
             }
 
             /// Just in case we will wait for pipe with timeout. In case signal didn't get processed.
-            if (send_signal && wait(100) && sig_value.sival_int == data_ready_num.load(std::memory_order_acquire))
+            if (send_signal && wait(pipe_read_timeout_ms) && sig_value.sival_int == data_ready_num.load(std::memory_order_acquire))
             {
                 size_t stack_trace_size = stack_trace.getSize();
                 size_t stack_trace_offset = stack_trace.getOffset();
diff --git a/tests/queries/0_stateless/01051_system_stack_trace.sql b/tests/queries/0_stateless/01051_system_stack_trace.sql
index e322462a46a..a7e6660d06e 100644
--- a/tests/queries/0_stateless/01051_system_stack_trace.sql
+++ b/tests/queries/0_stateless/01051_system_stack_trace.sql
@@ -1,4 +1,4 @@
--- Tags: race
+SET storage_system_stack_trace_pipe_read_timeout_ms = 10000;
 
 -- { echo }
 SELECT count() > 0 FROM system.stack_trace WHERE query_id != '';

From d7f4bc4c72f238d4de1ac9dfda786d3a678c29c5 Mon Sep 17 00:00:00 2001
From: Salvatore Mesoraca <salvatore.mesoraca@aiven.io>
Date: Wed, 15 Mar 2023 17:01:19 +0100
Subject: [PATCH 003/689] ActionsDAG: remove wrong optimization

Replacing AND with a casto to UInt8 can change the condition.
e.g.
0x100 cast to UInt8 is 0, but it should be 1.
0.1 cast to UInt8 is 0, but it should be 1.

Basically we can't cast anything to UInt8 so we completely remove
this optimization.

Closes #47317
---
 src/Interpreters/ActionsDAG.cpp               | 62 ++++++-------------
 .../02568_and_consistency.reference           |  5 ++
 .../0_stateless/02568_and_consistency.sql     | 42 +++++++++++++
 3 files changed, 66 insertions(+), 43 deletions(-)
 create mode 100644 tests/queries/0_stateless/02568_and_consistency.reference
 create mode 100644 tests/queries/0_stateless/02568_and_consistency.sql

diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index 46b5a93b28c..ac11862d375 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -1966,8 +1966,12 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown(
     }
 
     auto conjunction = getConjunctionNodes(predicate, allowed_nodes);
-    if (conjunction.rejected.size() == 1 && WhichDataType{removeNullable(conjunction.rejected.front()->result_type)}.isFloat())
+    if (conjunction.rejected.size() == 1 && !conjunction.rejected.front()->result_type->equals(*predicate->result_type)
+        && conjunction.allowed.front()->type == ActionType::COLUMN)
+    {
+        // No further optimization can be done
         return nullptr;
+    }
 
     auto actions = cloneActionsForConjunction(conjunction.allowed, all_inputs);
     if (!actions)
@@ -2017,55 +2021,26 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown(
     else
     {
         /// Predicate is conjunction, where both allowed and rejected sets are not empty.
-        /// Replace this node to conjunction of rejected predicates.
 
         NodeRawConstPtrs new_children = std::move(conjunction.rejected);
 
-        if (new_children.size() == 1)
+        if (new_children.size() == 1 && new_children.front()->result_type->equals(*predicate->result_type))
         {
-            /// Rejected set has only one predicate.
-            if (new_children.front()->result_type->equals(*predicate->result_type))
-            {
-                /// If it's type is same, just add alias.
-                Node node;
-                node.type = ActionType::ALIAS;
-                node.result_name = predicate->result_name;
-                node.result_type = predicate->result_type;
-                node.children.swap(new_children);
-                *predicate = std::move(node);
-            }
-            else if (!WhichDataType{removeNullable(new_children.front()->result_type)}.isFloat())
-            {
-                /// If type is different, cast column.
-                /// This case is possible, cause AND can use any numeric type as argument.
-                /// But casting floats to UInt8 or Bool produces different results.
-                /// so we can't apply this optimization to them.
-                Node node;
-                node.type = ActionType::COLUMN;
-                node.result_name = predicate->result_type->getName();
-                node.column = DataTypeString().createColumnConst(0, node.result_name);
-                node.result_type = std::make_shared<DataTypeString>();
-
-                const auto * right_arg = &nodes.emplace_back(std::move(node));
-                const auto * left_arg = new_children.front();
-
-                predicate->children = {left_arg, right_arg};
-                auto arguments = prepareFunctionArguments(predicate->children);
-
-                FunctionOverloadResolverPtr func_builder_cast = CastInternalOverloadResolver<CastType::nonAccurate>::createImpl();
-
-                predicate->function_base = func_builder_cast->build(arguments);
-                predicate->function = predicate->function_base->prepare(arguments);
-            }
+            /// Rejected set has only one predicate. And the type is the same as the result_type.
+            /// Just add alias.
+            Node node;
+            node.type = ActionType::ALIAS;
+            node.result_name = predicate->result_name;
+            node.result_type = predicate->result_type;
+            node.children.swap(new_children);
+            *predicate = std::move(node);
         }
         else
         {
-            /// Predicate is function AND, which still have more then one argument.
-            /// Or there is only one argument that is a float and we can't just
-            /// remove the AND.
+            /// Predicate is function AND, which still have more then one argument
+            /// or it has one argument of the wrong type.
             /// Just update children and rebuild it.
-            predicate->children.swap(new_children);
-            if (WhichDataType{removeNullable(predicate->children.front()->result_type)}.isFloat())
+            if (new_children.size() == 1)
             {
                 Node node;
                 node.type = ActionType::COLUMN;
@@ -2073,8 +2048,9 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown(
                 node.column = DataTypeUInt8().createColumnConst(0, 1u);
                 node.result_type = std::make_shared<DataTypeUInt8>();
                 const auto * const_col = &nodes.emplace_back(std::move(node));
-                predicate->children.emplace_back(const_col);
+                new_children.emplace_back(const_col);
             }
+            predicate->children.swap(new_children);
             auto arguments = prepareFunctionArguments(predicate->children);
 
             FunctionOverloadResolverPtr func_builder_and = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionAnd>());
diff --git a/tests/queries/0_stateless/02568_and_consistency.reference b/tests/queries/0_stateless/02568_and_consistency.reference
new file mode 100644
index 00000000000..07a8041d0ee
--- /dev/null
+++ b/tests/queries/0_stateless/02568_and_consistency.reference
@@ -0,0 +1,5 @@
+=
+1554690688
+=
+1554690688
+=
diff --git a/tests/queries/0_stateless/02568_and_consistency.sql b/tests/queries/0_stateless/02568_and_consistency.sql
new file mode 100644
index 00000000000..4e76da78427
--- /dev/null
+++ b/tests/queries/0_stateless/02568_and_consistency.sql
@@ -0,0 +1,42 @@
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (c0 Int32, PRIMARY KEY (c0)) ENGINE=MergeTree;
+INSERT INTO t1 VALUES (1554690688);
+
+select '=';
+
+SELECT MIN(t1.c0)
+FROM t1
+GROUP BY
+    (-sign(cos(t1.c0))) * (-max2(t1.c0, t1.c0 / t1.c0)),
+    t1.c0 * t1.c0,
+    sign(-exp(-t1.c0))
+HAVING -(-(MIN(t1.c0) + MIN(t1.c0))) AND (pow('{b' > '-657301241', log(-1004522121)) IS NOT NULL)
+UNION ALL
+SELECT MIN(t1.c0)
+FROM t1
+GROUP BY
+    (-sign(cos(t1.c0))) * (-max2(t1.c0, t1.c0 / t1.c0)),
+    t1.c0 * t1.c0,
+    sign(-exp(-t1.c0))
+HAVING NOT (-(-(MIN(t1.c0) + MIN(t1.c0))) AND (pow('{b' > '-657301241', log(-1004522121)) IS NOT NULL))
+UNION ALL
+SELECT MIN(t1.c0)
+FROM t1
+GROUP BY
+    (-sign(cos(t1.c0))) * (-max2(t1.c0, t1.c0 / t1.c0)),
+    t1.c0 * t1.c0,
+    sign(-exp(-t1.c0))
+HAVING (-(-(MIN(t1.c0) + MIN(t1.c0))) AND (pow('{b' > '-657301241', log(-1004522121)) IS NOT NULL)) IS NULL
+SETTINGS aggregate_functions_null_for_empty = 1, enable_optimize_predicate_expression = 0;
+
+select '=';
+
+SELECT MIN(t1.c0)
+FROM t1
+GROUP BY t1.c0
+HAVING and(MIN(t1.c0) + MIN(t1.c0), 1)
+SETTINGS aggregate_functions_null_for_empty = 1, enable_optimize_predicate_expression = 0;
+
+select '=';
+
+DROP TABLE IF EXISTS t1;

From e7d19cc45f5e9b42d2b6b6a063835c6e9614e3c1 Mon Sep 17 00:00:00 2001
From: Salvatore Mesoraca <salvatore.mesoraca@aiven.io>
Date: Wed, 15 Mar 2023 17:15:22 +0100
Subject: [PATCH 004/689] Fix test that expected CH to apply a wrong
 optimization

The result of minus is an int64.
Casting that result to uint8 instead of bool can lead
to wrong results e.g.
256 cast to uint8 is 0, but it should be `true`
---
 tests/queries/0_stateless/01655_plan_optimizations.reference | 2 +-
 tests/queries/0_stateless/01655_plan_optimizations.sh        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01655_plan_optimizations.reference b/tests/queries/0_stateless/01655_plan_optimizations.reference
index f870a52284c..48d99647b43 100644
--- a/tests/queries/0_stateless/01655_plan_optimizations.reference
+++ b/tests/queries/0_stateless/01655_plan_optimizations.reference
@@ -53,7 +53,7 @@ Filter column: notEquals(y, 0)
 9	10
 > one condition of filter should be pushed down after aggregating, other condition is casted
 Filter column
-FUNCTION _CAST(minus(s, 4) :: 1, UInt8 :: 3) -> and(notEquals(y, 0), minus(s, 4))
+FUNCTION and(minus(s, 4) :: 1, 1 :: 3) -> and(notEquals(y, 0), minus(s, 4)) UInt8 : 2
 Aggregating
 Filter column: notEquals(y, 0)
 0	1
diff --git a/tests/queries/0_stateless/01655_plan_optimizations.sh b/tests/queries/0_stateless/01655_plan_optimizations.sh
index aaecdc390cb..ec856c9bf27 100755
--- a/tests/queries/0_stateless/01655_plan_optimizations.sh
+++ b/tests/queries/0_stateless/01655_plan_optimizations.sh
@@ -56,7 +56,7 @@ $CLICKHOUSE_CLIENT -q "
         select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y
     ) where y != 0 and s - 4
     settings enable_optimize_predicate_expression=0" |
-    grep -o "Aggregating\|Filter column\|Filter column: notEquals(y, 0)\|FUNCTION _CAST(minus(s, 4) :: 1, UInt8 :: 3) -> and(notEquals(y, 0), minus(s, 4))"
+    grep -o "Aggregating\|Filter column\|Filter column: notEquals(y, 0)\|FUNCTION and(minus(s, 4) :: 1, 1 :: 3) -> and(notEquals(y, 0), minus(s, 4)) UInt8 : 2"
 $CLICKHOUSE_CLIENT -q "
     select s, y from (
         select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y

From 89a76c2d771473a8d99d4d0522961e787e829657 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 18 Mar 2023 23:45:57 +0100
Subject: [PATCH 005/689] Use LLVM 16

---
 .github/workflows/nightly.yml                 |  4 +--
 docker/packager/packager                      | 19 +++++------
 docker/test/fasttest/run.sh                   |  2 +-
 docker/test/fuzzer/run-fuzzer.sh              |  2 +-
 docker/test/keeper-jepsen/run.sh              |  2 +-
 docker/test/server-jepsen/run.sh              |  2 +-
 docker/test/util/Dockerfile                   |  2 +-
 docs/en/development/build.md                  |  5 ++-
 docs/en/development/continuous-integration.md |  2 +-
 docs/en/development/developer-instruction.md  |  2 +-
 tests/ci/ci_config.py                         | 34 +++++++++----------
 11 files changed, 37 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index f6d6d192f48..b42052f91fe 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -123,8 +123,8 @@ jobs:
       SONAR_SCANNER_VERSION: 4.7.0.2747
       SONAR_SERVER_URL: "https://sonarcloud.io"
       BUILD_WRAPPER_OUT_DIR: build_wrapper_output_directory # Directory where build-wrapper output will be placed
-      CC: clang-15
-      CXX: clang++-15
+      CC: clang-16
+      CXX: clang++-16
     steps:
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
diff --git a/docker/packager/packager b/docker/packager/packager
index 58dd299fd6d..35d08231c26 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -341,17 +341,16 @@ if __name__ == "__main__":
     parser.add_argument(
         "--compiler",
         choices=(
-            "clang-15",
-            "clang-15-darwin",
-            "clang-15-darwin-aarch64",
-            "clang-15-aarch64",
-            "clang-15-aarch64-v80compat",
-            "clang-15-ppc64le",
-            "clang-15-amd64-compat",
-            "clang-15-freebsd",
-            "gcc-11",
+            "clang-16",
+            "clang-16-darwin",
+            "clang-16-darwin-aarch64",
+            "clang-16-aarch64",
+            "clang-16-aarch64-v80compat",
+            "clang-16-ppc64le",
+            "clang-16-amd64-compat",
+            "clang-16-freebsd",
         ),
-        default="clang-15",
+        default="clang-16",
         help="a compiler to use",
     )
     parser.add_argument(
diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index 086276bed55..d3bc3487035 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -9,7 +9,7 @@ trap 'kill $(jobs -pr) ||:' EXIT
 stage=${stage:-}
 
 # Compiler version, normally set by Dockerfile
-export LLVM_VERSION=${LLVM_VERSION:-13}
+export LLVM_VERSION=${LLVM_VERSION:-16}
 
 # A variable to pass additional flags to CMake.
 # Here we explicitly default it to nothing so that bash doesn't complain about
diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index 75f2a0af358..d2c8de7a211 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -15,7 +15,7 @@ stage=${stage:-}
 script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 echo "$script_dir"
 repo_dir=ch
-BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-15_debug_none_unsplitted_disable_False_binary"}
+BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-16_debug_none_unsplitted_disable_False_binary"}
 BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"}
 
 function git_clone_with_retry
diff --git a/docker/test/keeper-jepsen/run.sh b/docker/test/keeper-jepsen/run.sh
index 5e321b7c347..694d7fcd916 100644
--- a/docker/test/keeper-jepsen/run.sh
+++ b/docker/test/keeper-jepsen/run.sh
@@ -2,7 +2,7 @@
 set -euo pipefail
 
 
-CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-15_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"}
+CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-16_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"}
 CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""}
 
 
diff --git a/docker/test/server-jepsen/run.sh b/docker/test/server-jepsen/run.sh
index 4a966d50f74..cea4da27866 100644
--- a/docker/test/server-jepsen/run.sh
+++ b/docker/test/server-jepsen/run.sh
@@ -2,7 +2,7 @@
 set -euo pipefail
 
 
-CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-15_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"}
+CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-16_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"}
 CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""}
 
 
diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile
index 0ee426f4e4d..9af2b8c1d2a 100644
--- a/docker/test/util/Dockerfile
+++ b/docker/test/util/Dockerfile
@@ -6,7 +6,7 @@ ARG apt_archive="http://archive.ubuntu.com"
 RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
 
 # 15.0.2
-ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=15
+ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=16
 
 RUN apt-get update \
     && apt-get install \
diff --git a/docs/en/development/build.md b/docs/en/development/build.md
index d52b018a5a7..40994214316 100644
--- a/docs/en/development/build.md
+++ b/docs/en/development/build.md
@@ -45,8 +45,8 @@ For other Linux distribution - check the availability of the [prebuild packages]
 #### Use the latest clang for Builds
 
 ``` bash
-export CC=clang-15
-export CXX=clang++-15
+export CC=clang-16
+export CXX=clang++-16
 ```
 
 In this example we use version 15 that is the latest as of Sept 2022.
@@ -159,4 +159,3 @@ The CI checks build the binaries on each commit to [ClickHouse](https://github.c
 1. Find the type of package for your operating system that you need and download the files.
 
 ![build artifact check](images/find-build-artifact.png)
-
diff --git a/docs/en/development/continuous-integration.md b/docs/en/development/continuous-integration.md
index 232eee5b3cf..738c5458cc3 100644
--- a/docs/en/development/continuous-integration.md
+++ b/docs/en/development/continuous-integration.md
@@ -102,7 +102,7 @@ Builds ClickHouse in various configurations for use in further steps. You have t
 
 ### Report Details
 
-- **Compiler**: `clang-15`, optionally with the name of a target platform
+- **Compiler**: `clang-16`, optionally with the name of a target platform
 - **Build type**: `Debug` or `RelWithDebInfo` (cmake).
 - **Sanitizer**: `none` (without sanitizers), `address` (ASan), `memory` (MSan), `undefined` (UBSan), or `thread` (TSan).
 - **Status**: `success` or `fail`
diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md
index ace5ab79bb4..536abcb7130 100644
--- a/docs/en/development/developer-instruction.md
+++ b/docs/en/development/developer-instruction.md
@@ -146,7 +146,7 @@ While inside the `build` directory, configure your build by running CMake. Befor
     export CC=clang CXX=clang++
     cmake ..
 
-If you installed clang using the automatic installation script above, also specify the version of clang installed in the first command, e.g. `export CC=clang-15 CXX=clang++-15`. The clang version will be in the script output.
+If you installed clang using the automatic installation script above, also specify the version of clang installed in the first command, e.g. `export CC=clang-16 CXX=clang++-16`. The clang version will be in the script output.
 
 The `CC` variable specifies the compiler for C (short for C Compiler), and `CXX` variable instructs which C++ compiler is to be used for building.
 
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index 2f35b337cb3..89fde058507 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -8,7 +8,7 @@ BuildConfig = Dict[str, ConfValue]
 CI_CONFIG = {
     "build_config": {
         "package_release": {
-            "compiler": "clang-15",
+            "compiler": "clang-16",
             "build_type": "",
             "sanitizer": "",
             "package_type": "deb",
@@ -18,7 +18,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "coverity": {
-            "compiler": "clang-15",
+            "compiler": "clang-16",
             "build_type": "",
             "sanitizer": "",
             "package_type": "coverity",
@@ -27,7 +27,7 @@ CI_CONFIG = {
             "official": False,
         },
         "package_aarch64": {
-            "compiler": "clang-15-aarch64",
+            "compiler": "clang-16-aarch64",
             "build_type": "",
             "sanitizer": "",
             "package_type": "deb",
@@ -37,7 +37,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "package_asan": {
-            "compiler": "clang-15",
+            "compiler": "clang-16",
             "build_type": "",
             "sanitizer": "address",
             "package_type": "deb",
@@ -45,7 +45,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "package_ubsan": {
-            "compiler": "clang-15",
+            "compiler": "clang-16",
             "build_type": "",
             "sanitizer": "undefined",
             "package_type": "deb",
@@ -53,7 +53,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "package_tsan": {
-            "compiler": "clang-15",
+            "compiler": "clang-16",
             "build_type": "",
             "sanitizer": "thread",
             "package_type": "deb",
@@ -61,7 +61,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "package_msan": {
-            "compiler": "clang-15",
+            "compiler": "clang-16",
             "build_type": "",
             "sanitizer": "memory",
             "package_type": "deb",
@@ -69,7 +69,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "package_debug": {
-            "compiler": "clang-15",
+            "compiler": "clang-16",
             "build_type": "debug",
             "sanitizer": "",
             "package_type": "deb",
@@ -77,7 +77,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "binary_release": {
-            "compiler": "clang-15",
+            "compiler": "clang-16",
             "build_type": "",
             "sanitizer": "",
             "package_type": "binary",
@@ -85,7 +85,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "binary_tidy": {
-            "compiler": "clang-15",
+            "compiler": "clang-16",
             "build_type": "debug",
             "sanitizer": "",
             "package_type": "binary",
@@ -94,7 +94,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "binary_darwin": {
-            "compiler": "clang-15-darwin",
+            "compiler": "clang-16-darwin",
             "build_type": "",
             "sanitizer": "",
             "package_type": "binary",
@@ -103,7 +103,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "binary_aarch64": {
-            "compiler": "clang-15-aarch64",
+            "compiler": "clang-16-aarch64",
             "build_type": "",
             "sanitizer": "",
             "package_type": "binary",
@@ -111,7 +111,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "binary_aarch64_v80compat": {
-            "compiler": "clang-15-aarch64-v80compat",
+            "compiler": "clang-16-aarch64-v80compat",
             "build_type": "",
             "sanitizer": "",
             "package_type": "binary",
@@ -120,7 +120,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "binary_freebsd": {
-            "compiler": "clang-15-freebsd",
+            "compiler": "clang-16-freebsd",
             "build_type": "",
             "sanitizer": "",
             "package_type": "binary",
@@ -129,7 +129,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "binary_darwin_aarch64": {
-            "compiler": "clang-15-darwin-aarch64",
+            "compiler": "clang-16-darwin-aarch64",
             "build_type": "",
             "sanitizer": "",
             "package_type": "binary",
@@ -138,7 +138,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "binary_ppc64le": {
-            "compiler": "clang-15-ppc64le",
+            "compiler": "clang-16-ppc64le",
             "build_type": "",
             "sanitizer": "",
             "package_type": "binary",
@@ -147,7 +147,7 @@ CI_CONFIG = {
             "with_coverage": False,
         },
         "binary_amd64_compat": {
-            "compiler": "clang-15-amd64-compat",
+            "compiler": "clang-16-amd64-compat",
             "build_type": "",
             "sanitizer": "",
             "package_type": "binary",

From 0f7f6201003d317c5e8bf33ea8e6a8f6b5cc59fc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 19 Mar 2023 01:07:11 +0100
Subject: [PATCH 006/689] Update one more place

---
 docker/test/codebrowser/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile
index b76b8234c81..804a665cfa8 100644
--- a/docker/test/codebrowser/Dockerfile
+++ b/docker/test/codebrowser/Dockerfile
@@ -31,7 +31,7 @@ RUN arch=${TARGETARCH:-amd64} \
         arm64) rarch=aarch64 ;; \
         *) exit 1 ;; \
     esac \
-    && ln -rsf /usr/lib/$rarch-linux-gnu/libclang-15.so.15 /usr/lib/$rarch-linux-gnu/libclang-15.so.1
+    && ln -rsf /usr/lib/$rarch-linux-gnu/libclang-16.so.15 /usr/lib/$rarch-linux-gnu/libclang-16.so.1
 
 # repo versions doesn't work correctly with C++17
 # also we push reports to s3, so we add index.html to subfolder urls

From 0f039f83590339ab636e9b2e59c25ed22227d246 Mon Sep 17 00:00:00 2001
From: Kuba Kaflik <kuba.kaflik@clickhouse.com>
Date: Tue, 21 Mar 2023 09:09:57 +0100
Subject: [PATCH 007/689] Add Google Cloud Storage S3 compatible table function

---
 src/TableFunctions/TableFunctionS3.h | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h
index 859da9e9201..a2e93476448 100644
--- a/src/TableFunctions/TableFunctionS3.h
+++ b/src/TableFunctions/TableFunctionS3.h
@@ -93,4 +93,18 @@ private:
 
 }
 
+class TableFunctionGCS : public TableFunctionS3
+{
+public:
+    static constexpr auto name = "gcs";
+    std::string getName() const override
+    {
+        return name;
+    }
+private:
+    const char * getStorageTypeName() const override { return "GCS"; }
+};
+
+}
+
 #endif

From e6ddfc3486985393040222aec24ea70a4c60e7b8 Mon Sep 17 00:00:00 2001
From: Kuba Kaflik <kuba.kaflik@clickhouse.com>
Date: Tue, 21 Mar 2023 09:51:37 +0100
Subject: [PATCH 008/689] Update GCS table function docs

---
 docs/en/sql-reference/table-functions/gcs.md | 184 +++++++++++++++++++
 src/TableFunctions/TableFunctionS3.h         |   8 +
 2 files changed, 192 insertions(+)
 create mode 100644 docs/en/sql-reference/table-functions/gcs.md

diff --git a/docs/en/sql-reference/table-functions/gcs.md b/docs/en/sql-reference/table-functions/gcs.md
new file mode 100644
index 00000000000..8427a2db224
--- /dev/null
+++ b/docs/en/sql-reference/table-functions/gcs.md
@@ -0,0 +1,184 @@
+---
+slug: /en/sql-reference/table-functions/gcs
+sidebar_position: 45
+sidebar_label: s3
+keywords: [gcs, bucket]
+---
+
+# gcs Table Function
+
+Provides a table-like interface to select/insert files in [Google Cloud Storage](https://cloud.google.com/storage/).
+
+**Syntax**
+
+``` sql
+gcs(path [,hmac_key, hmac_secret] [,format] [,structure] [,compression])
+```
+
+:::tip GCS
+The GCS Table Function integrates with Google Cloud Storage by using the GCS XML API and HMAC keys. See the [Google interoperability docs]( https://cloud.google.com/storage/docs/interoperability) for more details about the endpoint and HMAC. 
+
+:::
+
+**Arguments**
+
+-   `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [here](../../engines/table-engines/integrations/gcs.md#wildcards-in-path).
+
+  :::note GCS
+  The GCS path is in this format as the endpoint for the Google XML API is different than the JSON API:
+  ```
+  https://storage.googleapis.com/<bucket>/<folder>/<filename(s)>
+  ```
+  and not ~~https://storage.cloud.google.com~~.
+  :::
+
+-   `format` — The [format](../../interfaces/formats.md#formats) of the file.
+-   `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`.
+-   `compression` — Parameter is optional. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. By default, it will autodetect compression by file extension.
+
+**Returned value**
+
+A table with the specified structure for reading or writing data in the specified file.
+
+**Examples**
+
+Selecting the first two rows from the table from S3 file `https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv`:
+
+``` sql
+SELECT *
+FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32')
+LIMIT 2;
+```
+
+``` text
+┌─column1─┬─column2─┬─column3─┐
+│       1 │       2 │       3 │
+│       3 │       2 │       1 │
+└─────────┴─────────┴─────────┘
+```
+
+The similar but from file with `gzip` compression:
+
+``` sql
+SELECT *
+FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv.gz', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32', 'gzip')
+LIMIT 2;
+```
+
+``` text
+┌─column1─┬─column2─┬─column3─┐
+│       1 │       2 │       3 │
+│       3 │       2 │       1 │
+└─────────┴─────────┴─────────┘
+```
+
+## Usage
+
+Suppose that we have several files with following URIs on S3:
+
+-   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_1.csv'
+-   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_2.csv'
+-   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_3.csv'
+-   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_4.csv'
+-   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_1.csv'
+-   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_2.csv'
+-   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_3.csv'
+-   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_4.csv'
+
+Count the amount of rows in files ending with numbers from 1 to 3:
+
+``` sql
+SELECT count(*)
+FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}.csv', 'CSV', 'name String, value UInt32')
+```
+
+``` text
+┌─count()─┐
+│      18 │
+└─────────┘
+```
+
+Count the total amount of rows in all files in these two directories:
+
+``` sql
+SELECT count(*)
+FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/{some,another}_prefix/*', 'CSV', 'name String, value UInt32')
+```
+
+``` text
+┌─count()─┐
+│      24 │
+└─────────┘
+```
+
+:::warning
+If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`.
+:::
+
+Count the total amount of rows in files named `file-000.csv`, `file-001.csv`, … , `file-999.csv`:
+
+``` sql
+SELECT count(*)
+FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV', 'name String, value UInt32');
+```
+
+``` text
+┌─count()─┐
+│      12 │
+└─────────┘
+```
+
+Insert data into file `test-data.csv.gz`:
+
+``` sql
+INSERT INTO FUNCTION gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip')
+VALUES ('test-data', 1), ('test-data-2', 2);
+```
+
+Insert data into file `test-data.csv.gz` from existing table:
+
+``` sql
+INSERT INTO FUNCTION gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip')
+SELECT name, value FROM existing_table;
+```
+
+Glob ** can be used for recursive directory traversal. Consider the below example, it will fetch all files from `my-test-bucket-768` directory recursively:
+
+``` sql
+SELECT * FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/**', 'CSV', 'name String, value UInt32', 'gzip');
+```
+
+The below get data from all `test-data.csv.gz` files from any folder inside `my-test-bucket` directory recursively:
+
+``` sql
+SELECT * FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/**/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip');
+```
+
+## Partitioned Write
+
+If you specify `PARTITION BY` expression when inserting data into `S3` table, a separate file is created for each partition value. Splitting the data into separate files helps to improve reading operations efficiency.
+
+**Examples**
+
+1. Using partition ID in a key creates separate files:
+
+```sql
+INSERT INTO TABLE FUNCTION
+    gcs('http://bucket.amazonaws.com/my_bucket/file_{_partition_id}.csv', 'CSV', 'a String, b UInt32, c UInt32')
+    PARTITION BY a VALUES ('x', 2, 3), ('x', 4, 5), ('y', 11, 12), ('y', 13, 14), ('z', 21, 22), ('z', 23, 24);
+```
+As a result, the data is written into three files: `file_x.csv`, `file_y.csv`, and `file_z.csv`.
+
+2. Using partition ID in a bucket name creates files in different buckets:
+
+```sql
+INSERT INTO TABLE FUNCTION
+    gcs('http://bucket.amazonaws.com/my_bucket_{_partition_id}/file.csv', 'CSV', 'a UInt32, b UInt32, c UInt32')
+    PARTITION BY a VALUES (1, 2, 3), (1, 4, 5), (10, 11, 12), (10, 13, 14), (20, 21, 22), (20, 23, 24);
+```
+As a result, the data is written into three files in different buckets: `my_bucket_1/file.csv`, `my_bucket_10/file.csv`, and `my_bucket_20/file.csv`.
+
+**See Also**
+
+-   [S3 table function](s3.md)
+-   [S3 engine](../../engines/table-engines/integrations/s3.md)
diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h
index a2e93476448..ed8cd3bd41a 100644
--- a/src/TableFunctions/TableFunctionS3.h
+++ b/src/TableFunctions/TableFunctionS3.h
@@ -97,6 +97,14 @@ class TableFunctionGCS : public TableFunctionS3
 {
 public:
     static constexpr auto name = "gcs";
+    static constexpr auto signature = " - url\n"
+                                      " - url, format\n"
+                                      " - url, format, structure\n"
+                                      " - url, hmac_key, hmac_secret\n"
+                                      " - url, format, structure, compression_method\n"
+                                      " - url, hmac_key, hmac_secret, format\n"
+                                      " - url, hmac_key, hmac_secret, format, structure\n"
+                                      " - url, hmac_key, hmac_secret, format, structure, compression_method";
     std::string getName() const override
     {
         return name;

From e2c32c3bc072e2290620d975ada42c37bcabcc52 Mon Sep 17 00:00:00 2001
From: Kuba Kaflik <kuba.kaflik@clickhouse.com>
Date: Tue, 21 Mar 2023 13:46:37 +0100
Subject: [PATCH 009/689] Update GCS table function docs

---
 docs/en/sql-reference/table-functions/gcs.md | 40 ++++++++++----------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/docs/en/sql-reference/table-functions/gcs.md b/docs/en/sql-reference/table-functions/gcs.md
index 8427a2db224..dcf49a5108b 100644
--- a/docs/en/sql-reference/table-functions/gcs.md
+++ b/docs/en/sql-reference/table-functions/gcs.md
@@ -42,11 +42,11 @@ A table with the specified structure for reading or writing data in the specifie
 
 **Examples**
 
-Selecting the first two rows from the table from S3 file `https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv`:
+Selecting the first two rows from the table from GCS file `https://storage.googleapis.com/my-test-bucket-768/data.csv`:
 
 ``` sql
 SELECT *
-FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32')
+FROM gcs('https://storage.googleapis.com/my-test-bucket-768/data.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32')
 LIMIT 2;
 ```
 
@@ -61,7 +61,7 @@ The similar but from file with `gzip` compression:
 
 ``` sql
 SELECT *
-FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv.gz', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32', 'gzip')
+FROM gcs('https://storage.googleapis.com/my-test-bucket-768/data.csv.gz', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32', 'gzip')
 LIMIT 2;
 ```
 
@@ -74,22 +74,22 @@ LIMIT 2;
 
 ## Usage
 
-Suppose that we have several files with following URIs on S3:
+Suppose that we have several files with following URIs on GCS:
 
--   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_1.csv'
--   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_2.csv'
--   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_3.csv'
--   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_4.csv'
--   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_1.csv'
--   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_2.csv'
--   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_3.csv'
--   'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_4.csv'
+-   'https://storage.googleapis.com/my-test-bucket-768/some_prefix/some_file_1.csv'
+-   'https://storage.googleapis.com/my-test-bucket-768/some_prefix/some_file_2.csv'
+-   'https://storage.googleapis.com/my-test-bucket-768/some_prefix/some_file_3.csv'
+-   'https://storage.googleapis.com/my-test-bucket-768/some_prefix/some_file_4.csv'
+-   'https://storage.googleapis.com/my-test-bucket-768/another_prefix/some_file_1.csv'
+-   'https://storage.googleapis.com/my-test-bucket-768/another_prefix/some_file_2.csv'
+-   'https://storage.googleapis.com/my-test-bucket-768/another_prefix/some_file_3.csv'
+-   'https://storage.googleapis.com/my-test-bucket-768/another_prefix/some_file_4.csv'
 
 Count the amount of rows in files ending with numbers from 1 to 3:
 
 ``` sql
 SELECT count(*)
-FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}.csv', 'CSV', 'name String, value UInt32')
+FROM gcs('https://storage.googleapis.com/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}.csv', 'CSV', 'name String, value UInt32')
 ```
 
 ``` text
@@ -102,7 +102,7 @@ Count the total amount of rows in all files in these two directories:
 
 ``` sql
 SELECT count(*)
-FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/{some,another}_prefix/*', 'CSV', 'name String, value UInt32')
+FROM gcs('https://storage.googleapis.com/my-test-bucket-768/{some,another}_prefix/*', 'CSV', 'name String, value UInt32')
 ```
 
 ``` text
@@ -119,7 +119,7 @@ Count the total amount of rows in files named `file-000.csv`, `file-001.csv`, 
 
 ``` sql
 SELECT count(*)
-FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV', 'name String, value UInt32');
+FROM gcs('https://storage.googleapis.com/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV', 'name String, value UInt32');
 ```
 
 ``` text
@@ -131,32 +131,32 @@ FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768
 Insert data into file `test-data.csv.gz`:
 
 ``` sql
-INSERT INTO FUNCTION gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip')
+INSERT INTO FUNCTION gcs('https://storage.googleapis.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip')
 VALUES ('test-data', 1), ('test-data-2', 2);
 ```
 
 Insert data into file `test-data.csv.gz` from existing table:
 
 ``` sql
-INSERT INTO FUNCTION gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip')
+INSERT INTO FUNCTION gcs('https://storage.googleapis.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip')
 SELECT name, value FROM existing_table;
 ```
 
 Glob ** can be used for recursive directory traversal. Consider the below example, it will fetch all files from `my-test-bucket-768` directory recursively:
 
 ``` sql
-SELECT * FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/**', 'CSV', 'name String, value UInt32', 'gzip');
+SELECT * FROM gcs('https://storage.googleapis.com/my-test-bucket-768/**', 'CSV', 'name String, value UInt32', 'gzip');
 ```
 
 The below get data from all `test-data.csv.gz` files from any folder inside `my-test-bucket` directory recursively:
 
 ``` sql
-SELECT * FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/**/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip');
+SELECT * FROM gcs('https://storage.googleapis.com/my-test-bucket-768/**/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip');
 ```
 
 ## Partitioned Write
 
-If you specify `PARTITION BY` expression when inserting data into `S3` table, a separate file is created for each partition value. Splitting the data into separate files helps to improve reading operations efficiency.
+If you specify `PARTITION BY` expression when inserting data into `GCS` table, a separate file is created for each partition value. Splitting the data into separate files helps to improve reading operations efficiency.
 
 **Examples**
 

From d0a54ab21b2107ee6893a7480533c79c2919fd75 Mon Sep 17 00:00:00 2001
From: Kuba Kaflik <kuba.kaflik@clickhouse.com>
Date: Tue, 21 Mar 2023 14:45:58 +0100
Subject: [PATCH 010/689] Update GCS table function docs

---
 docs/en/sql-reference/table-functions/gcs.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/table-functions/gcs.md b/docs/en/sql-reference/table-functions/gcs.md
index dcf49a5108b..bfa7f36fa48 100644
--- a/docs/en/sql-reference/table-functions/gcs.md
+++ b/docs/en/sql-reference/table-functions/gcs.md
@@ -22,7 +22,7 @@ The GCS Table Function integrates with Google Cloud Storage by using the GCS XML
 
 **Arguments**
 
--   `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [here](../../engines/table-engines/integrations/gcs.md#wildcards-in-path).
+-   `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings.
 
   :::note GCS
   The GCS path is in this format as the endpoint for the Google XML API is different than the JSON API:

From 576efc1da3384148664202acef1cdbd27ddc8a08 Mon Sep 17 00:00:00 2001
From: Kuba Kaflik <kuba.kaflik@clickhouse.com>
Date: Wed, 22 Mar 2023 06:58:09 +0100
Subject: [PATCH 011/689] register GCP function in factory

---
 src/TableFunctions/TableFunctionS3.cpp        | 5 +++++
 src/TableFunctions/registerTableFunctions.cpp | 1 +
 src/TableFunctions/registerTableFunctions.h   | 1 +
 3 files changed, 7 insertions(+)

diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp
index f082b192ee0..6f4e6acec8a 100644
--- a/src/TableFunctions/TableFunctionS3.cpp
+++ b/src/TableFunctions/TableFunctionS3.cpp
@@ -183,6 +183,11 @@ void registerTableFunctionOSS(TableFunctionFactory & factory)
     factory.registerFunction<TableFunctionOSS>();
 }
 
+void registerTableFunctionGCS(TableFunctionFactory & factory)
+{
+    factory.registerFunction<TableFunctionGCS>();
+}
+
 }
 
 #endif
diff --git a/src/TableFunctions/registerTableFunctions.cpp b/src/TableFunctions/registerTableFunctions.cpp
index 7b2b989e724..c692173e689 100644
--- a/src/TableFunctions/registerTableFunctions.cpp
+++ b/src/TableFunctions/registerTableFunctions.cpp
@@ -28,6 +28,7 @@ void registerTableFunctions()
     registerTableFunctionS3Cluster(factory);
     registerTableFunctionCOS(factory);
     registerTableFunctionOSS(factory);
+    registerTableFunctionGCS(factory);
     registerTableFunctionHudi(factory);
     registerTableFunctionDeltaLake(factory);
 #if USE_AVRO
diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h
index 911aae199e2..af1b7129ec4 100644
--- a/src/TableFunctions/registerTableFunctions.h
+++ b/src/TableFunctions/registerTableFunctions.h
@@ -25,6 +25,7 @@ void registerTableFunctionS3(TableFunctionFactory & factory);
 void registerTableFunctionS3Cluster(TableFunctionFactory & factory);
 void registerTableFunctionCOS(TableFunctionFactory & factory);
 void registerTableFunctionOSS(TableFunctionFactory & factory);
+void registerTableFunctionGCS(TableFunctionFactory & factory);
 void registerTableFunctionHudi(TableFunctionFactory & factory);
 void registerTableFunctionDeltaLake(TableFunctionFactory & factory);
 #if USE_AVRO

From d8e8dc39e8b929ce0299f96a1be4065a74b45b99 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 3 Apr 2023 01:29:55 +0200
Subject: [PATCH 012/689] Fix test

---
 .../test_vertical_merges_from_compact_parts.py                | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
index 1781ed7c976..6fdf0942ee4 100644
--- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
+++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
@@ -41,7 +41,9 @@ def test_vertical_merges_from_comapact_parts(start_cluster):
                 vertical_merge_algorithm_min_rows_to_activate = 1,
                 vertical_merge_algorithm_min_columns_to_activate = 1,
                 min_bytes_for_wide_part = 0,
-                min_rows_for_wide_part = 100;
+                min_rows_for_wide_part = 100,
+                compress_marks = 0,
+                compress_primary_key = 0;
         """.format(
                 i
             )

From 01728d6d391c07e40c42cbd50151be5de472a62e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 4 Apr 2023 01:05:06 +0200
Subject: [PATCH 013/689] Fix tests

---
 .../configs/no_compress_marks.xml                           | 6 ++++++
 .../test_backward_compatibility/configs/wide_parts_only.xml | 4 +---
 tests/integration/test_backward_compatibility/test.py       | 2 +-
 .../test_vertical_merges_from_compact_parts.py              | 5 ++---
 4 files changed, 10 insertions(+), 7 deletions(-)
 create mode 100644 tests/integration/test_backward_compatibility/configs/no_compress_marks.xml

diff --git a/tests/integration/test_backward_compatibility/configs/no_compress_marks.xml b/tests/integration/test_backward_compatibility/configs/no_compress_marks.xml
new file mode 100644
index 00000000000..cc968525bbb
--- /dev/null
+++ b/tests/integration/test_backward_compatibility/configs/no_compress_marks.xml
@@ -0,0 +1,6 @@
+<clickhouse>
+    <merge_tree>
+        <compress_marks>0</compress_marks>
+        <compress_primary_key>0</compress_primary_key>
+    </merge_tree>
+</clickhouse>
diff --git a/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml b/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml
index c823dd02d5a..e9cf053f1c5 100644
--- a/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml
+++ b/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml
@@ -1,7 +1,5 @@
 <clickhouse>
     <merge_tree>
         <min_bytes_for_wide_part>0</min_bytes_for_wide_part>
-        <compress_marks>0</compress_marks>
-        <compress_primary_key>0</compress_primary_key>
-    </merge_tree>    
+    </merge_tree>
 </clickhouse>
diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py
index 01ed02720f8..e4ee6aa2de9 100644
--- a/tests/integration/test_backward_compatibility/test.py
+++ b/tests/integration/test_backward_compatibility/test.py
@@ -12,7 +12,7 @@ node1 = cluster.add_instance(
     with_installed_binary=True,
 )
 node2 = cluster.add_instance(
-    "node2", main_configs=["configs/wide_parts_only.xml"], with_zookeeper=True
+    "node2", main_configs=["configs/wide_parts_only.xml", "configs/no_compress_marks.xml"], with_zookeeper=True
 )
 
 
diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
index 6fdf0942ee4..4b144a37ce9 100644
--- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
+++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
@@ -14,6 +14,7 @@ node_old = cluster.add_instance(
 )
 node_new = cluster.add_instance(
     "node2",
+    main_configs=["configs/no_compress_marks.xml"],
     with_zookeeper=True,
     stay_alive=True,
 )
@@ -41,9 +42,7 @@ def test_vertical_merges_from_comapact_parts(start_cluster):
                 vertical_merge_algorithm_min_rows_to_activate = 1,
                 vertical_merge_algorithm_min_columns_to_activate = 1,
                 min_bytes_for_wide_part = 0,
-                min_rows_for_wide_part = 100,
-                compress_marks = 0,
-                compress_primary_key = 0;
+                min_rows_for_wide_part = 100;
         """.format(
                 i
             )

From 806c7a5f9d1a6869cc464f79b6bfd83028874540 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Mon, 3 Apr 2023 23:13:50 +0000
Subject: [PATCH 014/689] Automatic style fix

---
 tests/integration/test_backward_compatibility/test.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py
index e4ee6aa2de9..ea1d3ab9c07 100644
--- a/tests/integration/test_backward_compatibility/test.py
+++ b/tests/integration/test_backward_compatibility/test.py
@@ -12,7 +12,9 @@ node1 = cluster.add_instance(
     with_installed_binary=True,
 )
 node2 = cluster.add_instance(
-    "node2", main_configs=["configs/wide_parts_only.xml", "configs/no_compress_marks.xml"], with_zookeeper=True
+    "node2",
+    main_configs=["configs/wide_parts_only.xml", "configs/no_compress_marks.xml"],
+    with_zookeeper=True,
 )
 
 

From 49c95a535ab5982a03b3dea731692893ed559806 Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Wed, 12 Apr 2023 20:26:57 +0200
Subject: [PATCH 015/689] Updated to add error or completed status in zookeeper
 for a cluster for backup/restore, to avoid interpreting previously failed
 backup/restore when zookeeper is unable to remove nodes

---
 src/Backups/BackupCoordinationLocal.cpp     |  2 +-
 src/Backups/BackupCoordinationLocal.h       |  2 +-
 src/Backups/BackupCoordinationRemote.cpp    | 12 ++++++----
 src/Backups/BackupCoordinationRemote.h      |  2 +-
 src/Backups/BackupCoordinationStage.h       |  4 ++++
 src/Backups/BackupCoordinationStageSync.cpp | 14 ++++++++++++
 src/Backups/BackupCoordinationStageSync.h   |  1 +
 src/Backups/BackupsWorker.cpp               | 25 +++++++++++++--------
 src/Backups/IBackupCoordination.h           |  2 +-
 src/Backups/IRestoreCoordination.h          |  2 +-
 src/Backups/RestoreCoordinationLocal.cpp    |  2 +-
 src/Backups/RestoreCoordinationLocal.h      |  2 +-
 src/Backups/RestoreCoordinationRemote.cpp   | 12 ++++++----
 src/Backups/RestoreCoordinationRemote.h     |  2 +-
 14 files changed, 59 insertions(+), 25 deletions(-)

diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp
index 27e0f173cf3..47b67693114 100644
--- a/src/Backups/BackupCoordinationLocal.cpp
+++ b/src/Backups/BackupCoordinationLocal.cpp
@@ -15,7 +15,7 @@ BackupCoordinationLocal::BackupCoordinationLocal(bool plain_backup_)
 
 BackupCoordinationLocal::~BackupCoordinationLocal() = default;
 
-void BackupCoordinationLocal::setStage(const String &, const String &)
+void BackupCoordinationLocal::setStage(const String &, const String &, const bool &)
 {
 }
 
diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h
index 60fcc014720..1f6bb84972e 100644
--- a/src/Backups/BackupCoordinationLocal.h
+++ b/src/Backups/BackupCoordinationLocal.h
@@ -22,7 +22,7 @@ public:
     BackupCoordinationLocal(bool plain_backup_);
     ~BackupCoordinationLocal() override;
 
-    void setStage(const String & new_stage, const String & message) override;
+    void setStage(const String & new_stage, const String & message, const bool & for_cluster = false) override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp
index 8e6b5db91b1..48f1ce3eef7 100644
--- a/src/Backups/BackupCoordinationRemote.cpp
+++ b/src/Backups/BackupCoordinationRemote.cpp
@@ -252,13 +252,17 @@ void BackupCoordinationRemote::removeAllNodes()
 }
 
 
-void BackupCoordinationRemote::setStage(const String & new_stage, const String & message)
+void BackupCoordinationRemote::setStage(const String & new_stage, const String & message, const bool & for_cluster)
 {
-    stage_sync->set(current_host, new_stage, message);
+    if (for_cluster)
+        stage_sync->setStageForCluster(new_stage);
+    else
+        stage_sync->set(current_host, new_stage, message);
 }
 
 void BackupCoordinationRemote::setError(const Exception & exception)
 {
+    stage_sync->setStageForCluster(Stage::ERROR);
     stage_sync->setError(current_host, exception);
 }
 
@@ -777,8 +781,8 @@ bool BackupCoordinationRemote::hasConcurrentBackups(const std::atomic<size_t> &)
                 String status;
                 if (zk->tryGet(root_zookeeper_path + "/" + existing_backup_path + "/stage", status))
                 {
-                    /// If status is not COMPLETED it could be because the backup failed, check if 'error' exists
-                    if (status != Stage::COMPLETED && !zk->exists(root_zookeeper_path + "/" + existing_backup_path + "/error"))
+                    /// Check if some other restore is in progress
+                    if (status == Stage::SCHEDULED_TO_START)
                     {
                         LOG_WARNING(log, "Found a concurrent backup: {}, current backup: {}", existing_backup_uuid, toString(backup_uuid));
                         result = true;
diff --git a/src/Backups/BackupCoordinationRemote.h b/src/Backups/BackupCoordinationRemote.h
index 949dd9c9bf0..40ce2ae6ccc 100644
--- a/src/Backups/BackupCoordinationRemote.h
+++ b/src/Backups/BackupCoordinationRemote.h
@@ -33,7 +33,7 @@ public:
 
     ~BackupCoordinationRemote() override;
 
-    void setStage(const String & new_stage, const String & message) override;
+    void setStage(const String & new_stage, const String & message, const bool & for_cluster = false) override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
diff --git a/src/Backups/BackupCoordinationStage.h b/src/Backups/BackupCoordinationStage.h
index 40a4b262caa..41cd66346a2 100644
--- a/src/Backups/BackupCoordinationStage.h
+++ b/src/Backups/BackupCoordinationStage.h
@@ -43,6 +43,10 @@ namespace BackupCoordinationStage
 
     /// Coordination stage meaning that a host finished its work.
     constexpr const char * COMPLETED = "completed";
+
+    /// Coordination stage meaning that backup/restore has failed due to an error
+    /// Check '/error' for the error message
+    constexpr const char * ERROR = "error";
 }
 
 }
diff --git a/src/Backups/BackupCoordinationStageSync.cpp b/src/Backups/BackupCoordinationStageSync.cpp
index effb00085c3..5cbeec0ec76 100644
--- a/src/Backups/BackupCoordinationStageSync.cpp
+++ b/src/Backups/BackupCoordinationStageSync.cpp
@@ -61,6 +61,20 @@ void BackupCoordinationStageSync::set(const String & current_host, const String
     });
 }
 
+void BackupCoordinationStageSync::setStageForCluster(const String & new_stage)
+{
+    auto holder = with_retries.createRetriesControlHolder("setStageForCluster");
+    holder.retries_ctl.retryLoop(
+        [&, &zookeeper = holder.faulty_zookeeper]()
+        {
+            with_retries.renewZooKeeper(zookeeper);
+            zookeeper->trySet(zookeeper_path, new_stage);
+            auto code = zookeeper->trySet(zookeeper_path, new_stage);
+            if (code != Coordination::Error::ZOK)
+                throw zkutil::KeeperException(code, zookeeper_path);
+        });
+}
+
 void BackupCoordinationStageSync::setError(const String & current_host, const Exception & exception)
 {
     auto holder = with_retries.createRetriesControlHolder("setError");
diff --git a/src/Backups/BackupCoordinationStageSync.h b/src/Backups/BackupCoordinationStageSync.h
index 56081f8779c..9dde4e3095f 100644
--- a/src/Backups/BackupCoordinationStageSync.h
+++ b/src/Backups/BackupCoordinationStageSync.h
@@ -16,6 +16,7 @@ public:
 
     /// Sets the stage of the current host and signal other hosts if there were other hosts waiting for that.
     void set(const String & current_host, const String & new_stage, const String & message);
+    void setStageForCluster(const String & new_stage);
     void setError(const String & current_host, const Exception & exception);
 
     /// Sets the stage of the current host and waits until all hosts come to the same stage.
diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp
index 4b17174a8de..aae9cfd620f 100644
--- a/src/Backups/BackupsWorker.cpp
+++ b/src/Backups/BackupsWorker.cpp
@@ -368,6 +368,7 @@ void BackupsWorker::doBackup(
 
             /// Wait until all the hosts have written their backup entries.
             backup_coordination->waitForStage(Stage::COMPLETED);
+            backup_coordination->setStage(Stage::COMPLETED, /* message */ "", /* for_cluster */ true);
         }
         else
         {
@@ -654,12 +655,26 @@ void BackupsWorker::doRestore(
         /// (If this isn't ON CLUSTER query RestorerFromBackup will check access rights later.)
         ClusterPtr cluster;
         bool on_cluster = !restore_query->cluster.empty();
+
         if (on_cluster)
         {
             restore_query->cluster = context->getMacros()->expand(restore_query->cluster);
             cluster = context->getCluster(restore_query->cluster);
             restore_settings.cluster_host_ids = cluster->getHostIDs();
+        }
 
+        /// Make a restore coordination.
+        if (!restore_coordination)
+            restore_coordination = makeRestoreCoordination(context, restore_settings, /* remote= */ on_cluster);
+
+        if (!allow_concurrent_restores && restore_coordination->hasConcurrentRestores(std::ref(num_active_restores)))
+            throw Exception(
+                ErrorCodes::CONCURRENT_ACCESS_NOT_SUPPORTED,
+                "Concurrent restores not supported, turn on setting 'allow_concurrent_restores'");
+
+
+        if (on_cluster)
+        {
             /// We cannot just use access checking provided by the function executeDDLQueryOnCluster(): it would be incorrect
             /// because different replicas can contain different set of tables and so the required access rights can differ too.
             /// So the right way is pass through the entire cluster and check access for each host.
@@ -676,15 +691,6 @@ void BackupsWorker::doRestore(
             }
         }
 
-        /// Make a restore coordination.
-        if (!restore_coordination)
-            restore_coordination = makeRestoreCoordination(context, restore_settings, /* remote= */ on_cluster);
-
-        if (!allow_concurrent_restores && restore_coordination->hasConcurrentRestores(std::ref(num_active_restores)))
-            throw Exception(
-                ErrorCodes::CONCURRENT_ACCESS_NOT_SUPPORTED,
-                "Concurrent restores not supported, turn on setting 'allow_concurrent_restores'");
-
         /// Do RESTORE.
         if (on_cluster)
         {
@@ -703,6 +709,7 @@ void BackupsWorker::doRestore(
 
             /// Wait until all the hosts have written their backup entries.
             restore_coordination->waitForStage(Stage::COMPLETED);
+            restore_coordination->setStage(Stage::COMPLETED, /* message */ "", /* for_cluster */ true);
         }
         else
         {
diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h
index 75d9202374b..614e6a16db8 100644
--- a/src/Backups/IBackupCoordination.h
+++ b/src/Backups/IBackupCoordination.h
@@ -21,7 +21,7 @@ public:
     virtual ~IBackupCoordination() = default;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    virtual void setStage(const String & new_stage, const String & message) = 0;
+    virtual void setStage(const String & new_stage, const String & message, const bool & for_cluster = false) = 0;
     virtual void setError(const Exception & exception) = 0;
     virtual Strings waitForStage(const String & stage_to_wait) = 0;
     virtual Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) = 0;
diff --git a/src/Backups/IRestoreCoordination.h b/src/Backups/IRestoreCoordination.h
index 2f9e8d171f6..599a698a1f9 100644
--- a/src/Backups/IRestoreCoordination.h
+++ b/src/Backups/IRestoreCoordination.h
@@ -18,7 +18,7 @@ public:
     virtual ~IRestoreCoordination() = default;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    virtual void setStage(const String & new_stage, const String & message) = 0;
+    virtual void setStage(const String & new_stage, const String & message, const bool & for_cluster = false) = 0;
     virtual void setError(const Exception & exception) = 0;
     virtual Strings waitForStage(const String & stage_to_wait) = 0;
     virtual Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) = 0;
diff --git a/src/Backups/RestoreCoordinationLocal.cpp b/src/Backups/RestoreCoordinationLocal.cpp
index 068c4fe7e52..f689277f5b6 100644
--- a/src/Backups/RestoreCoordinationLocal.cpp
+++ b/src/Backups/RestoreCoordinationLocal.cpp
@@ -11,7 +11,7 @@ RestoreCoordinationLocal::RestoreCoordinationLocal() : log(&Poco::Logger::get("R
 
 RestoreCoordinationLocal::~RestoreCoordinationLocal() = default;
 
-void RestoreCoordinationLocal::setStage(const String &, const String &)
+void RestoreCoordinationLocal::setStage(const String &, const String &, const bool &)
 {
 }
 
diff --git a/src/Backups/RestoreCoordinationLocal.h b/src/Backups/RestoreCoordinationLocal.h
index e27f0d1ef88..4456ad966d4 100644
--- a/src/Backups/RestoreCoordinationLocal.h
+++ b/src/Backups/RestoreCoordinationLocal.h
@@ -19,7 +19,7 @@ public:
     ~RestoreCoordinationLocal() override;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    void setStage(const String & new_stage, const String & message) override;
+    void setStage(const String & new_stage, const String & message, const bool & for_cluster = false) override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp
index cc03f0c4a2a..0a89b1cd4e7 100644
--- a/src/Backups/RestoreCoordinationRemote.cpp
+++ b/src/Backups/RestoreCoordinationRemote.cpp
@@ -90,13 +90,17 @@ void RestoreCoordinationRemote::createRootNodes()
         });
 }
 
-void RestoreCoordinationRemote::setStage(const String & new_stage, const String & message)
+void RestoreCoordinationRemote::setStage(const String & new_stage, const String & message, const bool & for_cluster)
 {
-    stage_sync->set(current_host, new_stage, message);
+    if (for_cluster)
+        stage_sync->setStageForCluster(new_stage);
+    else
+        stage_sync->set(current_host, new_stage, message);
 }
 
 void RestoreCoordinationRemote::setError(const Exception & exception)
 {
+    stage_sync->setStageForCluster(Stage::ERROR);
     stage_sync->setError(current_host, exception);
 }
 
@@ -282,8 +286,8 @@ bool RestoreCoordinationRemote::hasConcurrentRestores(const std::atomic<size_t>
                     String status;
                     if (zk->tryGet(root_zookeeper_path + "/" + existing_restore_path + "/stage", status))
                     {
-                        /// If status is not COMPLETED it could be because the restore failed, check if 'error' exists
-                        if (status != Stage::COMPLETED && !zk->exists(root_zookeeper_path + "/" + existing_restore_path + "/error"))
+                        /// Check if some other restore is in progress
+                        if (status == Stage::SCHEDULED_TO_START)
                         {
                             LOG_WARNING(log, "Found a concurrent restore: {}, current restore: {}", existing_restore_uuid, toString(restore_uuid));
                             result = true;
diff --git a/src/Backups/RestoreCoordinationRemote.h b/src/Backups/RestoreCoordinationRemote.h
index eb0fcff9c2d..21a38f01fa6 100644
--- a/src/Backups/RestoreCoordinationRemote.h
+++ b/src/Backups/RestoreCoordinationRemote.h
@@ -26,7 +26,7 @@ public:
     ~RestoreCoordinationRemote() override;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    void setStage(const String & new_stage, const String & message) override;
+    void setStage(const String & new_stage, const String & message, const bool & for_cluster = false) override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;

From d4b2297e9fa53336cd4d05919a1048ad742018cd Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Thu, 13 Apr 2023 09:53:39 +0200
Subject: [PATCH 016/689] Fixed comment

---
 src/Backups/BackupCoordinationRemote.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp
index 48f1ce3eef7..cd4901eb5ae 100644
--- a/src/Backups/BackupCoordinationRemote.cpp
+++ b/src/Backups/BackupCoordinationRemote.cpp
@@ -781,7 +781,7 @@ bool BackupCoordinationRemote::hasConcurrentBackups(const std::atomic<size_t> &)
                 String status;
                 if (zk->tryGet(root_zookeeper_path + "/" + existing_backup_path + "/stage", status))
                 {
-                    /// Check if some other restore is in progress
+                    /// Check if some other backup is in progress
                     if (status == Stage::SCHEDULED_TO_START)
                     {
                         LOG_WARNING(log, "Found a concurrent backup: {}, current backup: {}", existing_backup_uuid, toString(backup_uuid));

From cf5d9a175ae7c61ad0a7293fd49a86a4f7e09f01 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 14 Apr 2023 16:34:19 +0200
Subject: [PATCH 017/689] Revert "Merge pull request #48760 from
 ClickHouse/revert-46089-background-memory-tracker"

This reverts commit a61ed332239e4a35af1b9cd31479560da6f08cca, reversing
changes made to 5f01b8a2b59f6f340c65bfbc4ed0dd655d997ff9.
---
 .../settings.md                               | 33 ++++++++++++++--
 programs/server/Server.cpp                    | 20 ++++++++++
 src/Common/CurrentMetrics.cpp                 |  1 +
 src/Common/MemoryTracker.cpp                  |  8 ++++
 src/Common/MemoryTracker.h                    | 19 ++++++++++
 src/Core/ServerSettings.h                     |  2 +
 src/Interpreters/ThreadStatusExt.cpp          |  1 +
 src/Storages/MergeTree/MergeList.cpp          |  5 +++
 src/Storages/MergeTree/MergeList.h            |  2 +
 src/Storages/StorageMergeTree.cpp             | 11 +++++-
 src/Storages/StorageReplicatedMergeTree.cpp   | 10 ++++-
 .../test_merges_memory_limit/__init__.py      |  0
 .../test_merges_memory_limit/test.py          | 38 +++++++++++++++++++
 13 files changed, 145 insertions(+), 5 deletions(-)
 create mode 100644 tests/integration/test_merges_memory_limit/__init__.py
 create mode 100644 tests/integration/test_merges_memory_limit/test.py

diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md
index 7c97d0ab640..a9f0cc276ff 100644
--- a/docs/en/operations/server-configuration-parameters/settings.md
+++ b/docs/en/operations/server-configuration-parameters/settings.md
@@ -1045,7 +1045,7 @@ Default value: `0`.
 
 ## background_pool_size {#background_pool_size}
 
-Sets the number of threads performing background merges and mutations for tables with MergeTree engines. This setting is also could be applied  at server startup from the `default` profile configuration for backward compatibility at the ClickHouse server start. You can only increase the number of threads at runtime. To lower the number of threads you have to restart the server. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance.
+Sets the number of threads performing background merges and mutations for tables with MergeTree engines. This setting is also could be applied at server startup from the `default` profile configuration for backward compatibility at the ClickHouse server start. You can only increase the number of threads at runtime. To lower the number of threads you have to restart the server. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance.
 
 Before changing it, please also take a look at related MergeTree settings, such as [number_of_free_entries_in_pool_to_lower_max_size_of_merge](../../operations/settings/merge-tree-settings.md#number-of-free-entries-in-pool-to-lower-max-size-of-merge) and [number_of_free_entries_in_pool_to_execute_mutation](../../operations/settings/merge-tree-settings.md#number-of-free-entries-in-pool-to-execute-mutation).
 
@@ -1063,8 +1063,8 @@ Default value: 16.
 
 ## background_merges_mutations_concurrency_ratio {#background_merges_mutations_concurrency_ratio}
 
-Sets a ratio between the number of threads and the number of background merges and mutations that can be executed concurrently. For example if the ratio equals to 2 and
-`background_pool_size` is set to 16 then ClickHouse can execute 32 background merges concurrently. This is possible, because background operation could be suspended and postponed. This is needed to give small merges more execution priority. You can only increase this ratio at runtime. To lower it you have to restart the server.
+Sets a ratio between the number of threads and the number of background merges and mutations that can be executed concurrently. For example, if the ratio equals to 2 and
+`background_pool_size` is set to 16 then ClickHouse can execute 32 background merges concurrently. This is possible, because background operations could be suspended and postponed. This is needed to give small merges more execution priority. You can only increase this ratio at runtime. To lower it you have to restart the server.
 The same as for `background_pool_size` setting `background_merges_mutations_concurrency_ratio` could be applied from the `default` profile for backward compatibility.
 
 Possible values:
@@ -1079,6 +1079,33 @@ Default value: 2.
 <background_merges_mutations_concurrency_ratio>3</background_merges_mutations_concurrency_ratio>
 ```
 
+## merges_mutations_memory_usage_soft_limit {#merges_mutations_memory_usage_soft_limit}
+
+Sets the limit on how much RAM is allowed to use for performing merge and mutation operations.
+Zero means unlimited.
+If ClickHouse reaches this limit, it won't schedule any new background merge or mutation operations but will continue to execute already scheduled tasks.
+
+Possible values:
+
+-   Any positive integer.
+
+**Example**
+
+```xml
+<merges_mutations_memory_usage_soft_limit>0</merges_mutations_memory_usage_soft_limit>
+```
+
+## merges_mutations_memory_usage_to_ram_ratio {#merges_mutations_memory_usage_to_ram_ratio}
+
+The default `merges_mutations_memory_usage_soft_limit` value is calculated as `memory_amount * merges_mutations_memory_usage_to_ram_ratio`.
+
+Default value: `0.5`.
+
+**See also**
+
+-   [max_memory_usage](../../operations/settings/query-complexity.md#settings_max_memory_usage)
+-   [merges_mutations_memory_usage_soft_limit](#merges_mutations_memory_usage_soft_limit)
+
 ## background_merges_mutations_scheduling_policy {#background_merges_mutations_scheduling_policy}
 
 Algorithm used to select next merge or mutation to be executed by background thread pool. Policy may be changed at runtime without server restart.
diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 8c0d50bae55..cba7a4c4778 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -135,6 +135,7 @@ namespace CurrentMetrics
     extern const Metric Revision;
     extern const Metric VersionInteger;
     extern const Metric MemoryTracking;
+    extern const Metric MergesMutationsMemoryTracking;
     extern const Metric MaxDDLEntryID;
     extern const Metric MaxPushedDDLEntryID;
 }
@@ -1225,6 +1226,25 @@ try
             total_memory_tracker.setDescription("(total)");
             total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
 
+            size_t merges_mutations_memory_usage_soft_limit = server_settings_.merges_mutations_memory_usage_soft_limit;
+
+            size_t default_merges_mutations_server_memory_usage = static_cast<size_t>(memory_amount * server_settings_.merges_mutations_memory_usage_to_ram_ratio);
+            if (merges_mutations_memory_usage_soft_limit == 0 || merges_mutations_memory_usage_soft_limit > default_merges_mutations_server_memory_usage)
+            {
+                merges_mutations_memory_usage_soft_limit = default_merges_mutations_server_memory_usage;
+                LOG_WARNING(log, "Setting merges_mutations_memory_usage_soft_limit was set to {}"
+                    " ({} available * {:.2f} merges_mutations_memory_usage_to_ram_ratio)",
+                    formatReadableSizeWithBinarySuffix(merges_mutations_memory_usage_soft_limit),
+                    formatReadableSizeWithBinarySuffix(memory_amount),
+                    server_settings_.merges_mutations_memory_usage_to_ram_ratio);
+            }
+
+            LOG_INFO(log, "Merges and mutations memory limit is set to {}",
+                formatReadableSizeWithBinarySuffix(merges_mutations_memory_usage_soft_limit));
+            background_memory_tracker.setSoftLimit(merges_mutations_memory_usage_soft_limit);
+            background_memory_tracker.setDescription("(background)");
+            background_memory_tracker.setMetric(CurrentMetrics::MergesMutationsMemoryTracking);
+
             total_memory_tracker.setAllowUseJemallocMemory(server_settings_.allow_use_jemalloc_memory);
 
             auto * global_overcommit_tracker = global_context->getGlobalOvercommitTracker();
diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp
index 81c1481e2de..ea248d996a7 100644
--- a/src/Common/CurrentMetrics.cpp
+++ b/src/Common/CurrentMetrics.cpp
@@ -53,6 +53,7 @@
     M(QueryThread, "Number of query processing threads") \
     M(ReadonlyReplica, "Number of Replicated tables that are currently in readonly state due to re-initialization after ZooKeeper session loss or due to startup without ZooKeeper configured.") \
     M(MemoryTracking, "Total amount of memory (bytes) allocated by the server.") \
+    M(MergesMutationsMemoryTracking, "Total amount of memory (bytes) allocated by background tasks (merges and mutations).") \
     M(EphemeralNode, "Number of ephemeral nodes hold in ZooKeeper.") \
     M(ZooKeeperSession, "Number of sessions (connections) to ZooKeeper. Should be no more than one, because using more than one connection to ZooKeeper may lead to bugs due to lack of linearizability (stale reads) that ZooKeeper consistency model allows.") \
     M(ZooKeeperWatch, "Number of watches (event subscriptions) in ZooKeeper.") \
diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp
index 674d8d469af..9bff365483d 100644
--- a/src/Common/MemoryTracker.cpp
+++ b/src/Common/MemoryTracker.cpp
@@ -96,6 +96,7 @@ using namespace std::chrono_literals;
 static constexpr size_t log_peak_memory_usage_every = 1ULL << 30;
 
 MemoryTracker total_memory_tracker(nullptr, VariableContext::Global);
+MemoryTracker background_memory_tracker(&total_memory_tracker, VariableContext::User);
 
 std::atomic<Int64> MemoryTracker::free_memory_in_allocator_arenas;
 
@@ -528,3 +529,10 @@ void MemoryTracker::setOrRaiseProfilerLimit(Int64 value)
     while ((value == 0 || old_value < value) && !profiler_limit.compare_exchange_weak(old_value, value))
         ;
 }
+
+bool canEnqueueBackgroundTask()
+{
+    auto limit = background_memory_tracker.getSoftLimit();
+    auto amount = background_memory_tracker.get();
+    return limit == 0 || amount < limit;
+}
diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h
index 0d7748856bd..260005fd536 100644
--- a/src/Common/MemoryTracker.h
+++ b/src/Common/MemoryTracker.h
@@ -110,6 +110,22 @@ public:
         return amount.load(std::memory_order_relaxed);
     }
 
+    // Merges and mutations may pass memory ownership to other threads thus in the end of execution
+    // MemoryTracker for background task may have a non-zero counter.
+    // This method is intended to fix the counter inside of background_memory_tracker.
+    // NOTE: We can't use alloc/free methods to do it, because they also will change the value inside
+    // of total_memory_tracker.
+    void adjustOnBackgroundTaskEnd(const MemoryTracker * child)
+    {
+        auto background_memory_consumption = child->amount.load(std::memory_order_relaxed);
+        amount.fetch_sub(background_memory_consumption, std::memory_order_relaxed);
+
+        // Also fix CurrentMetrics::MergesMutationsMemoryTracking
+        auto metric_loaded = metric.load(std::memory_order_relaxed);
+        if (metric_loaded != CurrentMetrics::end())
+            CurrentMetrics::sub(metric_loaded, background_memory_consumption);
+    }
+
     Int64 getPeak() const
     {
         return peak.load(std::memory_order_relaxed);
@@ -220,3 +236,6 @@ public:
 };
 
 extern MemoryTracker total_memory_tracker;
+extern MemoryTracker background_memory_tracker;
+
+bool canEnqueueBackgroundTask();
diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h
index aabc89cc6d7..2d8c37783db 100644
--- a/src/Core/ServerSettings.h
+++ b/src/Core/ServerSettings.h
@@ -40,6 +40,8 @@ namespace DB
     M(String, temporary_data_in_cache, "", "Cache disk name for temporary data.", 0) \
     M(UInt64, max_server_memory_usage, 0, "Limit on total memory usage. Zero means Unlimited.", 0) \
     M(Double, max_server_memory_usage_to_ram_ratio, 0.9, "Same as max_server_memory_usage but in to ram ratio. Allows to lower max memory on low-memory systems.", 0) \
+    M(UInt64, merges_mutations_memory_usage_soft_limit, 0, "Limit on total memory usage for merges and mutations. Zero means Unlimited.", 0) \
+    M(Double, merges_mutations_memory_usage_to_ram_ratio, 0.5, "Same as merges_mutations_memory_usage_soft_limit but in to ram ratio. Allows to lower memory limit on low-memory systems.", 0) \
     M(Bool, allow_use_jemalloc_memory, true, "Allows to use jemalloc memory.", 0) \
     \
     M(UInt64, max_concurrent_queries, 0, "Limit on total number of concurrently executed queries. Zero means Unlimited.", 0) \
diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp
index fd4a6b5e996..559652fe56c 100644
--- a/src/Interpreters/ThreadStatusExt.cpp
+++ b/src/Interpreters/ThreadStatusExt.cpp
@@ -84,6 +84,7 @@ ThreadGroupPtr ThreadGroup::createForBackgroundProcess(ContextPtr storage_contex
     group->memory_tracker.setProfilerStep(settings.memory_profiler_step);
     group->memory_tracker.setSampleProbability(settings.memory_profiler_sample_probability);
     group->memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator);
+    group->memory_tracker.setParent(&background_memory_tracker);
     if (settings.memory_tracker_fault_probability > 0.0)
         group->memory_tracker.setFaultProbability(settings.memory_tracker_fault_probability);
 
diff --git a/src/Storages/MergeTree/MergeList.cpp b/src/Storages/MergeTree/MergeList.cpp
index 0bf662921ad..d54079bc7a5 100644
--- a/src/Storages/MergeTree/MergeList.cpp
+++ b/src/Storages/MergeTree/MergeList.cpp
@@ -78,4 +78,9 @@ MergeInfo MergeListElement::getInfo() const
     return res;
 }
 
+MergeListElement::~MergeListElement()
+{
+    background_memory_tracker.adjustOnBackgroundTaskEnd(&getMemoryTracker());
+}
+
 }
diff --git a/src/Storages/MergeTree/MergeList.h b/src/Storages/MergeTree/MergeList.h
index 9c8c2ebd1e4..d8271a66b45 100644
--- a/src/Storages/MergeTree/MergeList.h
+++ b/src/Storages/MergeTree/MergeList.h
@@ -113,6 +113,8 @@ struct MergeListElement : boost::noncopyable
     MergeListElement * ptr() { return this; }
 
     MergeListElement & ref() { return *this; }
+
+    ~MergeListElement();
 };
 
 /** Maintains a list of currently running merges.
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 34bf5d55270..6cb3ce35e5b 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -7,6 +7,7 @@
 #include <base/sort.h>
 #include <Backups/BackupEntriesCollector.h>
 #include <Databases/IDatabase.h>
+#include <Common/MemoryTracker.h>
 #include <Common/escapeForFileName.h>
 #include <Common/ProfileEventsScope.h>
 #include <Common/typeid_cast.h>
@@ -41,6 +42,7 @@
 #include <Processors/QueryPlan/QueryPlan.h>
 #include <Processors/QueryPlan/BuildQueryPipelineSettings.h>
 #include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
+#include <fmt/core.h>
 
 namespace DB
 {
@@ -918,7 +920,14 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
 
     SelectPartsDecision select_decision = SelectPartsDecision::CANNOT_SELECT;
 
-    if (partition_id.empty())
+    if (!canEnqueueBackgroundTask())
+    {
+        if (out_disable_reason)
+            *out_disable_reason = fmt::format("Current background tasks memory usage ({}) is more than the limit ({})",
+                formatReadableSizeWithBinarySuffix(background_memory_tracker.get()),
+                formatReadableSizeWithBinarySuffix(background_memory_tracker.getSoftLimit()));
+    }
+    else if (partition_id.empty())
     {
         UInt64 max_source_parts_size = merger_mutator.getMaxSourcePartsSizeForMerge();
         bool merge_with_ttl_allowed = getTotalMergesWithTTLInMergeList() < data_settings->max_number_of_merges_with_ttl_in_pool;
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 5cd02c33d55..4a36cf03c2a 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -5,6 +5,7 @@
 
 #include <base/hex.h>
 #include <Common/Macros.h>
+#include <Common/MemoryTracker.h>
 #include <Common/ProfileEventsScope.h>
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/ZooKeeper/KeeperException.h>
@@ -3223,7 +3224,14 @@ void StorageReplicatedMergeTree::mergeSelectingTask()
 
         auto merges_and_mutations_queued = queue.countMergesAndPartMutations();
         size_t merges_and_mutations_sum = merges_and_mutations_queued.merges + merges_and_mutations_queued.mutations;
-        if (merges_and_mutations_sum >= storage_settings_ptr->max_replicated_merges_in_queue)
+        if (!canEnqueueBackgroundTask())
+        {
+            LOG_TRACE(log, "Reached memory limit for the background tasks ({}), so won't select new parts to merge or mutate."
+                "Current background tasks memory usage: {}.",
+                formatReadableSizeWithBinarySuffix(background_memory_tracker.getSoftLimit()),
+                formatReadableSizeWithBinarySuffix(background_memory_tracker.get()));
+        }
+        else if (merges_and_mutations_sum >= storage_settings_ptr->max_replicated_merges_in_queue)
         {
             LOG_TRACE(log, "Number of queued merges ({}) and part mutations ({})"
                 " is greater than max_replicated_merges_in_queue ({}), so won't select new parts to merge or mutate.",
diff --git a/tests/integration/test_merges_memory_limit/__init__.py b/tests/integration/test_merges_memory_limit/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_merges_memory_limit/test.py b/tests/integration/test_merges_memory_limit/test.py
new file mode 100644
index 00000000000..04729f3a01c
--- /dev/null
+++ b/tests/integration/test_merges_memory_limit/test.py
@@ -0,0 +1,38 @@
+import pytest
+
+from helpers.cluster import ClickHouseCluster
+
+cluster = ClickHouseCluster(__file__)
+
+node = cluster.add_instance("node")
+
+
+@pytest.fixture(scope="module", autouse=True)
+def start_cluster():
+    try:
+        cluster.start()
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+
+def test_memory_limit_success():
+    node.query(
+        "CREATE TABLE test_merge_oom ENGINE=AggregatingMergeTree ORDER BY id EMPTY AS SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(20000)"
+    )
+    node.query("SYSTEM STOP MERGES test_merge_oom")
+    node.query(
+        "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(10000)"
+    )
+    node.query(
+        "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(10000)"
+    )
+    node.query(
+        "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(10000)"
+    )
+    _, error = node.query_and_get_answer_with_error(
+        "SYSTEM START MERGES test_merge_oom;SET optimize_throw_if_noop=1;OPTIMIZE TABLE test_merge_oom FINAL"
+    )
+
+    assert not error
+    node.query("DROP TABLE test_merge_oom")

From a1f21a5fc427e860199aa9e1db3c1c4177a340f7 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 14 Apr 2023 16:00:32 +0200
Subject: [PATCH 018/689] Do not log peak for background memory tracker

---
 src/Common/MemoryTracker.cpp | 8 ++++++--
 src/Common/MemoryTracker.h   | 1 +
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp
index 9bff365483d..81cac2617c5 100644
--- a/src/Common/MemoryTracker.cpp
+++ b/src/Common/MemoryTracker.cpp
@@ -96,13 +96,17 @@ using namespace std::chrono_literals;
 static constexpr size_t log_peak_memory_usage_every = 1ULL << 30;
 
 MemoryTracker total_memory_tracker(nullptr, VariableContext::Global);
-MemoryTracker background_memory_tracker(&total_memory_tracker, VariableContext::User);
+MemoryTracker background_memory_tracker(&total_memory_tracker, VariableContext::User, false);
 
 std::atomic<Int64> MemoryTracker::free_memory_in_allocator_arenas;
 
 MemoryTracker::MemoryTracker(VariableContext level_) : parent(&total_memory_tracker), level(level_) {}
 MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_) : parent(parent_), level(level_) {}
-
+MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_, bool log_peak_memory_usage_in_destructor_)
+    : parent(parent_)
+    , log_peak_memory_usage_in_destructor(log_peak_memory_usage_in_destructor_)
+    , level(level_)
+{}
 
 MemoryTracker::~MemoryTracker()
 {
diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h
index 260005fd536..4e29d40c953 100644
--- a/src/Common/MemoryTracker.h
+++ b/src/Common/MemoryTracker.h
@@ -98,6 +98,7 @@ public:
 
     explicit MemoryTracker(VariableContext level_ = VariableContext::Thread);
     explicit MemoryTracker(MemoryTracker * parent_, VariableContext level_ = VariableContext::Thread);
+    MemoryTracker(MemoryTracker * parent_, VariableContext level_, bool log_peak_memory_usage_in_destructor_);
 
     ~MemoryTracker();
 

From 74c6ca558b3301427368941f3e0df031b04cc10d Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Fri, 14 Apr 2023 18:03:46 +0200
Subject: [PATCH 019/689] Removed line from test_disallow_concurrrency for CI
 checks

---
 .../test_backup_restore_on_cluster/test_disallow_concurrency.py  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
index 0d8fad96438..a76af00d339 100644
--- a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
+++ b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
@@ -6,7 +6,6 @@ import concurrent
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import TSV, assert_eq_with_retry
 
-
 cluster = ClickHouseCluster(__file__)
 
 num_nodes = 10

From 93572ab42768195fccc77a809e149355a3f8065d Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Sat, 15 Apr 2023 13:43:04 +0200
Subject: [PATCH 020/689] Removed parameter from setStage function and added
 function setStageForCluster

---
 src/Backups/BackupCoordinationLocal.cpp   |  6 +++++-
 src/Backups/BackupCoordinationLocal.h     |  3 ++-
 src/Backups/BackupCoordinationRemote.cpp  | 12 +++++++-----
 src/Backups/BackupCoordinationRemote.h    |  3 ++-
 src/Backups/BackupsWorker.cpp             |  4 ++--
 src/Backups/IBackupCoordination.h         |  3 ++-
 src/Backups/IRestoreCoordination.h        |  3 ++-
 src/Backups/RestoreCoordinationLocal.cpp  |  6 +++++-
 src/Backups/RestoreCoordinationLocal.h    |  3 ++-
 src/Backups/RestoreCoordinationRemote.cpp | 12 +++++++-----
 src/Backups/RestoreCoordinationRemote.h   |  3 ++-
 11 files changed, 38 insertions(+), 20 deletions(-)

diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp
index 47b67693114..5b7ee37618b 100644
--- a/src/Backups/BackupCoordinationLocal.cpp
+++ b/src/Backups/BackupCoordinationLocal.cpp
@@ -15,7 +15,11 @@ BackupCoordinationLocal::BackupCoordinationLocal(bool plain_backup_)
 
 BackupCoordinationLocal::~BackupCoordinationLocal() = default;
 
-void BackupCoordinationLocal::setStage(const String &, const String &, const bool &)
+void BackupCoordinationLocal::setStage(const String &, const String &)
+{
+}
+
+void BackupCoordinationLocal::setStageForCluster(const String &)
 {
 }
 
diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h
index 1f6bb84972e..f1ffa8e8517 100644
--- a/src/Backups/BackupCoordinationLocal.h
+++ b/src/Backups/BackupCoordinationLocal.h
@@ -22,7 +22,8 @@ public:
     BackupCoordinationLocal(bool plain_backup_);
     ~BackupCoordinationLocal() override;
 
-    void setStage(const String & new_stage, const String & message, const bool & for_cluster = false) override;
+    void setStage(const String & new_stage, const String & message) override;
+    void setStageForCluster(const String & new_stage) override; /// Sets stage for cluster
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp
index cd4901eb5ae..c5c4efa3530 100644
--- a/src/Backups/BackupCoordinationRemote.cpp
+++ b/src/Backups/BackupCoordinationRemote.cpp
@@ -252,12 +252,14 @@ void BackupCoordinationRemote::removeAllNodes()
 }
 
 
-void BackupCoordinationRemote::setStage(const String & new_stage, const String & message, const bool & for_cluster)
+void BackupCoordinationRemote::setStage(const String & new_stage, const String & message)
 {
-    if (for_cluster)
-        stage_sync->setStageForCluster(new_stage);
-    else
-        stage_sync->set(current_host, new_stage, message);
+    stage_sync->set(current_host, new_stage, message);
+}
+
+void BackupCoordinationRemote::setStageForCluster(const String & new_stage)
+{
+    stage_sync->setStageForCluster(new_stage);
 }
 
 void BackupCoordinationRemote::setError(const Exception & exception)
diff --git a/src/Backups/BackupCoordinationRemote.h b/src/Backups/BackupCoordinationRemote.h
index 40ce2ae6ccc..c659cb0d459 100644
--- a/src/Backups/BackupCoordinationRemote.h
+++ b/src/Backups/BackupCoordinationRemote.h
@@ -33,7 +33,8 @@ public:
 
     ~BackupCoordinationRemote() override;
 
-    void setStage(const String & new_stage, const String & message, const bool & for_cluster = false) override;
+    void setStage(const String & new_stage, const String & message) override;
+    void setStageForCluster(const String & new_stage) override; /// Sets stage for cluster
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp
index aae9cfd620f..de05cc2b092 100644
--- a/src/Backups/BackupsWorker.cpp
+++ b/src/Backups/BackupsWorker.cpp
@@ -368,7 +368,7 @@ void BackupsWorker::doBackup(
 
             /// Wait until all the hosts have written their backup entries.
             backup_coordination->waitForStage(Stage::COMPLETED);
-            backup_coordination->setStage(Stage::COMPLETED, /* message */ "", /* for_cluster */ true);
+            backup_coordination->setStageForCluster(Stage::COMPLETED);
         }
         else
         {
@@ -709,7 +709,7 @@ void BackupsWorker::doRestore(
 
             /// Wait until all the hosts have written their backup entries.
             restore_coordination->waitForStage(Stage::COMPLETED);
-            restore_coordination->setStage(Stage::COMPLETED, /* message */ "", /* for_cluster */ true);
+            restore_coordination->setStageForCluster(Stage::COMPLETED);
         }
         else
         {
diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h
index 614e6a16db8..6caae1dd741 100644
--- a/src/Backups/IBackupCoordination.h
+++ b/src/Backups/IBackupCoordination.h
@@ -21,7 +21,8 @@ public:
     virtual ~IBackupCoordination() = default;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    virtual void setStage(const String & new_stage, const String & message, const bool & for_cluster = false) = 0;
+    virtual void setStage(const String & new_stage, const String & message) = 0;
+    virtual void setStageForCluster(const String & new_stage) = 0;
     virtual void setError(const Exception & exception) = 0;
     virtual Strings waitForStage(const String & stage_to_wait) = 0;
     virtual Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) = 0;
diff --git a/src/Backups/IRestoreCoordination.h b/src/Backups/IRestoreCoordination.h
index 599a698a1f9..a5c8db84c86 100644
--- a/src/Backups/IRestoreCoordination.h
+++ b/src/Backups/IRestoreCoordination.h
@@ -18,7 +18,8 @@ public:
     virtual ~IRestoreCoordination() = default;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    virtual void setStage(const String & new_stage, const String & message, const bool & for_cluster = false) = 0;
+    virtual void setStage(const String & new_stage, const String & message) = 0;
+    virtual void setStageForCluster(const String & new_stage) = 0; /// Sets stage for cluster
     virtual void setError(const Exception & exception) = 0;
     virtual Strings waitForStage(const String & stage_to_wait) = 0;
     virtual Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) = 0;
diff --git a/src/Backups/RestoreCoordinationLocal.cpp b/src/Backups/RestoreCoordinationLocal.cpp
index f689277f5b6..513204c931c 100644
--- a/src/Backups/RestoreCoordinationLocal.cpp
+++ b/src/Backups/RestoreCoordinationLocal.cpp
@@ -11,7 +11,11 @@ RestoreCoordinationLocal::RestoreCoordinationLocal() : log(&Poco::Logger::get("R
 
 RestoreCoordinationLocal::~RestoreCoordinationLocal() = default;
 
-void RestoreCoordinationLocal::setStage(const String &, const String &, const bool &)
+void RestoreCoordinationLocal::setStage(const String &, const String &)
+{
+}
+
+void RestoreCoordinationLocal::setStageForCluster(const String &)
 {
 }
 
diff --git a/src/Backups/RestoreCoordinationLocal.h b/src/Backups/RestoreCoordinationLocal.h
index 4456ad966d4..0e4f4f01846 100644
--- a/src/Backups/RestoreCoordinationLocal.h
+++ b/src/Backups/RestoreCoordinationLocal.h
@@ -19,7 +19,8 @@ public:
     ~RestoreCoordinationLocal() override;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    void setStage(const String & new_stage, const String & message, const bool & for_cluster = false) override;
+    void setStage(const String & new_stage, const String & message) override;
+    void setStageForCluster(const String & new_stage) override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp
index 0a89b1cd4e7..2c2187a1eb5 100644
--- a/src/Backups/RestoreCoordinationRemote.cpp
+++ b/src/Backups/RestoreCoordinationRemote.cpp
@@ -90,12 +90,14 @@ void RestoreCoordinationRemote::createRootNodes()
         });
 }
 
-void RestoreCoordinationRemote::setStage(const String & new_stage, const String & message, const bool & for_cluster)
+void RestoreCoordinationRemote::setStage(const String & new_stage, const String & message)
 {
-    if (for_cluster)
-        stage_sync->setStageForCluster(new_stage);
-    else
-        stage_sync->set(current_host, new_stage, message);
+    stage_sync->set(current_host, new_stage, message);
+}
+
+void RestoreCoordinationRemote::setStageForCluster(const String & new_stage)
+{
+    stage_sync->setStageForCluster(new_stage);
 }
 
 void RestoreCoordinationRemote::setError(const Exception & exception)
diff --git a/src/Backups/RestoreCoordinationRemote.h b/src/Backups/RestoreCoordinationRemote.h
index 21a38f01fa6..947d08a66e5 100644
--- a/src/Backups/RestoreCoordinationRemote.h
+++ b/src/Backups/RestoreCoordinationRemote.h
@@ -26,7 +26,8 @@ public:
     ~RestoreCoordinationRemote() override;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    void setStage(const String & new_stage, const String & message, const bool & for_cluster = false) override;
+    void setStage(const String & new_stage, const String & message) override;
+    void setStageForCluster(const String & new_stage) override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;

From 6f501f983ed0f9d63df4abdbbe25afe64283776b Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Mon, 17 Apr 2023 17:47:03 +0200
Subject: [PATCH 021/689] Update test.py

---
 tests/integration/test_merges_memory_limit/test.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/integration/test_merges_memory_limit/test.py b/tests/integration/test_merges_memory_limit/test.py
index 04729f3a01c..22bf143f4f7 100644
--- a/tests/integration/test_merges_memory_limit/test.py
+++ b/tests/integration/test_merges_memory_limit/test.py
@@ -27,9 +27,6 @@ def test_memory_limit_success():
     node.query(
         "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(10000)"
     )
-    node.query(
-        "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(10000)"
-    )
     _, error = node.query_and_get_answer_with_error(
         "SYSTEM START MERGES test_merge_oom;SET optimize_throw_if_noop=1;OPTIMIZE TABLE test_merge_oom FINAL"
     )

From 8cc425cd2946243f3756eed577d128705e781ceb Mon Sep 17 00:00:00 2001
From: Manas Alekar <manas@clickhouse.com>
Date: Tue, 18 Apr 2023 00:14:35 -0700
Subject: [PATCH 022/689] INTO OUTFILE enhancements

---
 src/Client/ClientBase.cpp             | 14 +++++++++++++-
 src/Parsers/ASTQueryWithOutput.h      |  1 +
 src/Parsers/ParserQueryWithOutput.cpp |  6 ++++++
 3 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 64a18479ca9..91c8c7e6e79 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -571,6 +571,12 @@ try
                 CompressionMethod compression_method = chooseCompressionMethod(out_file, compression_method_string);
                 UInt64 compression_level = 3;
 
+                if (query_with_output->is_outfile_append && compression_method != CompressionMethod::None) {
+                    throw Exception(
+                        ErrorCodes::BAD_ARGUMENTS,
+                        "Cannot append to compressed file. Please use uncompressed file or remove APPEND keyword.");
+                }
+
                 if (query_with_output->compression_level)
                 {
                     const auto & compression_level_node = query_with_output->compression_level->as<ASTLiteral &>();
@@ -585,8 +591,14 @@ try
                             range.second);
                 }
 
+                auto flags = O_WRONLY | O_EXCL;
+                if (query_with_output->is_outfile_append)
+                    flags |= O_APPEND;
+                else
+                    flags |= O_CREAT;
+
                 out_file_buf = wrapWriteBufferWithCompressionMethod(
-                    std::make_unique<WriteBufferFromFile>(out_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_EXCL | O_CREAT),
+                    std::make_unique<WriteBufferFromFile>(out_file, DBMS_DEFAULT_BUFFER_SIZE, flags),
                     compression_method,
                     static_cast<int>(compression_level)
                 );
diff --git a/src/Parsers/ASTQueryWithOutput.h b/src/Parsers/ASTQueryWithOutput.h
index 892d911e2e2..09f08772468 100644
--- a/src/Parsers/ASTQueryWithOutput.h
+++ b/src/Parsers/ASTQueryWithOutput.h
@@ -16,6 +16,7 @@ class ASTQueryWithOutput : public IAST
 public:
     ASTPtr out_file;
     bool is_into_outfile_with_stdout;
+    bool is_outfile_append;
     ASTPtr format;
     ASTPtr settings_ast;
     ASTPtr compression;
diff --git a/src/Parsers/ParserQueryWithOutput.cpp b/src/Parsers/ParserQueryWithOutput.cpp
index 2fb7c406d74..1ba44fc9939 100644
--- a/src/Parsers/ParserQueryWithOutput.cpp
+++ b/src/Parsers/ParserQueryWithOutput.cpp
@@ -100,6 +100,12 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
         if (!out_file_p.parse(pos, query_with_output.out_file, expected))
             return false;
 
+        ParserKeyword s_append("APPEND");
+        if (s_append.ignore(pos, expected))
+        {
+            query_with_output.is_outfile_append = true;
+        }
+
         ParserKeyword s_stdout("AND STDOUT");
         if (s_stdout.ignore(pos, expected))
         {

From 2efc7492270a7583654d88cb4a5391d01dccab92 Mon Sep 17 00:00:00 2001
From: Manas Alekar <manas@clickhouse.com>
Date: Tue, 18 Apr 2023 00:30:00 -0700
Subject: [PATCH 023/689] Add test for INTO FILE ... APPEND.

---
 .../02001_append_output_file.reference            |  2 ++
 .../0_stateless/02001_append_output_file.sh       | 15 +++++++++++++++
 2 files changed, 17 insertions(+)
 create mode 100644 tests/queries/0_stateless/02001_append_output_file.reference
 create mode 100755 tests/queries/0_stateless/02001_append_output_file.sh

diff --git a/tests/queries/0_stateless/02001_append_output_file.reference b/tests/queries/0_stateless/02001_append_output_file.reference
new file mode 100644
index 00000000000..6f51dfc24e1
--- /dev/null
+++ b/tests/queries/0_stateless/02001_append_output_file.reference
@@ -0,0 +1,2 @@
+Hello, World! From client.
+Hello, World! From local.
diff --git a/tests/queries/0_stateless/02001_append_output_file.sh b/tests/queries/0_stateless/02001_append_output_file.sh
new file mode 100755
index 00000000000..5cbae5b8cb3
--- /dev/null
+++ b/tests/queries/0_stateless/02001_append_output_file.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+set -e
+
+[ -e "${CLICKHOUSE_TMP}"/test_append_to_output_file] && rm "${CLICKHOUSE_TMP}"/test_append_to_output_file
+
+${CLICKHOUSE_CLIENT} --query "SELECT * FROM (SELECT 'Hello, World! From client.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_append_to_output_file'"
+${CLICKHOUSE_LOCAL} --query "SELECT * FROM (SELECT 'Hello, World! From local.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_append_to_output_file' APPEND"
+cat ${CLICKHOUSE_TMP}/test_append_to_output_file
+
+rm -f "${CLICKHOUSE_TMP}/test_append_to_output_file"

From 8874ede98a05a5e41670e5c9a666c2eaa6faced0 Mon Sep 17 00:00:00 2001
From: Manas Alekar <manas@clickhouse.com>
Date: Tue, 18 Apr 2023 00:56:14 -0700
Subject: [PATCH 024/689] Fix style in test script.

---
 tests/queries/0_stateless/02001_append_output_file.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02001_append_output_file.sh b/tests/queries/0_stateless/02001_append_output_file.sh
index 5cbae5b8cb3..47ac0183d91 100755
--- a/tests/queries/0_stateless/02001_append_output_file.sh
+++ b/tests/queries/0_stateless/02001_append_output_file.sh
@@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 set -e
 
-[ -e "${CLICKHOUSE_TMP}"/test_append_to_output_file] && rm "${CLICKHOUSE_TMP}"/test_append_to_output_file
+[ -e "${CLICKHOUSE_TMP}"/test_append_to_output_file ] && rm "${CLICKHOUSE_TMP}"/test_append_to_output_file
 
 ${CLICKHOUSE_CLIENT} --query "SELECT * FROM (SELECT 'Hello, World! From client.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_append_to_output_file'"
 ${CLICKHOUSE_LOCAL} --query "SELECT * FROM (SELECT 'Hello, World! From local.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_append_to_output_file' APPEND"

From 3fe36a3db6215b57bd751362914f6da4ab93a14d Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sun, 9 Apr 2023 12:50:26 +0000
Subject: [PATCH 025/689] wip AsyncLoader

---
 src/Common/AsyncLoader.h  | 556 ++++++++++++++++++++++++++++++++++++++
 src/Common/ErrorCodes.cpp |   3 +
 2 files changed, 559 insertions(+)
 create mode 100644 src/Common/AsyncLoader.h

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
new file mode 100644
index 00000000000..dd35621443d
--- /dev/null
+++ b/src/Common/AsyncLoader.h
@@ -0,0 +1,556 @@
+#pragma once
+
+#include <condition_variable>
+#include <exception>
+#include <memory>
+#include <map>
+#include <mutex>
+#include <unordered_set>
+#include <unordered_map>
+#include <boost/noncopyable.hpp>
+#include <base/defines.h>
+#include <base/types.h>
+#include <Common/CurrentMetrics.h>
+#include <Common/ThreadPool.h>
+#include <Common/ErrorCodes.h>
+#include <Common/Exception.h>
+#include <Common/noexcept_scope.h>
+#include <Common/setThreadName.h>
+
+namespace DB
+{
+
+class LoadJob;
+using LoadJobPtr = std::shared_ptr<LoadJob>;
+using LoadJobSet = std::unordered_set<LoadJobPtr>;
+class LoadTask;
+class AsyncLoader;
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+    extern const int ASYNC_LOAD_FAILED;
+    extern const int ASYNC_LOAD_CANCELED;
+    extern const int ASYNC_LOAD_DEPENDENCY_FAILED;
+}
+
+enum class LoadStatus
+{
+    PENDING, // Load is not finished yet
+    SUCCESS, // Load was successful
+    FAILED   // Load failed or canceled
+};
+
+class LoadJob : private boost::noncopyable
+{
+public:
+    template <class Func>
+    LoadJob(LoadJobSet && dependencies_, const String & name_, Func && func_)
+        : dependencies(std::move(dependencies_))
+        , name(name_)
+        , func(std::forward<Func>(func_))
+    {}
+
+    LoadStatus status() const
+    {
+        std::unique_lock lock{mutex};
+        return !is_finished ? LoadStatus::PENDING : (exception ? LoadStatus::FAILED : LoadStatus::SUCCESS);
+    }
+
+    void wait()
+    {
+        std::unique_lock lock{mutex};
+        finished.wait(lock, [this] { return is_finished; });
+    }
+
+    void get()
+    {
+        std::unique_lock lock{mutex};
+        finished.wait(lock, [this] { return is_finished; });
+        if (exception)
+            std::rethrow_exception(exception);
+    }
+
+    const LoadJobSet dependencies; // jobs to be done before this one, with ownership
+    const String name;
+    std::atomic<ssize_t> priority{0};
+
+private:
+    friend class AsyncLoader;
+
+    void setSuccess()
+    {
+        std::unique_lock lock{mutex};
+        is_finished = true;
+        finished.notify_all();
+    }
+
+    void setFailure(const std::exception_ptr & ptr)
+    {
+        std::unique_lock lock{mutex};
+        is_finished = true;
+        exception = ptr;
+        finished.notify_all();
+    }
+
+    std::function<void(const LoadJob & self)> func;
+
+    std::mutex mutex;
+    std::condition_variable finished;
+    bool is_finished = false;
+    std::exception_ptr exception;
+};
+
+template <class Func>
+LoadJobPtr makeLoadJob(LoadJobSet && dependencies, const String & name, Func && func)
+{
+    return std::make_shared<LoadJob>(std::move(dependencies), name, std::forward<Func>(func));
+}
+
+// TODO(serxa): write good comment
+class AsyncLoader : private boost::noncopyable
+{
+private:
+    // Key of a pending job in ready queue
+    struct ReadyKey {
+        ssize_t priority;
+        UInt64 ready_seqno;
+
+        bool operator<(const ReadyKey & rhs) const
+        {
+            if (priority == rhs.priority)
+                return ready_seqno < rhs.ready_seqno;
+            return priority > rhs.priority;
+        }
+    };
+
+    // Scheduling information for a pending job
+    struct Info {
+        ssize_t priority = 0;
+        size_t dependencies_left = 0;
+        UInt64 ready_seqno = 0; // zero means that job is not in ready queue
+        LoadJobSet dependent_jobs; // TODO(serxa): clean it on remove jobs
+
+        ReadyKey key() const
+        {
+            return {priority, ready_seqno};
+        }
+    };
+
+public:
+    using Metric = CurrentMetrics::Metric;
+
+    // Helper class that removes all not started jobs in destructor and wait all executing jobs to finish
+    class Task
+    {
+    public:
+        Task()
+            : loader(nullptr)
+        {}
+
+        Task(AsyncLoader * loader_, LoadJobSet && jobs_)
+            : loader(loader_)
+            , jobs(std::move(jobs_))
+        {}
+
+        Task(const Task & o) = delete;
+        Task & operator=(const Task & o) = delete;
+
+        Task(Task && o) noexcept
+            : loader(std::exchange(o.loader, nullptr))
+            , jobs(std::move(o.jobs))
+        {}
+
+        Task & operator=(Task && o) noexcept
+        {
+            loader = std::exchange(o.loader, nullptr);
+            jobs = std::move(o.jobs);
+            return *this;
+        }
+
+        void merge(Task && o)
+        {
+            if (!loader)
+            {
+                *this = std::move(o);
+            }
+            else
+            {
+                chassert(loader == o.loader);
+                jobs.merge(o.jobs);
+                o.loader = nullptr;
+            }
+        }
+
+        ~Task()
+        {
+            // Remove jobs that are not ready and wait for jobs that are in progress
+            if (loader)
+                loader->remove(jobs);
+        }
+
+        // Do not track jobs in this task
+        void detach()
+        {
+            loader = nullptr;
+            jobs.clear();
+        }
+
+    private:
+        AsyncLoader * loader;
+        LoadJobSet jobs;
+    };
+
+    AsyncLoader(Metric metric_threads, Metric metric_active_threads, size_t max_threads_)
+        : max_threads(max_threads_)
+        , pool(metric_threads, metric_active_threads, max_threads)
+    {}
+
+    void start()
+    {
+        std::unique_lock lock{mutex};
+        is_running = true;
+        for (size_t i = 0; workers < max_threads && i < ready_queue.size(); i++)
+            spawn(lock);
+    }
+
+    void stop()
+    {
+        std::unique_lock lock{mutex};
+        is_stopping = true;
+    }
+
+    [[nodiscard]] Task schedule(LoadJobSet && jobs, ssize_t priority = 0)
+    {
+        std::unique_lock lock{mutex};
+
+        // Sanity checks
+        for (const auto & job : jobs)
+        {
+            if (job->status() != LoadStatus::PENDING)
+                throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Trying to schedule already finished load job '{}'", job->name);
+            if (scheduled_jobs.contains(job))
+                throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Load job '{}' has been already scheduled", job->name);
+        }
+
+        // TODO(serxa): ensure scheduled_jobs graph will have no cycles, otherwise we have infinite recursion and other strange stuff?
+
+        // We do not want any exception to be throws after this point, because the following code is not exception-safe
+        DENY_ALLOCATIONS_IN_SCOPE;
+
+        // Schedule all incoming jobs
+        for (const auto & job : jobs)
+        {
+            NOEXCEPT_SCOPE({
+                ALLOW_ALLOCATIONS_IN_SCOPE;
+                scheduled_jobs.emplace(job, Info{.priority = priority});
+            )};
+            job->priority.store(priority); // Set user-facing priority
+        }
+
+        // Process incoming dependencies
+        for (const auto & job : jobs)
+        {
+            Info & info = scheduled_jobs.find(job)->second;
+            for (const auto & dep : job->dependencies)
+            {
+                // Register every dependency on scheduled job with back-link to dependent job
+                if (auto dep_info = scheduled_jobs.find(dep); dep_info != scheduled_jobs.end())
+                {
+                    NOEXCEPT_SCOPE({
+                        ALLOW_ALLOCATIONS_IN_SCOPE;
+                        dep_info->second.dependent_jobs.insert(job);
+                    });
+                    info.dependencies_left++;
+                }
+                else
+                {
+                    // TODO(serxa): check status: (1) pending: it is wrong - throw? (2) success: good - no dep. (3) failed: propagate failure!
+                }
+
+                // Priority inheritance: prioritize deps to have at least given `priority` to avoid priority inversion
+                prioritize(dep, priority, lock);
+            }
+
+            // Place jobs w/o dependencies to ready queue
+            if (info.dependencies_left == 0)
+                enqueue(info, job, lock);
+        }
+
+        return Task(this, std::move(jobs));
+    }
+
+    // Increase priority of a job and all its dependencies recursively
+    void prioritize(const LoadJobPtr & job, ssize_t new_priority)
+    {
+        DENY_ALLOCATIONS_IN_SCOPE;
+        std::unique_lock lock{mutex};
+        prioritize(job, new_priority, lock);
+    }
+
+    // Remove finished jobs, cancel scheduled jobs, wait for executing jobs to finish and remove them
+    void remove(const LoadJobSet & jobs)
+    {
+        DENY_ALLOCATIONS_IN_SCOPE;
+        std::unique_lock lock{mutex};
+        for (const auto & job : jobs)
+        {
+            if (auto it = finished_jobs.find(job); it != finished_jobs.end()) // Job is already finished
+            {
+                finished_jobs.erase(it);
+            }
+            else if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
+            {
+                if (info->second.dependencies_left > 0) // Job is not ready yet
+                    canceled(job);
+                else if (info->second.ready_seqno) // Job is enqueued in ready queue
+                {
+                    ready_queue.erase(info->second.key());
+                    info->second.ready_seqno = 0;
+                    canceled(job);
+                }
+                else // Job is currently executing
+                {
+                    lock.unlock();
+                    job->wait(); // Wait for job to finish
+                    lock.lock();
+                }
+                finished_jobs.erase(job);
+            }
+        }
+    }
+
+private:
+    void canceled(const LoadJobPtr & job, std::unique_lock<std::mutex> &)
+    {
+        std::exception_ptr e;
+        NOEXCEPT_SCOPE({
+            ALLOW_ALLOCATIONS_IN_SCOPE;
+            e = std::make_exception_ptr(
+                Exception(ASYNC_LOAD_CANCELED,
+                    "Load job '{}' canceled",
+                    job->name));
+        )};
+        failed(job, e, lock);
+    }
+
+    void loaded(const LoadJobPtr & job, std::unique_lock<std::mutex> & lock)
+    {
+        // Notify waiters
+        job->setSuccess();
+
+        // Update dependent jobs and enqueue if ready
+        chassert(scheduled_jobs.contains(job)); // Job was pending
+        for (const auto & dep : scheduled_jobs[job].dependent_jobs)
+        {
+            chassert(scheduled_jobs.contains(dep)); // All depended jobs must be pending
+            Info & dep_info = scheduled_jobs[dep];
+            if (--dep_info.dependencies_left == 0)
+                enqueue(dep_info, dep, lock);
+        }
+
+        finish(job);
+    }
+
+    void failed(const LoadJobPtr & job, std::exception_ptr exception_from_job, std::unique_lock<std::mutex> & lock)
+    {
+        // Notify waiters
+        job->setFailure(exception_from_job);
+
+        // Recurse into all dependent jobs
+        chassert(scheduled_jobs.contains(job)); // Job was pending
+        Info & info = scheduled_jobs[job];
+        LoadJobSet dependent;
+        dependent.swap(info.dependent_jobs); // To avoid container modification during recursion
+        for (const auto & dep : dependent)
+        {
+            std::exception_ptr e;
+            NOEXCEPT_SCOPE({
+                ALLOW_ALLOCATIONS_IN_SCOPE;
+                e = std::make_exception_ptr(
+                    Exception(ASYNC_LOAD_DEPENDENCY_FAILED,
+                        "Load job '{}' -> {}",
+                        dep->name,
+                        getExceptionMessage(exception_from_job, /* with_stack_trace = */ false)));
+            });
+            failed(dep, e, lock);
+        }
+
+        // Clean dependency graph edges
+        for (const auto & dep : job->dependencies)
+            if (auto dep_info = scheduled_jobs.find(dep); info != scheduled_jobs.end())
+                dep_info->second.dependent_jobs.erase(job);
+
+        // Job became finished
+        finish(job);
+    }
+
+    void finish(const LoadJobPtr & job, std::unique_lock<std::mutex> &)
+    {
+        scheduled_jobs.erase(job);
+        NOEXCEPT_SCOPE({
+            ALLOW_ALLOCATIONS_IN_SCOPE;
+            finished_jobs.insert(job);
+        )};
+    }
+
+    void prioritize(const LoadJobPtr & job, ssize_t new_priority, std::unique_lock<std::mutex> & lock)
+    {
+        if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
+        {
+            if (info->second.priority >= new_priority)
+                return; // Never lower priority
+
+            // Update priority and push job forward through ready queue if needed
+            if (info->second.ready_seqno)
+                ready_queue.erase(info->second.key());
+            info->second.priority = new_priority;
+            job->priority.store(new_priority); // Set user-facing priority (may affect executing jobs)
+            if (info->second.ready_seqno)
+            {
+                NOEXCEPT_SCOPE({
+                    ALLOW_ALLOCATIONS_IN_SCOPE;
+                    ready_queue.emplace(info->second.key(), job);
+                });
+            }
+
+            // Recurse into dependencies
+            for (const auto & dep : job->dependencies)
+                prioritize(dep, new_priority, lock);
+        }
+    }
+
+    void enqueue(Info & info, const LoadJobPtr & job, std::unique_lock<std::mutex> & lock)
+    {
+        chassert(info.dependencies_left == 0);
+        chassert(info.ready_seqno == 0);
+        info.ready_seqno = ++last_ready_seqno;
+        NOEXCEPT_SCOPE({
+            ALLOW_ALLOCATIONS_IN_SCOPE;
+            ready_queue.emplace(info.key(), job);
+        });
+
+        if (is_running && workers < max_threads) // TODO(serxa): Can we make max_thread changeable in runtime
+            spawn(lock);
+    }
+
+    void spawn(std::unique_lock<std::mutex> &)
+    {
+        // TODO(serxa): add metrics for number of active workers or executing jobs
+        workers++;
+        NOEXCEPT_SCOPE({
+            ALLOW_ALLOCATIONS_IN_SCOPE;
+            pool.scheduleOrThrowOnError([this] { worker(); });
+        });
+    }
+
+    void worker()
+    {
+        DENY_ALLOCATIONS_IN_SCOPE;
+
+        LoadJobPtr job;
+        std::exception_ptr exception_from_job;
+        while (true) { // TODO(serxa): shutdown
+
+            /// This is inside the loop to also reset previous thread names set inside the jobs
+            setThreadName("AsyncLoader");
+
+            {
+                std::unique_lock lock{mutex};
+
+                // Handle just executed job
+                if (exception_from_job)
+                    failed(job, exception_from_job, lock);
+                else if (job)
+                    loaded(job, lock);
+
+                if (ready_queue.empty())
+                {
+                    workers--;
+                    return;
+                }
+
+                // Take next job to be executed from the ready queue
+                auto it = ready_queue.begin();
+                job = it->second;
+                ready_queue.erase(it);
+                scheduled_jobs.find(job)->second.ready_seqno = 0; // This job is no longer in the ready queue
+            }
+
+            try
+            {
+                ALLOW_ALLOCATIONS_IN_SCOPE;
+                job->func(*job);
+                exception_from_job = {};
+            }
+            catch (...)
+            {
+                tryLogCurrentException(__PRETTY_FUNCTION__);
+                NOEXCEPT_SCOPE({
+                    ALLOW_ALLOCATIONS_IN_SCOPE;
+                    exception_from_job = std::make_exception_ptr(
+                        Exception(ASYNC_LOAD_FAILED,
+                            "Load job '{}' failed: {}",
+                            job->name,
+                            getCurrentExceptionMessage(/* with_stacktrace = */ true)));
+                });
+            }
+        }
+    }
+
+    std::mutex mutex;
+    bool is_running = false;
+    bool is_stopping = false;
+
+    // Full set of scheduled pending jobs along with scheduling info
+    std::unordered_map<LoadJobPtr, Info> scheduled_jobs;
+
+    // Subset of scheduled pending jobs with resolved dependencies (waiting for a thread to be executed)
+    // Represent a queue of jobs in order of decreasing priority and FIFO for jobs with equal priorities
+    std::map<ReadyKey, LoadJobPtr> ready_queue;
+
+    // Set of finished jobs (for introspection only, until job is removed)
+    LoadJobSet finished_jobs;
+
+    // Increasing counter for ReadyKey assignment (to preserve FIFO order of jobs with equal priority)
+    UInt64 last_ready_seqno = 0;
+
+    // For executing jobs. Note that we avoid using an internal queue of the pool to be able to prioritize jobs
+    size_t max_threads;
+    size_t workers = 0;
+    ThreadPool pool;
+};
+
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace CurrentMetrics
+{
+    extern const Metric TablesLoaderThreads;
+    extern const Metric TablesLoaderThreadsActive;
+}
+
+namespace DB
+{
+
+void test()
+{
+    AsyncLoader loader(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, 2);
+
+    auto job_func = [] (const LoadJob & self) {
+        std::cout << self.name << " done with priority " << self.priority << std::endl;
+    };
+
+    auto job1 = makeLoadJob({}, "job1", job_func);
+    auto job2 = makeLoadJob({ job1 }, "job2", job_func);
+    auto task1 = loader.schedule({ job1, job2 });
+
+    auto job3 = makeLoadJob({ job2 }, "job3", job_func);
+    auto job4 = makeLoadJob({ job2 }, "job4", job_func);
+    auto task2 = loader.schedule({ job3, job4 });
+    auto job5 = makeLoadJob({ job3, job4 }, "job5", job_func);
+    task2.merge(loader.schedule({ job5 }, -1));
+}
+
+}
diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp
index 9abf3bba8ff..fe99006eff4 100644
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@@ -650,6 +650,9 @@
     M(679, IO_URING_SUBMIT_ERROR) \
     M(690, MIXED_ACCESS_PARAMETER_TYPES) \
     M(691, UNKNOWN_ELEMENT_OF_ENUM) \
+    M(692, ASYNC_LOAD_FAILED) \
+    M(693, ASYNC_LOAD_CANCELED) \
+    M(694, ASYNC_LOAD_DEPENDENCY_FAILED) \
     \
     M(999, KEEPER_EXCEPTION) \
     M(1000, POCO_EXCEPTION) \

From 5d11706929e38ff6b2aa84161e709d43828cb4db Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Mon, 10 Apr 2023 23:14:20 +0000
Subject: [PATCH 026/689] make t work, add simple test

---
 src/Common/AsyncLoader.h                | 87 ++++++++++---------------
 src/Common/tests/gtest_async_loader.cpp | 66 +++++++++++++++++++
 2 files changed, 102 insertions(+), 51 deletions(-)
 create mode 100644 src/Common/tests/gtest_async_loader.cpp

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index dd35621443d..944251653e4 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -95,7 +95,7 @@ private:
 
     std::function<void(const LoadJob & self)> func;
 
-    std::mutex mutex;
+    mutable std::mutex mutex;
     std::condition_variable finished;
     bool is_finished = false;
     std::exception_ptr exception;
@@ -129,7 +129,7 @@ private:
         ssize_t priority = 0;
         size_t dependencies_left = 0;
         UInt64 ready_seqno = 0; // zero means that job is not in ready queue
-        LoadJobSet dependent_jobs; // TODO(serxa): clean it on remove jobs
+        LoadJobSet dependent_jobs;
 
         ReadyKey key() const
         {
@@ -206,6 +206,7 @@ public:
         , pool(metric_threads, metric_active_threads, max_threads)
     {}
 
+    // Start workers to execute scheduled load jobs
     void start()
     {
         std::unique_lock lock{mutex};
@@ -214,10 +215,24 @@ public:
             spawn(lock);
     }
 
+    // Wait for all load jobs to finish, including all new jobs. So at first take care to stop adding new jobs.
+    void wait()
+    {
+        pool.wait();
+    }
+
+    // Wait for currently executing jobs to finish, but do not run any other pending jobs.
+    // Not finished jobs are left in pending state:
+    //  - they can be resumed by calling start() again;
+    //  - or canceled using ~Task() or remove() later.
     void stop()
     {
-        std::unique_lock lock{mutex};
-        is_stopping = true;
+        {
+            std::unique_lock lock{mutex};
+            is_running = false;
+            // NOTE: there is no need to notify because workers never wait
+        }
+        pool.wait();
     }
 
     [[nodiscard]] Task schedule(LoadJobSet && jobs, ssize_t priority = 0)
@@ -244,7 +259,7 @@ public:
             NOEXCEPT_SCOPE({
                 ALLOW_ALLOCATIONS_IN_SCOPE;
                 scheduled_jobs.emplace(job, Info{.priority = priority});
-            )};
+            });
             job->priority.store(priority); // Set user-facing priority
         }
 
@@ -302,12 +317,12 @@ public:
             else if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
             {
                 if (info->second.dependencies_left > 0) // Job is not ready yet
-                    canceled(job);
+                    canceled(job, lock);
                 else if (info->second.ready_seqno) // Job is enqueued in ready queue
                 {
                     ready_queue.erase(info->second.key());
                     info->second.ready_seqno = 0;
-                    canceled(job);
+                    canceled(job, lock);
                 }
                 else // Job is currently executing
                 {
@@ -321,16 +336,16 @@ public:
     }
 
 private:
-    void canceled(const LoadJobPtr & job, std::unique_lock<std::mutex> &)
+    void canceled(const LoadJobPtr & job, std::unique_lock<std::mutex> & lock)
     {
         std::exception_ptr e;
         NOEXCEPT_SCOPE({
             ALLOW_ALLOCATIONS_IN_SCOPE;
             e = std::make_exception_ptr(
-                Exception(ASYNC_LOAD_CANCELED,
+                Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
                     "Load job '{}' canceled",
                     job->name));
-        )};
+        });
         failed(job, e, lock);
     }
 
@@ -349,7 +364,7 @@ private:
                 enqueue(dep_info, dep, lock);
         }
 
-        finish(job);
+        finish(job, lock);
     }
 
     void failed(const LoadJobPtr & job, std::exception_ptr exception_from_job, std::unique_lock<std::mutex> & lock)
@@ -368,7 +383,7 @@ private:
             NOEXCEPT_SCOPE({
                 ALLOW_ALLOCATIONS_IN_SCOPE;
                 e = std::make_exception_ptr(
-                    Exception(ASYNC_LOAD_DEPENDENCY_FAILED,
+                    Exception(ErrorCodes::ASYNC_LOAD_DEPENDENCY_FAILED,
                         "Load job '{}' -> {}",
                         dep->name,
                         getExceptionMessage(exception_from_job, /* with_stack_trace = */ false)));
@@ -378,11 +393,11 @@ private:
 
         // Clean dependency graph edges
         for (const auto & dep : job->dependencies)
-            if (auto dep_info = scheduled_jobs.find(dep); info != scheduled_jobs.end())
+            if (auto dep_info = scheduled_jobs.find(dep); dep_info != scheduled_jobs.end())
                 dep_info->second.dependent_jobs.erase(job);
 
         // Job became finished
-        finish(job);
+        finish(job, lock);
     }
 
     void finish(const LoadJobPtr & job, std::unique_lock<std::mutex> &)
@@ -391,7 +406,7 @@ private:
         NOEXCEPT_SCOPE({
             ALLOW_ALLOCATIONS_IN_SCOPE;
             finished_jobs.insert(job);
-        )};
+        });
     }
 
     void prioritize(const LoadJobPtr & job, ssize_t new_priority, std::unique_lock<std::mutex> & lock)
@@ -430,7 +445,7 @@ private:
             ready_queue.emplace(info.key(), job);
         });
 
-        if (is_running && workers < max_threads) // TODO(serxa): Can we make max_thread changeable in runtime
+        if (is_running && workers < max_threads) // TODO(serxa): Can we make max_thread changeable in runtime?
             spawn(lock);
     }
 
@@ -450,7 +465,7 @@ private:
 
         LoadJobPtr job;
         std::exception_ptr exception_from_job;
-        while (true) { // TODO(serxa): shutdown
+        while (true) {
 
             /// This is inside the loop to also reset previous thread names set inside the jobs
             setThreadName("AsyncLoader");
@@ -464,6 +479,9 @@ private:
                 else if (job)
                     loaded(job, lock);
 
+                if (!is_running)
+                    return;
+
                 if (ready_queue.empty())
                 {
                     workers--;
@@ -489,7 +507,7 @@ private:
                 NOEXCEPT_SCOPE({
                     ALLOW_ALLOCATIONS_IN_SCOPE;
                     exception_from_job = std::make_exception_ptr(
-                        Exception(ASYNC_LOAD_FAILED,
+                        Exception(ErrorCodes::ASYNC_LOAD_FAILED,
                             "Load job '{}' failed: {}",
                             job->name,
                             getCurrentExceptionMessage(/* with_stacktrace = */ true)));
@@ -500,7 +518,6 @@ private:
 
     std::mutex mutex;
     bool is_running = false;
-    bool is_stopping = false;
 
     // Full set of scheduled pending jobs along with scheduling info
     std::unordered_map<LoadJobPtr, Info> scheduled_jobs;
@@ -522,35 +539,3 @@ private:
 };
 
 }
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-namespace CurrentMetrics
-{
-    extern const Metric TablesLoaderThreads;
-    extern const Metric TablesLoaderThreadsActive;
-}
-
-namespace DB
-{
-
-void test()
-{
-    AsyncLoader loader(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, 2);
-
-    auto job_func = [] (const LoadJob & self) {
-        std::cout << self.name << " done with priority " << self.priority << std::endl;
-    };
-
-    auto job1 = makeLoadJob({}, "job1", job_func);
-    auto job2 = makeLoadJob({ job1 }, "job2", job_func);
-    auto task1 = loader.schedule({ job1, job2 });
-
-    auto job3 = makeLoadJob({ job2 }, "job3", job_func);
-    auto job4 = makeLoadJob({ job2 }, "job4", job_func);
-    auto task2 = loader.schedule({ job3, job4 });
-    auto job5 = makeLoadJob({ job3, job4 }, "job5", job_func);
-    task2.merge(loader.schedule({ job5 }, -1));
-}
-
-}
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
new file mode 100644
index 00000000000..f8d51c0eebd
--- /dev/null
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -0,0 +1,66 @@
+#include <gtest/gtest.h>
+
+#include <vector>
+#include <thread>
+#include <pcg_random.hpp>
+
+#include <base/types.h>
+#include <base/sleep.h>
+#include <Common/AsyncLoader.h>
+#include <Common/randomSeed.h>
+
+
+using namespace DB;
+
+
+namespace CurrentMetrics
+{
+    extern const Metric TablesLoaderThreads;
+    extern const Metric TablesLoaderThreadsActive;
+}
+
+struct AsyncLoaderTest
+{
+    AsyncLoader loader;
+
+    explicit AsyncLoaderTest(size_t max_threads = 1)
+        : loader(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, max_threads)
+    {}
+};
+
+TEST(AsyncLoader, Smoke)
+{
+    AsyncLoaderTest t(2);
+
+    static constexpr ssize_t low_priority = -1;
+
+    std::atomic<size_t> jobs_done{0};
+    std::atomic<size_t> low_priority_jobs_done{0};
+
+    auto job_func = [&] (const LoadJob & self) {
+        jobs_done++;
+        if (self.priority == low_priority)
+            low_priority_jobs_done++;
+    };
+
+    {
+        auto job1 = makeLoadJob({}, "job1", job_func);
+        auto job2 = makeLoadJob({ job1 }, "job2", job_func);
+        auto task1 = t.loader.schedule({ job1, job2 });
+
+        auto job3 = makeLoadJob({ job2 }, "job3", job_func);
+        auto job4 = makeLoadJob({ job2 }, "job4", job_func);
+        auto task2 = t.loader.schedule({ job3, job4 });
+        auto job5 = makeLoadJob({ job3, job4 }, "job5", job_func);
+        task2.merge(t.loader.schedule({ job5 }, low_priority));
+
+        t.loader.start();
+
+        t.loader.wait();
+    }
+
+    ASSERT_EQ(jobs_done, 5);
+    ASSERT_EQ(low_priority_jobs_done, 1);
+
+    t.loader.stop();
+}

From b4318b4ab630138ad187ef3ccf4e26887252954c Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Tue, 11 Apr 2023 00:08:55 +0000
Subject: [PATCH 027/689] fix style

---
 src/Common/AsyncLoader.h | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 944251653e4..8b4fa0bde26 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -112,7 +112,8 @@ class AsyncLoader : private boost::noncopyable
 {
 private:
     // Key of a pending job in ready queue
-    struct ReadyKey {
+    struct ReadyKey
+    {
         ssize_t priority;
         UInt64 ready_seqno;
 
@@ -125,7 +126,8 @@ private:
     };
 
     // Scheduling information for a pending job
-    struct Info {
+    struct Info
+    {
         ssize_t priority = 0;
         size_t dependencies_left = 0;
         UInt64 ready_seqno = 0; // zero means that job is not in ready queue
@@ -465,8 +467,8 @@ private:
 
         LoadJobPtr job;
         std::exception_ptr exception_from_job;
-        while (true) {
-
+        while (true)
+        {
             /// This is inside the loop to also reset previous thread names set inside the jobs
             setThreadName("AsyncLoader");
 

From 2c29c0b2f307939083c1197c86b9ac58e59ec7e4 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Thu, 13 Apr 2023 18:43:58 +0000
Subject: [PATCH 028/689] add dependency cycle detection

---
 src/Common/AsyncLoader.h                | 49 ++++++++++++++++++++++---
 src/Common/ErrorCodes.cpp               |  7 ++--
 src/Common/tests/gtest_async_loader.cpp | 44 ++++++++++++++++++++++
 3 files changed, 91 insertions(+), 9 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 8b4fa0bde26..6ccf3552721 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -28,7 +28,7 @@ class AsyncLoader;
 
 namespace ErrorCodes
 {
-    extern const int LOGICAL_ERROR;
+    extern const int ASYNC_LOAD_SCHEDULE_FAILED;
     extern const int ASYNC_LOAD_FAILED;
     extern const int ASYNC_LOAD_CANCELED;
     extern const int ASYNC_LOAD_DEPENDENCY_FAILED;
@@ -71,7 +71,7 @@ public:
             std::rethrow_exception(exception);
     }
 
-    const LoadJobSet dependencies; // jobs to be done before this one, with ownership
+    const LoadJobSet dependencies; // jobs to be done before this one (with ownership), it is `const` to make creation of cycles hard
     const String name;
     std::atomic<ssize_t> priority{0};
 
@@ -245,12 +245,13 @@ public:
         for (const auto & job : jobs)
         {
             if (job->status() != LoadStatus::PENDING)
-                throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Trying to schedule already finished load job '{}'", job->name);
+                throw Exception(ErrorCodes::ASYNC_LOAD_SCHEDULE_FAILED, "Trying to schedule already finished load job '{}'", job->name);
             if (scheduled_jobs.contains(job))
-                throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Load job '{}' has been already scheduled", job->name);
+                throw Exception(ErrorCodes::ASYNC_LOAD_SCHEDULE_FAILED, "Load job '{}' has been already scheduled", job->name);
         }
 
-        // TODO(serxa): ensure scheduled_jobs graph will have no cycles, otherwise we have infinite recursion and other strange stuff?
+        // Ensure scheduled_jobs graph will have no cycles. The only way to get a cycle is to add a cycle, assuming old jobs cannot reference new ones.
+        checkCycle(jobs, lock);
 
         // We do not want any exception to be throws after this point, because the following code is not exception-safe
         DENY_ALLOCATIONS_IN_SCOPE;
@@ -338,6 +339,41 @@ public:
     }
 
 private:
+    void checkCycle(const LoadJobSet & jobs, std::unique_lock<std::mutex> & lock)
+    {
+        LoadJobSet left = jobs;
+        LoadJobSet visited;
+        visited.reserve(left.size());
+        while (!left.empty())
+        {
+            LoadJobPtr job = *left.begin();
+            checkCycleImpl(job, left, visited, lock);
+        }
+    }
+
+    String checkCycleImpl(const LoadJobPtr & job, LoadJobSet & left, LoadJobSet & visited, std::unique_lock<std::mutex> & lock)
+    {
+        if (!left.contains(job))
+            return {}; // Do not consider external dependencies and already processed jobs
+        if (auto [_, inserted] = visited.insert(job); !inserted)
+        {
+            visited.erase(job); // Mark where cycle ends
+            return job->name;
+        }
+        for (const auto & dep : job->dependencies)
+        {
+            if (auto chain = checkCycleImpl(dep, left, visited, lock); !chain.empty())
+            {
+                if (!visited.contains(job)) // Check for cycle end
+                    throw Exception(ErrorCodes::ASYNC_LOAD_SCHEDULE_FAILED, "Load job dependency cycle detected: {} -> {}", job->name, chain);
+                else
+                    return fmt::format("{} -> {}", job->name, chain); // chain is not a cycle yet -- continue building
+            }
+        }
+        left.erase(job);
+        return {};
+    }
+
     void canceled(const LoadJobPtr & job, std::unique_lock<std::mutex> & lock)
     {
         std::exception_ptr e;
@@ -453,7 +489,6 @@ private:
 
     void spawn(std::unique_lock<std::mutex> &)
     {
-        // TODO(serxa): add metrics for number of active workers or executing jobs
         workers++;
         NOEXCEPT_SCOPE({
             ALLOW_ALLOCATIONS_IN_SCOPE;
@@ -521,6 +556,8 @@ private:
     std::mutex mutex;
     bool is_running = false;
 
+    // TODO(serxa): add metrics for number of jobs in every state
+
     // Full set of scheduled pending jobs along with scheduling info
     std::unordered_map<LoadJobPtr, Info> scheduled_jobs;
 
diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp
index fe99006eff4..0d203fc799c 100644
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@@ -650,9 +650,10 @@
     M(679, IO_URING_SUBMIT_ERROR) \
     M(690, MIXED_ACCESS_PARAMETER_TYPES) \
     M(691, UNKNOWN_ELEMENT_OF_ENUM) \
-    M(692, ASYNC_LOAD_FAILED) \
-    M(693, ASYNC_LOAD_CANCELED) \
-    M(694, ASYNC_LOAD_DEPENDENCY_FAILED) \
+    M(692, ASYNC_LOAD_SCHEDULE_FAILED) \
+    M(693, ASYNC_LOAD_FAILED) \
+    M(694, ASYNC_LOAD_CANCELED) \
+    M(695, ASYNC_LOAD_DEPENDENCY_FAILED) \
     \
     M(999, KEEPER_EXCEPTION) \
     M(1000, POCO_EXCEPTION) \
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index f8d51c0eebd..44c97bf0158 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -64,3 +64,47 @@ TEST(AsyncLoader, Smoke)
 
     t.loader.stop();
 }
+
+TEST(AsyncLoader, CycleDetection)
+{
+    AsyncLoaderTest t;
+
+    auto job_func = [&] (const LoadJob &) {};
+
+    LoadJobPtr cycle_breaker; // To avoid memleak we introduce with a cycle
+
+    try
+    {
+        std::vector<LoadJobPtr> jobs;
+        jobs.push_back(makeLoadJob({}, "job0", job_func));
+        jobs.push_back(makeLoadJob({ jobs[0] }, "job1", job_func));
+        jobs.push_back(makeLoadJob({ jobs[0], jobs[1] }, "job2", job_func));
+        jobs.push_back(makeLoadJob({ jobs[0], jobs[2] }, "job3", job_func));
+
+        // Actually it is hard to construct a cycle, but suppose someone was able to succeed violating constness
+        const_cast<LoadJobSet &>(jobs[1]->dependencies).insert(jobs[3]);
+        cycle_breaker = jobs[1];
+
+        // Add couple unrelated jobs
+        jobs.push_back(makeLoadJob({ jobs[1] }, "job4", job_func));
+        jobs.push_back(makeLoadJob({ jobs[4] }, "job5", job_func));
+        jobs.push_back(makeLoadJob({ jobs[3] }, "job6", job_func));
+        jobs.push_back(makeLoadJob({ jobs[1], jobs[2], jobs[3], jobs[4], jobs[5], jobs[6] }, "job7", job_func));
+
+        // Also add another not connected jobs
+        jobs.push_back(makeLoadJob({}, "job8", job_func));
+        jobs.push_back(makeLoadJob({}, "job9", job_func));
+        jobs.push_back(makeLoadJob({ jobs[9] }, "job10", job_func));
+
+        auto task1 = t.loader.schedule({ jobs.begin(), jobs.end()});
+        FAIL();
+    }
+    catch (Exception & e)
+    {
+        int present[] = { 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0 };
+        for (int i = 0; i < std::size(present); i++)
+            ASSERT_EQ(e.message().find(fmt::format("job{}", i)) != String::npos, present[i]);
+    }
+
+    const_cast<LoadJobSet &>(cycle_breaker->dependencies).clear();
+}

From 3226eadd947a232a6d2f617d3ac9dfe35b888b7c Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Thu, 13 Apr 2023 22:48:46 +0000
Subject: [PATCH 029/689] add simple randomized test

---
 src/Common/AsyncLoader.h                |  6 +++
 src/Common/tests/gtest_async_loader.cpp | 57 +++++++++++++++++++++++++
 2 files changed, 63 insertions(+)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 6ccf3552721..49cecdbbb8e 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -208,6 +208,12 @@ public:
         , pool(metric_threads, metric_active_threads, max_threads)
     {}
 
+    // WARNING: all Task instances returned by `schedule()` should be destructed before AsyncLoader
+    ~AsyncLoader()
+    {
+        stop();
+    }
+
     // Start workers to execute scheduled load jobs
     void start()
     {
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 44c97bf0158..3825f5d2592 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -1,5 +1,7 @@
 #include <gtest/gtest.h>
 
+#include <chrono>
+#include <mutex>
 #include <vector>
 #include <thread>
 #include <pcg_random.hpp>
@@ -23,6 +25,40 @@ struct AsyncLoaderTest
 {
     AsyncLoader loader;
 
+    std::mutex rng_mutex;
+    pcg64 rng{randomSeed()};
+
+    template <typename T>
+    T randomInt(T from, T to)
+    {
+        std::uniform_int_distribution<T> distribution(from, to);
+        std::scoped_lock lock(rng_mutex);
+        return distribution(rng);
+    }
+
+    void randomSleepUs(UInt64 min_us, UInt64 max_us, int probability_percent)
+    {
+        if (randomInt(0, 99) < probability_percent)
+            std::this_thread::sleep_for(std::chrono::microseconds(randomInt(min_us, max_us)));
+    }
+
+    template <typename JobFunc>
+    LoadJobSet randomJobSet(int job_count, int dep_probability_percent, JobFunc job_func)
+    {
+        std::vector<LoadJobPtr> jobs;
+        for (int j = 0; j < job_count; j++)
+        {
+            LoadJobSet deps;
+            for (int d = 0; d < j; d++)
+            {
+                if (randomInt(0, 99) < dep_probability_percent)
+                    deps.insert(jobs[d]);
+            }
+            jobs.push_back(makeLoadJob(std::move(deps), fmt::format("job{}", j), job_func));
+        }
+        return {jobs.begin(), jobs.end()};
+    }
+
     explicit AsyncLoaderTest(size_t max_threads = 1)
         : loader(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, max_threads)
     {}
@@ -108,3 +144,24 @@ TEST(AsyncLoader, CycleDetection)
 
     const_cast<LoadJobSet &>(cycle_breaker->dependencies).clear();
 }
+
+TEST(AsyncLoader, RandomTasks)
+{
+    AsyncLoaderTest t(16);
+    t.loader.start();
+
+    auto job_func = [&] (const LoadJob &)
+    {
+        t.randomSleepUs(100, 500, 5);
+    };
+
+    std::vector<AsyncLoader::Task> tasks;
+    for (int i = 0; i < 512; i++)
+    {
+        int job_count = t.randomInt(1, 32);
+        tasks.push_back(t.loader.schedule(t.randomJobSet(job_count, 5, job_func)));
+        t.randomSleepUs(100, 900, 20); // avg=100us
+    }
+    t.loader.wait();
+}
+

From 5e5a38cd640e71c764afdf7d347d466641144a3f Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Thu, 13 Apr 2023 23:18:38 +0000
Subject: [PATCH 030/689] add tests for job status/wait/cancel

---
 src/Common/AsyncLoader.h                | 15 +++++-----
 src/Common/tests/gtest_async_loader.cpp | 40 +++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 49cecdbbb8e..5f4c67ab065 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -58,12 +58,6 @@ public:
     }
 
     void wait()
-    {
-        std::unique_lock lock{mutex};
-        finished.wait(lock, [this] { return is_finished; });
-    }
-
-    void get()
     {
         std::unique_lock lock{mutex};
         finished.wait(lock, [this] { return is_finished; });
@@ -186,9 +180,16 @@ public:
 
         ~Task()
         {
-            // Remove jobs that are not ready and wait for jobs that are in progress
+            remove();
+        }
+
+        void remove()
+        {
             if (loader)
+            {
                 loader->remove(jobs);
+                detach();
+            }
         }
 
         // Do not track jobs in this task
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 3825f5d2592..7e76ac0e07a 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -21,6 +21,14 @@ namespace CurrentMetrics
     extern const Metric TablesLoaderThreadsActive;
 }
 
+namespace DB::ErrorCodes
+{
+    extern const int ASYNC_LOAD_SCHEDULE_FAILED;
+    extern const int ASYNC_LOAD_FAILED;
+    extern const int ASYNC_LOAD_CANCELED;
+    extern const int ASYNC_LOAD_DEPENDENCY_FAILED;
+}
+
 struct AsyncLoaderTest
 {
     AsyncLoader loader;
@@ -90,9 +98,18 @@ TEST(AsyncLoader, Smoke)
         auto job5 = makeLoadJob({ job3, job4 }, "job5", job_func);
         task2.merge(t.loader.schedule({ job5 }, low_priority));
 
+        std::thread waiter_thread([=] { job5->wait(); });
+
         t.loader.start();
 
+        job3->wait();
         t.loader.wait();
+        job4->wait();
+
+        waiter_thread.join();
+
+        ASSERT_EQ(job1->status(), LoadStatus::SUCCESS);
+        ASSERT_EQ(job2->status(), LoadStatus::SUCCESS);
     }
 
     ASSERT_EQ(jobs_done, 5);
@@ -145,6 +162,29 @@ TEST(AsyncLoader, CycleDetection)
     const_cast<LoadJobSet &>(cycle_breaker->dependencies).clear();
 }
 
+TEST(AsyncLoader, CancelPendingJob)
+{
+    AsyncLoaderTest t;
+
+    auto job_func = [&] (const LoadJob &) {};
+
+    auto job = makeLoadJob({}, "job", job_func);
+    auto task = t.loader.schedule({job});
+
+    task.remove(); // this cancels pending the job (async loader was not started to execute it)
+
+    ASSERT_EQ(job->status(), LoadStatus::FAILED);
+    try
+    {
+        job->wait();
+        FAIL();
+    }
+    catch (Exception & e)
+    {
+        ASSERT_EQ(e.code(), ErrorCodes::ASYNC_LOAD_CANCELED);
+    }
+}
+
 TEST(AsyncLoader, RandomTasks)
 {
     AsyncLoaderTest t(16);

From a9c51c4acabcbe4955731d279831dc38f98b4fcf Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 14 Apr 2023 00:04:31 +0000
Subject: [PATCH 031/689] add more tests for cancel

---
 src/Common/AsyncLoader.h                |  29 +++++--
 src/Common/tests/gtest_async_loader.cpp | 109 +++++++++++++++++++++++-
 2 files changed, 132 insertions(+), 6 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 5f4c67ab065..7bce762bb39 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -57,14 +57,30 @@ public:
         return !is_finished ? LoadStatus::PENDING : (exception ? LoadStatus::FAILED : LoadStatus::SUCCESS);
     }
 
-    void wait()
+    void wait() const
     {
         std::unique_lock lock{mutex};
+        waiters++;
         finished.wait(lock, [this] { return is_finished; });
+        waiters--;
         if (exception)
             std::rethrow_exception(exception);
     }
 
+    void waitNoThrow() const
+    {
+        std::unique_lock lock{mutex};
+        waiters++;
+        finished.wait(lock, [this] { return is_finished; });
+        waiters--;
+    }
+
+    size_t waiters_count() const
+    {
+        std::unique_lock lock{mutex};
+        return waiters;
+    }
+
     const LoadJobSet dependencies; // jobs to be done before this one (with ownership), it is `const` to make creation of cycles hard
     const String name;
     std::atomic<ssize_t> priority{0};
@@ -76,7 +92,8 @@ private:
     {
         std::unique_lock lock{mutex};
         is_finished = true;
-        finished.notify_all();
+        if (waiters > 0)
+            finished.notify_all();
     }
 
     void setFailure(const std::exception_ptr & ptr)
@@ -84,13 +101,15 @@ private:
         std::unique_lock lock{mutex};
         is_finished = true;
         exception = ptr;
-        finished.notify_all();
+        if (waiters > 0)
+            finished.notify_all();
     }
 
     std::function<void(const LoadJob & self)> func;
 
     mutable std::mutex mutex;
-    std::condition_variable finished;
+    mutable std::condition_variable finished;
+    mutable size_t waiters = 0;
     bool is_finished = false;
     std::exception_ptr exception;
 };
@@ -337,7 +356,7 @@ public:
                 else // Job is currently executing
                 {
                     lock.unlock();
-                    job->wait(); // Wait for job to finish
+                    job->waitNoThrow(); // Wait for job to finish
                     lock.lock();
                 }
                 finished_jobs.erase(job);
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 7e76ac0e07a..7ef831cbfd0 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -1,5 +1,6 @@
 #include <gtest/gtest.h>
 
+#include <barrier>
 #include <chrono>
 #include <mutex>
 #include <vector>
@@ -169,7 +170,7 @@ TEST(AsyncLoader, CancelPendingJob)
     auto job_func = [&] (const LoadJob &) {};
 
     auto job = makeLoadJob({}, "job", job_func);
-    auto task = t.loader.schedule({job});
+    auto task = t.loader.schedule({ job });
 
     task.remove(); // this cancels pending the job (async loader was not started to execute it)
 
@@ -185,6 +186,112 @@ TEST(AsyncLoader, CancelPendingJob)
     }
 }
 
+TEST(AsyncLoader, CancelPendingTask)
+{
+    AsyncLoaderTest t;
+
+    auto job_func = [&] (const LoadJob &) {};
+
+    auto job1 = makeLoadJob({}, "job1", job_func);
+    auto job2 = makeLoadJob({ job1 }, "job2", job_func);
+    auto task = t.loader.schedule({ job1, job2 });
+
+    task.remove(); // this cancels both jobs (async loader was not started to execute it)
+
+    ASSERT_EQ(job1->status(), LoadStatus::FAILED);
+    ASSERT_EQ(job2->status(), LoadStatus::FAILED);
+
+    try
+    {
+        job1->wait();
+        FAIL();
+    }
+    catch (Exception & e)
+    {
+        ASSERT_TRUE(e.code() == ErrorCodes::ASYNC_LOAD_CANCELED);
+    }
+
+    try
+    {
+        job2->wait();
+        FAIL();
+    }
+    catch (Exception & e)
+    {
+        // Result depend on non-deterministic cancel order
+        ASSERT_TRUE(e.code() == ErrorCodes::ASYNC_LOAD_CANCELED || e.code() == ErrorCodes::ASYNC_LOAD_DEPENDENCY_FAILED);
+    }
+}
+
+TEST(AsyncLoader, CancelPendingDependency)
+{
+    AsyncLoaderTest t;
+
+    auto job_func = [&] (const LoadJob &) {};
+
+    auto job1 = makeLoadJob({}, "job1", job_func);
+    auto job2 = makeLoadJob({ job1 }, "job2", job_func);
+    auto task1 = t.loader.schedule({ job1 });
+    auto task2 = t.loader.schedule({ job2 });
+
+    task1.remove(); // this cancels both jobs, due to dependency (async loader was not started to execute it)
+
+    ASSERT_EQ(job1->status(), LoadStatus::FAILED);
+    ASSERT_EQ(job2->status(), LoadStatus::FAILED);
+
+    try
+    {
+        job1->wait();
+        FAIL();
+    }
+    catch (Exception & e)
+    {
+        ASSERT_TRUE(e.code() == ErrorCodes::ASYNC_LOAD_CANCELED);
+    }
+
+    try
+    {
+        job2->wait();
+        FAIL();
+    }
+    catch (Exception & e)
+    {
+        ASSERT_TRUE(e.code() == ErrorCodes::ASYNC_LOAD_DEPENDENCY_FAILED);
+    }
+}
+
+TEST(AsyncLoader, CancelExecutingJob)
+{
+    AsyncLoaderTest t;
+    t.loader.start();
+
+    std::barrier sync(2);
+
+    auto job_func = [&] (const LoadJob &)
+    {
+        sync.arrive_and_wait(); // (A) sync with main thread
+        sync.arrive_and_wait(); // (B) wait for waiter
+        // signals (C)
+    };
+
+    auto job = makeLoadJob({}, "job", job_func);
+    auto task = t.loader.schedule({ job });
+
+    sync.arrive_and_wait(); // (A) wait for job to start executing
+    std::thread canceler([&]
+    {
+        task.remove(); // waits for (C)
+    });
+    while (job->waiters_count() == 0)
+        std::this_thread::yield();
+    ASSERT_EQ(job->status(), LoadStatus::PENDING);
+    sync.arrive_and_wait(); // (B) sync with job
+    canceler.join();
+
+    ASSERT_EQ(job->status(), LoadStatus::SUCCESS);
+    job->wait();
+}
+
 TEST(AsyncLoader, RandomTasks)
 {
     AsyncLoaderTest t(16);

From 3be06454134c01282e0c6b4bf58459589de7f9c6 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 14 Apr 2023 01:21:25 +0000
Subject: [PATCH 032/689] fix remove of executing tasks + add test

---
 src/Common/AsyncLoader.h                | 33 +++++++++------
 src/Common/tests/gtest_async_loader.cpp | 56 +++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 12 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 7bce762bb39..999559169ab 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -337,13 +337,12 @@ public:
     {
         DENY_ALLOCATIONS_IN_SCOPE;
         std::unique_lock lock{mutex};
+        // On the first pass:
+        // - cancel all not executing jobs to avoid races
+        // - do not wait executing jobs (otherwise, on unlock a worker could start executing a dependent job, that should be canceled)
         for (const auto & job : jobs)
         {
-            if (auto it = finished_jobs.find(job); it != finished_jobs.end()) // Job is already finished
-            {
-                finished_jobs.erase(it);
-            }
-            else if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
+            if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
             {
                 if (info->second.dependencies_left > 0) // Job is not ready yet
                     canceled(job, lock);
@@ -353,15 +352,25 @@ public:
                     info->second.ready_seqno = 0;
                     canceled(job, lock);
                 }
-                else // Job is currently executing
-                {
-                    lock.unlock();
-                    job->waitNoThrow(); // Wait for job to finish
-                    lock.lock();
-                }
-                finished_jobs.erase(job);
             }
         }
+        // On the second pass wait for executing jobs to finish
+        for (const auto & job : jobs)
+        {
+            if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
+            {
+                // Job is currently executing
+                chassert(info->second.dependencies_left == 0);
+                chassert(info->second.ready_seqno == 0);
+                lock.unlock();
+                job->waitNoThrow(); // Wait for job to finish
+                lock.lock();
+            }
+        }
+        // On the third pass all jobs are finished - remove them all
+        // It is better to do it under one lock to avoid exposing intermediate states
+        for (const auto & job : jobs)
+            finished_jobs.erase(job);
     }
 
 private:
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 7ef831cbfd0..2ba27264460 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -9,6 +9,7 @@
 
 #include <base/types.h>
 #include <base/sleep.h>
+#include <Common/Exception.h>
 #include <Common/AsyncLoader.h>
 #include <Common/randomSeed.h>
 
@@ -292,6 +293,61 @@ TEST(AsyncLoader, CancelExecutingJob)
     job->wait();
 }
 
+TEST(AsyncLoader, CancelExecutingTask)
+{
+    AsyncLoaderTest t(16);
+    t.loader.start();
+    std::barrier sync(2);
+
+    auto blocker_job_func = [&] (const LoadJob &)
+    {
+        sync.arrive_and_wait(); // (A) sync with main thread
+        sync.arrive_and_wait(); // (B) wait for waiter
+        // signals (C)
+    };
+
+    auto job_to_cancel_func = [&] (const LoadJob &)
+    {
+        FAIL(); // this job should be canceled
+    };
+
+    auto job_to_succeed_func = [&] (const LoadJob &)
+    {
+    };
+
+    // Make several iterations to catch the race (if any)
+    for (int iteration = 0; iteration < 10; iteration++) {
+        std::vector<LoadJobPtr> task1_jobs;
+        auto blocker_job = makeLoadJob({}, "blocker_job", blocker_job_func);
+        task1_jobs.push_back(blocker_job);
+        for (int i = 0; i < 100; i++)
+            task1_jobs.push_back(makeLoadJob({ blocker_job }, "job_to_cancel", job_to_cancel_func));
+        auto task1 = t.loader.schedule({ task1_jobs.begin(), task1_jobs.end() });
+        auto job_to_succeed = makeLoadJob({ blocker_job }, "job_to_succeed", job_to_succeed_func);
+        auto task2 = t.loader.schedule({ job_to_succeed });
+
+        sync.arrive_and_wait(); // (A) wait for job to start executing
+        std::thread canceler([&]
+        {
+            task1.remove(); // waits for (C)
+        });
+        while (blocker_job->waiters_count() == 0)
+            std::this_thread::yield();
+        ASSERT_EQ(blocker_job->status(), LoadStatus::PENDING);
+        sync.arrive_and_wait(); // (B) sync with job
+        canceler.join();
+        t.loader.wait();
+
+        ASSERT_EQ(blocker_job->status(), LoadStatus::SUCCESS);
+        ASSERT_EQ(job_to_succeed->status(), LoadStatus::SUCCESS);
+        for (const auto & job : task1_jobs)
+        {
+            if (job != blocker_job)
+                ASSERT_EQ(job->status(), LoadStatus::FAILED);
+        }
+    }
+}
+
 TEST(AsyncLoader, RandomTasks)
 {
     AsyncLoaderTest t(16);

From ba67f95525cc2613269a9b39c617022f248b80c5 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 14 Apr 2023 01:23:40 +0000
Subject: [PATCH 033/689] allow cancels on RandomTasks test shutdown

---
 src/Common/tests/gtest_async_loader.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 2ba27264460..ae0929e6e99 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -365,6 +365,5 @@ TEST(AsyncLoader, RandomTasks)
         tasks.push_back(t.loader.schedule(t.randomJobSet(job_count, 5, job_func)));
         t.randomSleepUs(100, 900, 20); // avg=100us
     }
-    t.loader.wait();
 }
 

From 4bc52a1b06976912c8cacee948b1ebebc8cbb9f4 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 14 Apr 2023 12:46:09 +0000
Subject: [PATCH 034/689] add concurrency test

---
 src/Common/tests/gtest_async_loader.cpp | 44 ++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 5 deletions(-)

diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index ae0929e6e99..4a86938817c 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -38,6 +38,10 @@ struct AsyncLoaderTest
     std::mutex rng_mutex;
     pcg64 rng{randomSeed()};
 
+    explicit AsyncLoaderTest(size_t max_threads = 1)
+        : loader(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, max_threads)
+    {}
+
     template <typename T>
     T randomInt(T from, T to)
     {
@@ -53,7 +57,7 @@ struct AsyncLoaderTest
     }
 
     template <typename JobFunc>
-    LoadJobSet randomJobSet(int job_count, int dep_probability_percent, JobFunc job_func)
+    LoadJobSet randomJobSet(int job_count, int dep_probability_percent, JobFunc job_func, std::string_view name_prefix = "job")
     {
         std::vector<LoadJobPtr> jobs;
         for (int j = 0; j < job_count; j++)
@@ -64,14 +68,20 @@ struct AsyncLoaderTest
                 if (randomInt(0, 99) < dep_probability_percent)
                     deps.insert(jobs[d]);
             }
-            jobs.push_back(makeLoadJob(std::move(deps), fmt::format("job{}", j), job_func));
+            jobs.push_back(makeLoadJob(std::move(deps), fmt::format("{}{}", name_prefix, j), job_func));
         }
         return {jobs.begin(), jobs.end()};
     }
 
-    explicit AsyncLoaderTest(size_t max_threads = 1)
-        : loader(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, max_threads)
-    {}
+    template <typename JobFunc>
+    LoadJobSet chainJobSet(int job_count, JobFunc job_func, std::string_view name_prefix = "job")
+    {
+        std::vector<LoadJobPtr> jobs;
+        jobs.push_back(makeLoadJob({}, fmt::format("{}{}", name_prefix, 0), job_func));
+        for (int j = 1; j < job_count; j++)
+            jobs.push_back(makeLoadJob({ jobs[j - 1] }, fmt::format("{}{}", name_prefix, j), job_func));
+        return {jobs.begin(), jobs.end()};
+    }
 };
 
 TEST(AsyncLoader, Smoke)
@@ -367,3 +377,27 @@ TEST(AsyncLoader, RandomTasks)
     }
 }
 
+TEST(AsyncLoader, TestConcurrency)
+{
+    AsyncLoaderTest t(10);
+    t.loader.start();
+
+    for (int concurrency = 1; concurrency <= 10; concurrency++)
+    {
+        std::barrier sync(concurrency);
+
+        std::atomic<int> executing{0};
+        auto job_func = [&] (const LoadJob &)
+        {
+            executing++;
+            ASSERT_LE(executing, concurrency);
+            sync.arrive_and_wait();
+            executing--;
+        };
+
+        std::vector<AsyncLoader::Task> tasks;
+        for (int i = 0; i < concurrency; i++)
+            tasks.push_back(t.loader.schedule(t.chainJobSet(5, job_func)));
+        t.loader.wait();
+    }
+}

From 9ac37ca3b85ac17906e2c5d8387ae3deb4f07f1c Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 14 Apr 2023 12:57:51 +0000
Subject: [PATCH 035/689] add  overload test

---
 src/Common/AsyncLoader.h                |  8 ++++++-
 src/Common/tests/gtest_async_loader.cpp | 29 +++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 999559169ab..9467a705c10 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -373,6 +373,12 @@ public:
             finished_jobs.erase(job);
     }
 
+    size_t getMaxThreads() const
+    {
+        std::unique_lock lock{mutex};
+        return pool.getMaxThreads();
+    }
+
 private:
     void checkCycle(const LoadJobSet & jobs, std::unique_lock<std::mutex> & lock)
     {
@@ -588,7 +594,7 @@ private:
         }
     }
 
-    std::mutex mutex;
+    mutable std::mutex mutex;
     bool is_running = false;
 
     // TODO(serxa): add metrics for number of jobs in every state
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 4a86938817c..68de24460bb 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -399,5 +399,34 @@ TEST(AsyncLoader, TestConcurrency)
         for (int i = 0; i < concurrency; i++)
             tasks.push_back(t.loader.schedule(t.chainJobSet(5, job_func)));
         t.loader.wait();
+        ASSERT_EQ(executing, 0);
+    }
+}
+
+TEST(AsyncLoader, TestOverload)
+{
+    AsyncLoaderTest t(3);
+    t.loader.start();
+
+    size_t max_threads = t.loader.getMaxThreads();
+    std::atomic<int> executing{0};
+
+    for (int concurrency = 4; concurrency <= 8; concurrency++)
+    {
+        auto job_func = [&] (const LoadJob &)
+        {
+            executing++;
+            t.randomSleepUs(100, 200, 100);
+            ASSERT_LE(executing, max_threads);
+            executing--;
+        };
+
+        t.loader.stop();
+        std::vector<AsyncLoader::Task> tasks;
+        for (int i = 0; i < concurrency; i++)
+            tasks.push_back(t.loader.schedule(t.chainJobSet(5, job_func)));
+        t.loader.start();
+        t.loader.wait();
+        ASSERT_EQ(executing, 0);
     }
 }

From df3c5212ff24f091d157c10e1b04c25697f6a470 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 14 Apr 2023 13:14:31 +0000
Subject: [PATCH 036/689] test deps are completed + refactoring

---
 src/Common/AsyncLoader.h                |  6 ++-
 src/Common/tests/gtest_async_loader.cpp | 64 +++++++++++++------------
 2 files changed, 37 insertions(+), 33 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 9467a705c10..96ea6165d34 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -105,7 +105,9 @@ private:
             finished.notify_all();
     }
 
-    std::function<void(const LoadJob & self)> func;
+    // TODO(serxa): add callback/status for cancel?
+
+    std::function<void(const LoadJobPtr & self)> func;
 
     mutable std::mutex mutex;
     mutable std::condition_variable finished;
@@ -576,7 +578,7 @@ private:
             try
             {
                 ALLOW_ALLOCATIONS_IN_SCOPE;
-                job->func(*job);
+                job->func(job);
                 exception_from_job = {};
             }
             catch (...)
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 68de24460bb..d913f8f6362 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -93,9 +93,9 @@ TEST(AsyncLoader, Smoke)
     std::atomic<size_t> jobs_done{0};
     std::atomic<size_t> low_priority_jobs_done{0};
 
-    auto job_func = [&] (const LoadJob & self) {
+    auto job_func = [&] (const LoadJobPtr & self) {
         jobs_done++;
-        if (self.priority == low_priority)
+        if (self->priority == low_priority)
             low_priority_jobs_done++;
     };
 
@@ -134,7 +134,7 @@ TEST(AsyncLoader, CycleDetection)
 {
     AsyncLoaderTest t;
 
-    auto job_func = [&] (const LoadJob &) {};
+    auto job_func = [&] (const LoadJobPtr &) {};
 
     LoadJobPtr cycle_breaker; // To avoid memleak we introduce with a cycle
 
@@ -178,7 +178,7 @@ TEST(AsyncLoader, CancelPendingJob)
 {
     AsyncLoaderTest t;
 
-    auto job_func = [&] (const LoadJob &) {};
+    auto job_func = [&] (const LoadJobPtr &) {};
 
     auto job = makeLoadJob({}, "job", job_func);
     auto task = t.loader.schedule({ job });
@@ -201,7 +201,7 @@ TEST(AsyncLoader, CancelPendingTask)
 {
     AsyncLoaderTest t;
 
-    auto job_func = [&] (const LoadJob &) {};
+    auto job_func = [&] (const LoadJobPtr &) {};
 
     auto job1 = makeLoadJob({}, "job1", job_func);
     auto job2 = makeLoadJob({ job1 }, "job2", job_func);
@@ -238,7 +238,7 @@ TEST(AsyncLoader, CancelPendingDependency)
 {
     AsyncLoaderTest t;
 
-    auto job_func = [&] (const LoadJob &) {};
+    auto job_func = [&] (const LoadJobPtr &) {};
 
     auto job1 = makeLoadJob({}, "job1", job_func);
     auto job2 = makeLoadJob({ job1 }, "job2", job_func);
@@ -278,7 +278,7 @@ TEST(AsyncLoader, CancelExecutingJob)
 
     std::barrier sync(2);
 
-    auto job_func = [&] (const LoadJob &)
+    auto job_func = [&] (const LoadJobPtr &)
     {
         sync.arrive_and_wait(); // (A) sync with main thread
         sync.arrive_and_wait(); // (B) wait for waiter
@@ -309,19 +309,19 @@ TEST(AsyncLoader, CancelExecutingTask)
     t.loader.start();
     std::barrier sync(2);
 
-    auto blocker_job_func = [&] (const LoadJob &)
+    auto blocker_job_func = [&] (const LoadJobPtr &)
     {
         sync.arrive_and_wait(); // (A) sync with main thread
         sync.arrive_and_wait(); // (B) wait for waiter
         // signals (C)
     };
 
-    auto job_to_cancel_func = [&] (const LoadJob &)
+    auto job_to_cancel_func = [&] (const LoadJobPtr &)
     {
         FAIL(); // this job should be canceled
     };
 
-    auto job_to_succeed_func = [&] (const LoadJob &)
+    auto job_to_succeed_func = [&] (const LoadJobPtr &)
     {
     };
 
@@ -358,25 +358,6 @@ TEST(AsyncLoader, CancelExecutingTask)
     }
 }
 
-TEST(AsyncLoader, RandomTasks)
-{
-    AsyncLoaderTest t(16);
-    t.loader.start();
-
-    auto job_func = [&] (const LoadJob &)
-    {
-        t.randomSleepUs(100, 500, 5);
-    };
-
-    std::vector<AsyncLoader::Task> tasks;
-    for (int i = 0; i < 512; i++)
-    {
-        int job_count = t.randomInt(1, 32);
-        tasks.push_back(t.loader.schedule(t.randomJobSet(job_count, 5, job_func)));
-        t.randomSleepUs(100, 900, 20); // avg=100us
-    }
-}
-
 TEST(AsyncLoader, TestConcurrency)
 {
     AsyncLoaderTest t(10);
@@ -387,7 +368,7 @@ TEST(AsyncLoader, TestConcurrency)
         std::barrier sync(concurrency);
 
         std::atomic<int> executing{0};
-        auto job_func = [&] (const LoadJob &)
+        auto job_func = [&] (const LoadJobPtr &)
         {
             executing++;
             ASSERT_LE(executing, concurrency);
@@ -413,7 +394,7 @@ TEST(AsyncLoader, TestOverload)
 
     for (int concurrency = 4; concurrency <= 8; concurrency++)
     {
-        auto job_func = [&] (const LoadJob &)
+        auto job_func = [&] (const LoadJobPtr &)
         {
             executing++;
             t.randomSleepUs(100, 200, 100);
@@ -430,3 +411,24 @@ TEST(AsyncLoader, TestOverload)
         ASSERT_EQ(executing, 0);
     }
 }
+
+TEST(AsyncLoader, RandomTasks)
+{
+    AsyncLoaderTest t(16);
+    t.loader.start();
+
+    auto job_func = [&] (const LoadJobPtr & self)
+    {
+        for (const auto & dep : self->dependencies)
+            ASSERT_EQ(dep->status(), LoadStatus::SUCCESS);
+        t.randomSleepUs(100, 500, 5);
+    };
+
+    std::vector<AsyncLoader::Task> tasks;
+    for (int i = 0; i < 512; i++)
+    {
+        int job_count = t.randomInt(1, 32);
+        tasks.push_back(t.loader.schedule(t.randomJobSet(job_count, 5, job_func)));
+        t.randomSleepUs(100, 900, 20); // avg=100us
+    }
+}

From 5ba1dc91e333ae824f1f0197442948757e82651e Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 14 Apr 2023 14:00:54 +0000
Subject: [PATCH 037/689] add LoadStatus::CANCELED

---
 src/Common/AsyncLoader.h                | 96 ++++++++++++++-----------
 src/Common/tests/gtest_async_loader.cpp | 24 +++----
 2 files changed, 67 insertions(+), 53 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 96ea6165d34..6af1b79bec8 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -36,9 +36,10 @@ namespace ErrorCodes
 
 enum class LoadStatus
 {
-    PENDING, // Load is not finished yet
-    SUCCESS, // Load was successful
-    FAILED   // Load failed or canceled
+    PENDING,  // Load job is not started yet
+    OK,       // Load job executed and was successful
+    FAILED,   // Load job executed and failed
+    CANCELED  // Load job is not going to be executed due to removal or dependency failure
 };
 
 class LoadJob : private boost::noncopyable
@@ -54,14 +55,14 @@ public:
     LoadStatus status() const
     {
         std::unique_lock lock{mutex};
-        return !is_finished ? LoadStatus::PENDING : (exception ? LoadStatus::FAILED : LoadStatus::SUCCESS);
+        return load_status;
     }
 
     void wait() const
     {
         std::unique_lock lock{mutex};
         waiters++;
-        finished.wait(lock, [this] { return is_finished; });
+        finished.wait(lock, [this] { return load_status != LoadStatus::PENDING; });
         waiters--;
         if (exception)
             std::rethrow_exception(exception);
@@ -71,7 +72,7 @@ public:
     {
         std::unique_lock lock{mutex};
         waiters++;
-        finished.wait(lock, [this] { return is_finished; });
+        finished.wait(lock, [this] { return load_status != LoadStatus::PENDING; });
         waiters--;
     }
 
@@ -88,18 +89,27 @@ public:
 private:
     friend class AsyncLoader;
 
-    void setSuccess()
+    void ok()
     {
         std::unique_lock lock{mutex};
-        is_finished = true;
+        load_status = LoadStatus::OK;
         if (waiters > 0)
             finished.notify_all();
     }
 
-    void setFailure(const std::exception_ptr & ptr)
+    void failed(const std::exception_ptr & ptr)
     {
         std::unique_lock lock{mutex};
-        is_finished = true;
+        load_status = LoadStatus::FAILED;
+        exception = ptr;
+        if (waiters > 0)
+            finished.notify_all();
+    }
+
+    void canceled(const std::exception_ptr & ptr)
+    {
+        std::unique_lock lock{mutex};
+        load_status = LoadStatus::CANCELED;
         exception = ptr;
         if (waiters > 0)
             finished.notify_all();
@@ -112,7 +122,7 @@ private:
     mutable std::mutex mutex;
     mutable std::condition_variable finished;
     mutable size_t waiters = 0;
-    bool is_finished = false;
+    LoadStatus load_status{LoadStatus::PENDING};
     std::exception_ptr exception;
 };
 
@@ -148,6 +158,12 @@ private:
         UInt64 ready_seqno = 0; // zero means that job is not in ready queue
         LoadJobSet dependent_jobs;
 
+        // Three independent states of a non-finished jobs
+        bool is_blocked() const { return dependencies_left > 0; }
+        bool is_ready() const { return dependencies_left == 0 && ready_seqno > 0; }
+        bool is_executing() const { return dependencies_left == 0 && ready_seqno == 0; }
+
+        // Get key of a ready job
         ReadyKey key() const
         {
             return {priority, ready_seqno};
@@ -318,8 +334,8 @@ public:
                 prioritize(dep, priority, lock);
             }
 
-            // Place jobs w/o dependencies to ready queue
-            if (info.dependencies_left == 0)
+            // Enqueue non-blocked jobs (w/o dependencies) to ready queue
+            if (!info.is_blocked())
                 enqueue(info, job, lock);
         }
 
@@ -346,14 +362,22 @@ public:
         {
             if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
             {
-                if (info->second.dependencies_left > 0) // Job is not ready yet
-                    canceled(job, lock);
-                else if (info->second.ready_seqno) // Job is enqueued in ready queue
+                if (info->second.is_executing())
+                    continue; // Skip executing jobs on the first pass
+                if (info->second.is_ready())
                 {
                     ready_queue.erase(info->second.key());
                     info->second.ready_seqno = 0;
-                    canceled(job, lock);
                 }
+                std::exception_ptr e;
+                NOEXCEPT_SCOPE({
+                    ALLOW_ALLOCATIONS_IN_SCOPE;
+                    e = std::make_exception_ptr(
+                        Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
+                            "Load job '{}' canceled",
+                            job->name));
+                });
+                markNotOk(job, e, LoadStatus::CANCELED, lock);
             }
         }
         // On the second pass wait for executing jobs to finish
@@ -362,8 +386,7 @@ public:
             if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
             {
                 // Job is currently executing
-                chassert(info->second.dependencies_left == 0);
-                chassert(info->second.ready_seqno == 0);
+                chassert(info->second.is_executing());
                 lock.unlock();
                 job->waitNoThrow(); // Wait for job to finish
                 lock.lock();
@@ -417,23 +440,10 @@ private:
         return {};
     }
 
-    void canceled(const LoadJobPtr & job, std::unique_lock<std::mutex> & lock)
-    {
-        std::exception_ptr e;
-        NOEXCEPT_SCOPE({
-            ALLOW_ALLOCATIONS_IN_SCOPE;
-            e = std::make_exception_ptr(
-                Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
-                    "Load job '{}' canceled",
-                    job->name));
-        });
-        failed(job, e, lock);
-    }
-
-    void loaded(const LoadJobPtr & job, std::unique_lock<std::mutex> & lock)
+    void markOk(const LoadJobPtr & job, std::unique_lock<std::mutex> & lock)
     {
         // Notify waiters
-        job->setSuccess();
+        job->ok();
 
         // Update dependent jobs and enqueue if ready
         chassert(scheduled_jobs.contains(job)); // Job was pending
@@ -441,17 +451,21 @@ private:
         {
             chassert(scheduled_jobs.contains(dep)); // All depended jobs must be pending
             Info & dep_info = scheduled_jobs[dep];
-            if (--dep_info.dependencies_left == 0)
+            dep_info.dependencies_left--;
+            if (!dep_info.is_blocked())
                 enqueue(dep_info, dep, lock);
         }
 
         finish(job, lock);
     }
 
-    void failed(const LoadJobPtr & job, std::exception_ptr exception_from_job, std::unique_lock<std::mutex> & lock)
+    void markNotOk(const LoadJobPtr & job, std::exception_ptr exception_from_job, LoadStatus status, std::unique_lock<std::mutex> & lock)
     {
         // Notify waiters
-        job->setFailure(exception_from_job);
+        if (status == LoadStatus::FAILED)
+            job->failed(exception_from_job);
+        else if (status == LoadStatus::CANCELED)
+            job->canceled(exception_from_job);
 
         // Recurse into all dependent jobs
         chassert(scheduled_jobs.contains(job)); // Job was pending
@@ -469,7 +483,7 @@ private:
                         dep->name,
                         getExceptionMessage(exception_from_job, /* with_stack_trace = */ false)));
             });
-            failed(dep, e, lock);
+            markNotOk(dep, e, LoadStatus::CANCELED, lock);
         }
 
         // Clean dependency graph edges
@@ -518,7 +532,7 @@ private:
 
     void enqueue(Info & info, const LoadJobPtr & job, std::unique_lock<std::mutex> & lock)
     {
-        chassert(info.dependencies_left == 0);
+        chassert(!info.is_blocked());
         chassert(info.ready_seqno == 0);
         info.ready_seqno = ++last_ready_seqno;
         NOEXCEPT_SCOPE({
@@ -555,9 +569,9 @@ private:
 
                 // Handle just executed job
                 if (exception_from_job)
-                    failed(job, exception_from_job, lock);
+                    markNotOk(job, exception_from_job, LoadStatus::FAILED, lock);
                 else if (job)
-                    loaded(job, lock);
+                    markOk(job, lock);
 
                 if (!is_running)
                     return;
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index d913f8f6362..d7706311fa4 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -120,8 +120,8 @@ TEST(AsyncLoader, Smoke)
 
         waiter_thread.join();
 
-        ASSERT_EQ(job1->status(), LoadStatus::SUCCESS);
-        ASSERT_EQ(job2->status(), LoadStatus::SUCCESS);
+        ASSERT_EQ(job1->status(), LoadStatus::OK);
+        ASSERT_EQ(job2->status(), LoadStatus::OK);
     }
 
     ASSERT_EQ(jobs_done, 5);
@@ -185,7 +185,7 @@ TEST(AsyncLoader, CancelPendingJob)
 
     task.remove(); // this cancels pending the job (async loader was not started to execute it)
 
-    ASSERT_EQ(job->status(), LoadStatus::FAILED);
+    ASSERT_EQ(job->status(), LoadStatus::CANCELED);
     try
     {
         job->wait();
@@ -209,8 +209,8 @@ TEST(AsyncLoader, CancelPendingTask)
 
     task.remove(); // this cancels both jobs (async loader was not started to execute it)
 
-    ASSERT_EQ(job1->status(), LoadStatus::FAILED);
-    ASSERT_EQ(job2->status(), LoadStatus::FAILED);
+    ASSERT_EQ(job1->status(), LoadStatus::CANCELED);
+    ASSERT_EQ(job2->status(), LoadStatus::CANCELED);
 
     try
     {
@@ -247,8 +247,8 @@ TEST(AsyncLoader, CancelPendingDependency)
 
     task1.remove(); // this cancels both jobs, due to dependency (async loader was not started to execute it)
 
-    ASSERT_EQ(job1->status(), LoadStatus::FAILED);
-    ASSERT_EQ(job2->status(), LoadStatus::FAILED);
+    ASSERT_EQ(job1->status(), LoadStatus::CANCELED);
+    ASSERT_EQ(job2->status(), LoadStatus::CANCELED);
 
     try
     {
@@ -299,7 +299,7 @@ TEST(AsyncLoader, CancelExecutingJob)
     sync.arrive_and_wait(); // (B) sync with job
     canceler.join();
 
-    ASSERT_EQ(job->status(), LoadStatus::SUCCESS);
+    ASSERT_EQ(job->status(), LoadStatus::OK);
     job->wait();
 }
 
@@ -348,12 +348,12 @@ TEST(AsyncLoader, CancelExecutingTask)
         canceler.join();
         t.loader.wait();
 
-        ASSERT_EQ(blocker_job->status(), LoadStatus::SUCCESS);
-        ASSERT_EQ(job_to_succeed->status(), LoadStatus::SUCCESS);
+        ASSERT_EQ(blocker_job->status(), LoadStatus::OK);
+        ASSERT_EQ(job_to_succeed->status(), LoadStatus::OK);
         for (const auto & job : task1_jobs)
         {
             if (job != blocker_job)
-                ASSERT_EQ(job->status(), LoadStatus::FAILED);
+                ASSERT_EQ(job->status(), LoadStatus::CANCELED);
         }
     }
 }
@@ -420,7 +420,7 @@ TEST(AsyncLoader, RandomTasks)
     auto job_func = [&] (const LoadJobPtr & self)
     {
         for (const auto & dep : self->dependencies)
-            ASSERT_EQ(dep->status(), LoadStatus::SUCCESS);
+            ASSERT_EQ(dep->status(), LoadStatus::OK);
         t.randomSleepUs(100, 500, 5);
     };
 

From 5837b09880365f88e433c6abb58964fe8e1f4624 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 14 Apr 2023 17:16:30 +0000
Subject: [PATCH 038/689] fix scheduling of jobs with dependencies on finished
 jobs

---
 src/Common/AsyncLoader.h                | 211 +++++++++++++++---------
 src/Common/ErrorCodes.cpp               |   1 -
 src/Common/tests/gtest_async_loader.cpp |  86 +++++++++-
 3 files changed, 211 insertions(+), 87 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 6af1b79bec8..84b709f14a1 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -31,7 +31,6 @@ namespace ErrorCodes
     extern const int ASYNC_LOAD_SCHEDULE_FAILED;
     extern const int ASYNC_LOAD_FAILED;
     extern const int ASYNC_LOAD_CANCELED;
-    extern const int ASYNC_LOAD_DEPENDENCY_FAILED;
 }
 
 enum class LoadStatus
@@ -58,14 +57,20 @@ public:
         return load_status;
     }
 
+    std::exception_ptr exception() const
+    {
+        std::unique_lock lock{mutex};
+        return load_exception;
+    }
+
     void wait() const
     {
         std::unique_lock lock{mutex};
         waiters++;
         finished.wait(lock, [this] { return load_status != LoadStatus::PENDING; });
         waiters--;
-        if (exception)
-            std::rethrow_exception(exception);
+        if (load_exception)
+            std::rethrow_exception(load_exception);
     }
 
     void waitNoThrow() const
@@ -101,7 +106,7 @@ private:
     {
         std::unique_lock lock{mutex};
         load_status = LoadStatus::FAILED;
-        exception = ptr;
+        load_exception = ptr;
         if (waiters > 0)
             finished.notify_all();
     }
@@ -110,7 +115,7 @@ private:
     {
         std::unique_lock lock{mutex};
         load_status = LoadStatus::CANCELED;
-        exception = ptr;
+        load_exception = ptr;
         if (waiters > 0)
             finished.notify_all();
     }
@@ -123,7 +128,7 @@ private:
     mutable std::condition_variable finished;
     mutable size_t waiters = 0;
     LoadStatus load_status{LoadStatus::PENDING};
-    std::exception_ptr exception;
+    std::exception_ptr load_exception;
 };
 
 template <class Func>
@@ -241,8 +246,9 @@ public:
         LoadJobSet jobs;
     };
 
-    AsyncLoader(Metric metric_threads, Metric metric_active_threads, size_t max_threads_)
-        : max_threads(max_threads_)
+    AsyncLoader(Metric metric_threads, Metric metric_active_threads, size_t max_threads_, bool log_failures_)
+        : log_failures(log_failures_)
+        , max_threads(max_threads_)
         , pool(metric_threads, metric_active_threads, max_threads)
     {}
 
@@ -310,7 +316,7 @@ public:
             job->priority.store(priority); // Set user-facing priority
         }
 
-        // Process incoming dependencies
+        // Process dependencies on scheduled pending jobs
         for (const auto & job : jobs)
         {
             Info & info = scheduled_jobs.find(job)->second;
@@ -324,14 +330,10 @@ public:
                         dep_info->second.dependent_jobs.insert(job);
                     });
                     info.dependencies_left++;
-                }
-                else
-                {
-                    // TODO(serxa): check status: (1) pending: it is wrong - throw? (2) success: good - no dep. (3) failed: propagate failure!
-                }
 
-                // Priority inheritance: prioritize deps to have at least given `priority` to avoid priority inversion
-                prioritize(dep, priority, lock);
+                    // Priority inheritance: prioritize deps to have at least given `priority` to avoid priority inversion
+                    prioritize(dep, priority, lock);
+                }
             }
 
             // Enqueue non-blocked jobs (w/o dependencies) to ready queue
@@ -339,6 +341,57 @@ public:
                 enqueue(info, job, lock);
         }
 
+        // Process dependencies on other jobs. It is done in a separate pass to facilitate propagation of cancel signals (if any).
+        for (const auto & job : jobs)
+        {
+            if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
+            {
+                for (const auto & dep : job->dependencies)
+                {
+                    if (scheduled_jobs.contains(dep))
+                        continue; // Skip dependencies on scheduled pending jobs (already processed)
+                    LoadStatus dep_status = dep->status();
+                    if (dep_status == LoadStatus::OK)
+                        continue; // Dependency on already successfully finished job -- it's okay.
+
+                    if (dep_status == LoadStatus::PENDING)
+                    {
+                        // Dependency on not scheduled pending job -- it's bad.
+                        // Probably, there is an error in `jobs` set: not all jobs were passed to `schedule()` call.
+                        // We are not going to run any dependent job, so cancel them all.
+                        std::exception_ptr e;
+                        NOEXCEPT_SCOPE({
+                            ALLOW_ALLOCATIONS_IN_SCOPE;
+                            e = std::make_exception_ptr(Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
+                                "Load job '{}' -> Load job '{}': not scheduled pending load job (it must be also scheduled), all dependent load jobs are canceled",
+                                job->name,
+                                dep->name));
+                        });
+                        finish(lock, job, LoadStatus::CANCELED, e);
+                        break; // This job is now finished, stop its dependencies processing
+                    }
+                    if (dep_status == LoadStatus::FAILED || dep_status == LoadStatus::CANCELED)
+                    {
+                        // Dependency on already failed or canceled job -- it's okay. Cancel all dependent jobs.
+                        std::exception_ptr e;
+                        NOEXCEPT_SCOPE({
+                            ALLOW_ALLOCATIONS_IN_SCOPE;
+                            e = std::make_exception_ptr(Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
+                                "Load job '{}' -> {}",
+                                job->name,
+                                getExceptionMessage(dep->exception(), /* with_stack_trace = */ false)));
+                        });
+                        finish(lock, job, LoadStatus::CANCELED, e);
+                        break; // This job is now finished, stop its dependencies processing
+                    }
+                }
+            }
+            else
+            {
+                // Job was already canceled on previous iteration of this cycle -- skip
+            }
+        }
+
         return Task(this, std::move(jobs));
     }
 
@@ -364,20 +417,12 @@ public:
             {
                 if (info->second.is_executing())
                     continue; // Skip executing jobs on the first pass
-                if (info->second.is_ready())
-                {
-                    ready_queue.erase(info->second.key());
-                    info->second.ready_seqno = 0;
-                }
                 std::exception_ptr e;
                 NOEXCEPT_SCOPE({
                     ALLOW_ALLOCATIONS_IN_SCOPE;
-                    e = std::make_exception_ptr(
-                        Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
-                            "Load job '{}' canceled",
-                            job->name));
+                    e = std::make_exception_ptr(Exception(ErrorCodes::ASYNC_LOAD_CANCELED, "Load job '{}' canceled", job->name));
                 });
-                markNotOk(job, e, LoadStatus::CANCELED, lock);
+                finish(lock, job, LoadStatus::CANCELED, e);
             }
         }
         // On the second pass wait for executing jobs to finish
@@ -440,63 +485,64 @@ private:
         return {};
     }
 
-    void markOk(const LoadJobPtr & job, std::unique_lock<std::mutex> & lock)
+    void finish(std::unique_lock<std::mutex> & lock, const LoadJobPtr & job, LoadStatus status, std::exception_ptr exception_from_job = {})
     {
-        // Notify waiters
-        job->ok();
-
-        // Update dependent jobs and enqueue if ready
-        chassert(scheduled_jobs.contains(job)); // Job was pending
-        for (const auto & dep : scheduled_jobs[job].dependent_jobs)
+        if (status == LoadStatus::OK)
         {
-            chassert(scheduled_jobs.contains(dep)); // All depended jobs must be pending
-            Info & dep_info = scheduled_jobs[dep];
-            dep_info.dependencies_left--;
-            if (!dep_info.is_blocked())
-                enqueue(dep_info, dep, lock);
+            // Notify waiters
+            job->ok();
+
+            // Update dependent jobs and enqueue if ready
+            chassert(scheduled_jobs.contains(job)); // Job was pending
+            for (const auto & dep : scheduled_jobs[job].dependent_jobs)
+            {
+                chassert(scheduled_jobs.contains(dep)); // All depended jobs must be pending
+                Info & dep_info = scheduled_jobs[dep];
+                dep_info.dependencies_left--;
+                if (!dep_info.is_blocked())
+                    enqueue(dep_info, dep, lock);
+            }
         }
-
-        finish(job, lock);
-    }
-
-    void markNotOk(const LoadJobPtr & job, std::exception_ptr exception_from_job, LoadStatus status, std::unique_lock<std::mutex> & lock)
-    {
-        // Notify waiters
-        if (status == LoadStatus::FAILED)
-            job->failed(exception_from_job);
-        else if (status == LoadStatus::CANCELED)
-            job->canceled(exception_from_job);
-
-        // Recurse into all dependent jobs
-        chassert(scheduled_jobs.contains(job)); // Job was pending
-        Info & info = scheduled_jobs[job];
-        LoadJobSet dependent;
-        dependent.swap(info.dependent_jobs); // To avoid container modification during recursion
-        for (const auto & dep : dependent)
+        else
         {
-            std::exception_ptr e;
-            NOEXCEPT_SCOPE({
-                ALLOW_ALLOCATIONS_IN_SCOPE;
-                e = std::make_exception_ptr(
-                    Exception(ErrorCodes::ASYNC_LOAD_DEPENDENCY_FAILED,
-                        "Load job '{}' -> {}",
-                        dep->name,
-                        getExceptionMessage(exception_from_job, /* with_stack_trace = */ false)));
-            });
-            markNotOk(dep, e, LoadStatus::CANCELED, lock);
-        }
+            // Notify waiters
+            if (status == LoadStatus::FAILED)
+                job->failed(exception_from_job);
+            else if (status == LoadStatus::CANCELED)
+                job->canceled(exception_from_job);
 
-        // Clean dependency graph edges
-        for (const auto & dep : job->dependencies)
-            if (auto dep_info = scheduled_jobs.find(dep); dep_info != scheduled_jobs.end())
-                dep_info->second.dependent_jobs.erase(job);
+            chassert(scheduled_jobs.contains(job)); // Job was pending
+            Info & info = scheduled_jobs[job];
+            if (info.is_ready())
+            {
+                ready_queue.erase(info.key());
+                info.ready_seqno = 0;
+            }
+
+            // Recurse into all dependent jobs
+            LoadJobSet dependent;
+            dependent.swap(info.dependent_jobs); // To avoid container modification during recursion
+            for (const auto & dep : dependent)
+            {
+                std::exception_ptr e;
+                NOEXCEPT_SCOPE({
+                    ALLOW_ALLOCATIONS_IN_SCOPE;
+                    e = std::make_exception_ptr(
+                        Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
+                            "Load job '{}' -> {}",
+                            dep->name,
+                            getExceptionMessage(exception_from_job, /* with_stack_trace = */ false)));
+                });
+                finish(lock, dep, LoadStatus::CANCELED, e);
+            }
+
+            // Clean dependency graph edges
+            for (const auto & dep : job->dependencies)
+                if (auto dep_info = scheduled_jobs.find(dep); dep_info != scheduled_jobs.end())
+                    dep_info->second.dependent_jobs.erase(job);
+        }
 
         // Job became finished
-        finish(job, lock);
-    }
-
-    void finish(const LoadJobPtr & job, std::unique_lock<std::mutex> &)
-    {
         scheduled_jobs.erase(job);
         NOEXCEPT_SCOPE({
             ALLOW_ALLOCATIONS_IN_SCOPE;
@@ -569,9 +615,9 @@ private:
 
                 // Handle just executed job
                 if (exception_from_job)
-                    markNotOk(job, exception_from_job, LoadStatus::FAILED, lock);
+                    finish(lock, job, LoadStatus::FAILED, exception_from_job);
                 else if (job)
-                    markOk(job, lock);
+                    finish(lock, job, LoadStatus::OK);
 
                 if (!is_running)
                     return;
@@ -589,17 +635,18 @@ private:
                 scheduled_jobs.find(job)->second.ready_seqno = 0; // This job is no longer in the ready queue
             }
 
+            ALLOW_ALLOCATIONS_IN_SCOPE;
+
             try
             {
-                ALLOW_ALLOCATIONS_IN_SCOPE;
                 job->func(job);
                 exception_from_job = {};
             }
             catch (...)
             {
-                tryLogCurrentException(__PRETTY_FUNCTION__);
                 NOEXCEPT_SCOPE({
-                    ALLOW_ALLOCATIONS_IN_SCOPE;
+                    if (log_failures)
+                        tryLogCurrentException(__PRETTY_FUNCTION__);
                     exception_from_job = std::make_exception_ptr(
                         Exception(ErrorCodes::ASYNC_LOAD_FAILED,
                             "Load job '{}' failed: {}",
@@ -610,6 +657,8 @@ private:
         }
     }
 
+    const bool log_failures;
+
     mutable std::mutex mutex;
     bool is_running = false;
 
diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp
index 0d203fc799c..64d07ccc9cf 100644
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@@ -653,7 +653,6 @@
     M(692, ASYNC_LOAD_SCHEDULE_FAILED) \
     M(693, ASYNC_LOAD_FAILED) \
     M(694, ASYNC_LOAD_CANCELED) \
-    M(695, ASYNC_LOAD_DEPENDENCY_FAILED) \
     \
     M(999, KEEPER_EXCEPTION) \
     M(1000, POCO_EXCEPTION) \
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index d7706311fa4..5be715db2f6 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -3,6 +3,7 @@
 #include <barrier>
 #include <chrono>
 #include <mutex>
+#include <string_view>
 #include <vector>
 #include <thread>
 #include <pcg_random.hpp>
@@ -28,7 +29,6 @@ namespace DB::ErrorCodes
     extern const int ASYNC_LOAD_SCHEDULE_FAILED;
     extern const int ASYNC_LOAD_FAILED;
     extern const int ASYNC_LOAD_CANCELED;
-    extern const int ASYNC_LOAD_DEPENDENCY_FAILED;
 }
 
 struct AsyncLoaderTest
@@ -39,7 +39,7 @@ struct AsyncLoaderTest
     pcg64 rng{randomSeed()};
 
     explicit AsyncLoaderTest(size_t max_threads = 1)
-        : loader(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, max_threads)
+        : loader(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, max_threads, /* log_failures = */ false)
     {}
 
     template <typename T>
@@ -229,8 +229,7 @@ TEST(AsyncLoader, CancelPendingTask)
     }
     catch (Exception & e)
     {
-        // Result depend on non-deterministic cancel order
-        ASSERT_TRUE(e.code() == ErrorCodes::ASYNC_LOAD_CANCELED || e.code() == ErrorCodes::ASYNC_LOAD_DEPENDENCY_FAILED);
+        ASSERT_TRUE(e.code() == ErrorCodes::ASYNC_LOAD_CANCELED);
     }
 }
 
@@ -267,7 +266,7 @@ TEST(AsyncLoader, CancelPendingDependency)
     }
     catch (Exception & e)
     {
-        ASSERT_TRUE(e.code() == ErrorCodes::ASYNC_LOAD_DEPENDENCY_FAILED);
+        ASSERT_TRUE(e.code() == ErrorCodes::ASYNC_LOAD_CANCELED);
     }
 }
 
@@ -358,6 +357,83 @@ TEST(AsyncLoader, CancelExecutingTask)
     }
 }
 
+TEST(AsyncLoader, JobFailure)
+{
+    AsyncLoaderTest t;
+    t.loader.start();
+
+    std::string_view error_message = "test job failure";
+
+    auto job_func = [&] (const LoadJobPtr &) {
+        throw Exception(ErrorCodes::ASYNC_LOAD_FAILED, "{}", error_message);
+    };
+
+    auto job = makeLoadJob({}, "job", job_func);
+    auto task = t.loader.schedule({ job });
+
+    t.loader.wait();
+
+    ASSERT_EQ(job->status(), LoadStatus::FAILED);
+    try
+    {
+        job->wait();
+        FAIL();
+    }
+    catch (Exception & e)
+    {
+        ASSERT_EQ(e.code(), ErrorCodes::ASYNC_LOAD_FAILED);
+        ASSERT_TRUE(e.message().find(error_message) != String::npos);
+    }
+}
+
+TEST(AsyncLoader, ScheduleJobWithFailedDependencies)
+{
+    AsyncLoaderTest t;
+    t.loader.start();
+
+    std::string_view error_message = "test job failure";
+
+    auto failed_job_func = [&] (const LoadJobPtr &) {
+        throw Exception(ErrorCodes::ASYNC_LOAD_FAILED, "{}", error_message);
+    };
+
+    auto failed_job = makeLoadJob({}, "failed_job", failed_job_func);
+    auto failed_task = t.loader.schedule({ failed_job });
+
+    t.loader.wait();
+
+    auto job_func = [&] (const LoadJobPtr &) {};
+
+    auto job1 = makeLoadJob({ failed_job }, "job1", job_func);
+    auto job2 = makeLoadJob({ job1 }, "job2", job_func);
+    auto task = t.loader.schedule({ job1, job2 });
+
+    t.loader.wait();
+
+    ASSERT_EQ(job1->status(), LoadStatus::CANCELED);
+    ASSERT_EQ(job2->status(), LoadStatus::CANCELED);
+    try
+    {
+        job1->wait();
+        FAIL();
+    }
+    catch (Exception & e)
+    {
+        ASSERT_EQ(e.code(), ErrorCodes::ASYNC_LOAD_CANCELED);
+        ASSERT_TRUE(e.message().find(error_message) != String::npos);
+    }
+    try
+    {
+        job2->wait();
+        FAIL();
+    }
+    catch (Exception & e)
+    {
+        ASSERT_EQ(e.code(), ErrorCodes::ASYNC_LOAD_CANCELED);
+        ASSERT_TRUE(e.message().find(error_message) != String::npos);
+    }
+}
+
 TEST(AsyncLoader, TestConcurrency)
 {
     AsyncLoaderTest t(10);

From 51992ae96d7f3d165a2fd090d87817a9fd9daac1 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 14 Apr 2023 17:27:47 +0000
Subject: [PATCH 039/689] add ScheduleJobWithCanceledDependencies test

---
 src/Common/tests/gtest_async_loader.cpp | 40 +++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 5be715db2f6..e713e0be562 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -434,6 +434,46 @@ TEST(AsyncLoader, ScheduleJobWithFailedDependencies)
     }
 }
 
+TEST(AsyncLoader, ScheduleJobWithCanceledDependencies)
+{
+    AsyncLoaderTest t;
+
+    auto canceled_job_func = [&] (const LoadJobPtr &) {};
+    auto canceled_job = makeLoadJob({}, "canceled_job", canceled_job_func);
+    auto canceled_task = t.loader.schedule({ canceled_job });
+    canceled_task.remove();
+
+    t.loader.start();
+
+    auto job_func = [&] (const LoadJobPtr &) {};
+    auto job1 = makeLoadJob({ canceled_job }, "job1", job_func);
+    auto job2 = makeLoadJob({ job1 }, "job2", job_func);
+    auto task = t.loader.schedule({ job1, job2 });
+
+    t.loader.wait();
+
+    ASSERT_EQ(job1->status(), LoadStatus::CANCELED);
+    ASSERT_EQ(job2->status(), LoadStatus::CANCELED);
+    try
+    {
+        job1->wait();
+        FAIL();
+    }
+    catch (Exception & e)
+    {
+        ASSERT_EQ(e.code(), ErrorCodes::ASYNC_LOAD_CANCELED);
+    }
+    try
+    {
+        job2->wait();
+        FAIL();
+    }
+    catch (Exception & e)
+    {
+        ASSERT_EQ(e.code(), ErrorCodes::ASYNC_LOAD_CANCELED);
+    }
+}
+
 TEST(AsyncLoader, TestConcurrency)
 {
     AsyncLoaderTest t(10);

From d8bf775b9ace8e82973253c9ad4ee7ab4ae370ef Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 14 Apr 2023 19:29:24 +0000
Subject: [PATCH 040/689] fix worker shutdown

---
 src/Common/AsyncLoader.h                | 5 +----
 src/Common/tests/gtest_async_loader.cpp | 2 --
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 84b709f14a1..98f2e4153d4 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -619,10 +619,7 @@ private:
                 else if (job)
                     finish(lock, job, LoadStatus::OK);
 
-                if (!is_running)
-                    return;
-
-                if (ready_queue.empty())
+                if (!is_running || ready_queue.empty())
                 {
                     workers--;
                     return;
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index e713e0be562..8a1d9f8b2de 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -14,10 +14,8 @@
 #include <Common/AsyncLoader.h>
 #include <Common/randomSeed.h>
 
-
 using namespace DB;
 
-
 namespace CurrentMetrics
 {
     extern const Metric TablesLoaderThreads;

From 8c1f734f50edbbe56c1f6b8fa2fbc7c7bd0acc57 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 14 Apr 2023 20:49:56 +0000
Subject: [PATCH 041/689] add random test with dependent tasks

---
 src/Common/AsyncLoader.h                |  6 +++
 src/Common/tests/gtest_async_loader.cpp | 59 ++++++++++++++++++++++++-
 2 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 98f2e4153d4..6d19ed05463 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -449,6 +449,12 @@ public:
         return pool.getMaxThreads();
     }
 
+    size_t getScheduledJobCount() const
+    {
+        std::unique_lock lock{mutex};
+        return scheduled_jobs.size();
+    }
+
 private:
     void checkCycle(const LoadJobSet & jobs, std::unique_lock<std::mutex> & lock)
     {
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 8a1d9f8b2de..208d5590f3e 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -71,6 +71,25 @@ struct AsyncLoaderTest
         return {jobs.begin(), jobs.end()};
     }
 
+    template <typename JobFunc>
+    LoadJobSet randomJobSet(int job_count, int dep_probability_percent, std::vector<LoadJobPtr> external_deps, JobFunc job_func, std::string_view name_prefix = "job")
+    {
+        std::vector<LoadJobPtr> jobs;
+        for (int j = 0; j < job_count; j++)
+        {
+            LoadJobSet deps;
+            for (int d = 0; d < j; d++)
+            {
+                if (randomInt(0, 99) < dep_probability_percent)
+                    deps.insert(jobs[d]);
+            }
+            if (randomInt(0, 99) < dep_probability_percent)
+                deps.insert(external_deps[randomInt<size_t>(0, external_deps.size() - 1)]);
+            jobs.push_back(makeLoadJob(std::move(deps), fmt::format("{}{}", name_prefix, j), job_func));
+        }
+        return {jobs.begin(), jobs.end()};
+    }
+
     template <typename JobFunc>
     LoadJobSet chainJobSet(int job_count, JobFunc job_func, std::string_view name_prefix = "job")
     {
@@ -526,7 +545,7 @@ TEST(AsyncLoader, TestOverload)
     }
 }
 
-TEST(AsyncLoader, RandomTasks)
+TEST(AsyncLoader, RandomIndependentTasks)
 {
     AsyncLoaderTest t(16);
     t.loader.start();
@@ -546,3 +565,41 @@ TEST(AsyncLoader, RandomTasks)
         t.randomSleepUs(100, 900, 20); // avg=100us
     }
 }
+
+TEST(AsyncLoader, RandomDependentTasks)
+{
+    AsyncLoaderTest t(16);
+    t.loader.start();
+
+    std::mutex mutex;
+    std::condition_variable cv;
+    std::vector<AsyncLoader::Task> tasks;
+    std::vector<LoadJobPtr> all_jobs;
+
+    auto job_func = [&] (const LoadJobPtr & self)
+    {
+        for (const auto & dep : self->dependencies)
+            ASSERT_EQ(dep->status(), LoadStatus::OK);
+        cv.notify_one();
+    };
+
+    std::unique_lock lock{mutex};
+
+    int tasks_left = 1000;
+    while (tasks_left-- > 0)
+    {
+        cv.wait(lock, [&] { return t.loader.getScheduledJobCount() < 100; });
+
+        // Add one new task
+        int job_count = t.randomInt(1, 32);
+        LoadJobSet jobs = t.randomJobSet(job_count, 5, all_jobs, job_func);
+        all_jobs.insert(all_jobs.end(), jobs.begin(), jobs.end());
+        tasks.push_back(t.loader.schedule(std::move(jobs)));
+
+        // Cancel random old task
+        if (tasks.size() > 100)
+            tasks.erase(tasks.begin() + t.randomInt<size_t>(0, tasks.size() - 1));
+    }
+
+    t.loader.wait();
+}

From b86c7374509895d986d2d1c6587c62c8df20a233 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sat, 15 Apr 2023 14:22:01 +0000
Subject: [PATCH 042/689] make
 performance-inefficient-vector-operation,-warnings-as-errors happy

---
 src/Common/tests/gtest_async_loader.cpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 208d5590f3e..ad1cb206b94 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -58,6 +58,7 @@ struct AsyncLoaderTest
     LoadJobSet randomJobSet(int job_count, int dep_probability_percent, JobFunc job_func, std::string_view name_prefix = "job")
     {
         std::vector<LoadJobPtr> jobs;
+        jobs.reserve(job_count);
         for (int j = 0; j < job_count; j++)
         {
             LoadJobSet deps;
@@ -75,6 +76,7 @@ struct AsyncLoaderTest
     LoadJobSet randomJobSet(int job_count, int dep_probability_percent, std::vector<LoadJobPtr> external_deps, JobFunc job_func, std::string_view name_prefix = "job")
     {
         std::vector<LoadJobPtr> jobs;
+        jobs.reserve(job_count);
         for (int j = 0; j < job_count; j++)
         {
             LoadJobSet deps;
@@ -94,6 +96,7 @@ struct AsyncLoaderTest
     LoadJobSet chainJobSet(int job_count, JobFunc job_func, std::string_view name_prefix = "job")
     {
         std::vector<LoadJobPtr> jobs;
+        jobs.reserve(job_count);
         jobs.push_back(makeLoadJob({}, fmt::format("{}{}", name_prefix, 0), job_func));
         for (int j = 1; j < job_count; j++)
             jobs.push_back(makeLoadJob({ jobs[j - 1] }, fmt::format("{}{}", name_prefix, j), job_func));
@@ -158,6 +161,7 @@ TEST(AsyncLoader, CycleDetection)
     try
     {
         std::vector<LoadJobPtr> jobs;
+        jobs.reserve(16);
         jobs.push_back(makeLoadJob({}, "job0", job_func));
         jobs.push_back(makeLoadJob({ jobs[0] }, "job1", job_func));
         jobs.push_back(makeLoadJob({ jobs[0], jobs[1] }, "job2", job_func));
@@ -344,6 +348,7 @@ TEST(AsyncLoader, CancelExecutingTask)
     // Make several iterations to catch the race (if any)
     for (int iteration = 0; iteration < 10; iteration++) {
         std::vector<LoadJobPtr> task1_jobs;
+        task1_jobs.reserve(256);
         auto blocker_job = makeLoadJob({}, "blocker_job", blocker_job_func);
         task1_jobs.push_back(blocker_job);
         for (int i = 0; i < 100; i++)
@@ -510,6 +515,7 @@ TEST(AsyncLoader, TestConcurrency)
         };
 
         std::vector<AsyncLoader::Task> tasks;
+        tasks.reserve(concurrency);
         for (int i = 0; i < concurrency; i++)
             tasks.push_back(t.loader.schedule(t.chainJobSet(5, job_func)));
         t.loader.wait();
@@ -537,6 +543,7 @@ TEST(AsyncLoader, TestOverload)
 
         t.loader.stop();
         std::vector<AsyncLoader::Task> tasks;
+        tasks.reserve(concurrency);
         for (int i = 0; i < concurrency; i++)
             tasks.push_back(t.loader.schedule(t.chainJobSet(5, job_func)));
         t.loader.start();
@@ -558,6 +565,7 @@ TEST(AsyncLoader, RandomIndependentTasks)
     };
 
     std::vector<AsyncLoader::Task> tasks;
+    tasks.reserve(512);
     for (int i = 0; i < 512; i++)
     {
         int job_count = t.randomInt(1, 32);
@@ -586,6 +594,7 @@ TEST(AsyncLoader, RandomDependentTasks)
     std::unique_lock lock{mutex};
 
     int tasks_left = 1000;
+    tasks.reserve(tasks_left);
     while (tasks_left-- > 0)
     {
         cv.wait(lock, [&] { return t.loader.getScheduledJobCount() < 100; });

From 264956fecea261805c902ab1ba16c9da604c6259 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sat, 15 Apr 2023 19:48:10 +0000
Subject: [PATCH 043/689] add test for static priorities

---
 src/Common/AsyncLoader.h                | 26 +++++++++++-------
 src/Common/tests/gtest_async_loader.cpp | 35 +++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 9 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 6d19ed05463..5fff5ee89c1 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -144,23 +144,31 @@ private:
     // Key of a pending job in ready queue
     struct ReadyKey
     {
-        ssize_t priority;
-        UInt64 ready_seqno;
+        ssize_t priority; // Ascending order
+        ssize_t initial_priority; // Ascending order
+        UInt64 ready_seqno; // Descending order
 
         bool operator<(const ReadyKey & rhs) const
         {
-            if (priority == rhs.priority)
-                return ready_seqno < rhs.ready_seqno;
-            return priority > rhs.priority;
+            if (priority > rhs.priority)
+                return true;
+            if (priority < rhs.priority)
+                return false;
+            if (initial_priority > rhs.initial_priority)
+                return true;
+            if (initial_priority < rhs.initial_priority)
+                return false;
+            return ready_seqno < rhs.ready_seqno;
         }
     };
 
     // Scheduling information for a pending job
     struct Info
     {
-        ssize_t priority = 0;
+        ssize_t initial_priority = 0; // Initial priority passed into schedule()
+        ssize_t priority = 0; // Elevated priority, due to priority inheritance or prioritize()
         size_t dependencies_left = 0;
-        UInt64 ready_seqno = 0; // zero means that job is not in ready queue
+        UInt64 ready_seqno = 0; // Zero means that job is not in ready queue
         LoadJobSet dependent_jobs;
 
         // Three independent states of a non-finished jobs
@@ -171,7 +179,7 @@ private:
         // Get key of a ready job
         ReadyKey key() const
         {
-            return {priority, ready_seqno};
+            return {.priority = priority, .initial_priority = initial_priority, .ready_seqno = ready_seqno};
         }
     };
 
@@ -311,7 +319,7 @@ public:
         {
             NOEXCEPT_SCOPE({
                 ALLOW_ALLOCATIONS_IN_SCOPE;
-                scheduled_jobs.emplace(job, Info{.priority = priority});
+                scheduled_jobs.emplace(job, Info{.initial_priority = priority, .priority = priority});
             });
             job->priority.store(priority); // Set user-facing priority
         }
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index ad1cb206b94..a72cc0f994b 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -552,6 +552,41 @@ TEST(AsyncLoader, TestOverload)
     }
 }
 
+TEST(AsyncLoader, StaticPriorities)
+{
+    AsyncLoaderTest t(1);
+
+    std::string schedule;
+
+    auto job_func = [&] (const LoadJobPtr & self)
+    {
+        schedule += fmt::format("{}{}", self->name, self->priority);
+    };
+
+    std::vector<LoadJobPtr> jobs;
+    jobs.push_back(makeLoadJob({}, "A", job_func)); // 0
+    jobs.push_back(makeLoadJob({ jobs[0] }, "B", job_func)); // 1
+    jobs.push_back(makeLoadJob({ jobs[0] }, "C", job_func)); // 2
+    jobs.push_back(makeLoadJob({ jobs[0] }, "D", job_func)); // 3
+    jobs.push_back(makeLoadJob({ jobs[0] }, "E", job_func)); // 4
+    jobs.push_back(makeLoadJob({ jobs[3], jobs[4] }, "F", job_func)); // 5
+    jobs.push_back(makeLoadJob({ jobs[5] }, "G", job_func)); // 6
+    jobs.push_back(makeLoadJob({ jobs[6] }, "H", job_func)); // 7
+    auto task = t.loader.schedule({ jobs[0] }, 0);
+    task.merge(t.loader.schedule({ jobs[1] }, 3));
+    task.merge(t.loader.schedule({ jobs[2] }, 4));
+    task.merge(t.loader.schedule({ jobs[3] }, 1));
+    task.merge(t.loader.schedule({ jobs[4] }, 2));
+    task.merge(t.loader.schedule({ jobs[5] }, 0));
+    task.merge(t.loader.schedule({ jobs[6] }, 0));
+    task.merge(t.loader.schedule({ jobs[7] }, 9));
+
+    t.loader.start();
+    t.loader.wait();
+
+    ASSERT_EQ(schedule, "A9E9D9F9G9H9C4B3");
+}
+
 TEST(AsyncLoader, RandomIndependentTasks)
 {
     AsyncLoaderTest t(16);

From c736d9fd4fc4c79e6ea2f7c71ba0e8c7f437c878 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sun, 16 Apr 2023 08:35:59 +0000
Subject: [PATCH 044/689] fix test

---
 src/Common/tests/gtest_async_loader.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index a72cc0f994b..755f856d340 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -73,7 +73,7 @@ struct AsyncLoaderTest
     }
 
     template <typename JobFunc>
-    LoadJobSet randomJobSet(int job_count, int dep_probability_percent, std::vector<LoadJobPtr> external_deps, JobFunc job_func, std::string_view name_prefix = "job")
+    LoadJobSet randomJobSet(int job_count, int dep_probability_percent, const std::vector<LoadJobPtr> & external_deps, JobFunc job_func, std::string_view name_prefix = "job")
     {
         std::vector<LoadJobPtr> jobs;
         jobs.reserve(job_count);
@@ -85,7 +85,7 @@ struct AsyncLoaderTest
                 if (randomInt(0, 99) < dep_probability_percent)
                     deps.insert(jobs[d]);
             }
-            if (randomInt(0, 99) < dep_probability_percent)
+            if (!external_deps.empty() && randomInt(0, 99) < dep_probability_percent)
                 deps.insert(external_deps[randomInt<size_t>(0, external_deps.size() - 1)]);
             jobs.push_back(makeLoadJob(std::move(deps), fmt::format("{}{}", name_prefix, j), job_func));
         }

From 69719c0819c79a470ef08aa2e686a12330e0f945 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sun, 16 Apr 2023 11:04:59 +0000
Subject: [PATCH 045/689] allow to change max number of workers in runtime

---
 src/Common/AsyncLoader.h                | 22 ++++++++---
 src/Common/tests/gtest_async_loader.cpp | 49 +++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 5 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 5fff5ee89c1..d639c1031cd 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -120,8 +120,6 @@ private:
             finished.notify_all();
     }
 
-    // TODO(serxa): add callback/status for cancel?
-
     std::function<void(const LoadJobPtr & self)> func;
 
     mutable std::mutex mutex;
@@ -243,6 +241,7 @@ public:
         }
 
         // Do not track jobs in this task
+        // WARNING: Jobs will never be removed() and are going to be stored as finished_jobs until ~AsyncLoader()
         void detach()
         {
             loader = nullptr;
@@ -451,10 +450,23 @@ public:
             finished_jobs.erase(job);
     }
 
+    void setMaxThreads(size_t value)
+    {
+        std::unique_lock lock{mutex};
+        pool.setMaxThreads(value);
+        pool.setMaxFreeThreads(value);
+        pool.setQueueSize(value);
+        max_threads = value;
+        if (!is_running)
+            return;
+        for (size_t i = 0; workers < max_threads && i < ready_queue.size(); i++)
+            spawn(lock);
+    }
+
     size_t getMaxThreads() const
     {
         std::unique_lock lock{mutex};
-        return pool.getMaxThreads();
+        return max_threads;
     }
 
     size_t getScheduledJobCount() const
@@ -600,7 +612,7 @@ private:
             ready_queue.emplace(info.key(), job);
         });
 
-        if (is_running && workers < max_threads) // TODO(serxa): Can we make max_thread changeable in runtime?
+        if (is_running && workers < max_threads)
             spawn(lock);
     }
 
@@ -633,7 +645,7 @@ private:
                 else if (job)
                     finish(lock, job, LoadStatus::OK);
 
-                if (!is_running || ready_queue.empty())
+                if (!is_running || ready_queue.empty() || workers > max_threads)
                 {
                     workers--;
                     return;
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 755f856d340..52cc3436595 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -647,3 +647,52 @@ TEST(AsyncLoader, RandomDependentTasks)
 
     t.loader.wait();
 }
+
+TEST(AsyncLoader, SetMaxThreads)
+{
+    AsyncLoaderTest t(1);
+
+    std::atomic<int> sync_index{0};
+    std::atomic<int> executing{0};
+    int max_threads_values[] = {1, 2, 3, 4, 5, 4, 3, 2, 1, 5, 10, 5, 1, 20, 1};
+    std::vector<std::unique_ptr<std::barrier<>>> syncs;
+    syncs.reserve(std::size(max_threads_values));
+    for (int max_threads : max_threads_values)
+        syncs.push_back(std::make_unique<std::barrier<>>(max_threads + 1));
+
+
+    auto job_func = [&] (const LoadJobPtr &)
+    {
+        int idx = sync_index;
+        if (idx < syncs.size())
+        {
+            executing++;
+            syncs[idx]->arrive_and_wait(); // (A)
+            executing--;
+            syncs[idx]->arrive_and_wait(); // (B)
+        }
+    };
+
+    // Generate enough independent jobs
+    for (int i = 0; i < 1000; i++)
+        t.loader.schedule({makeLoadJob({}, "job", job_func)}).detach();
+
+    t.loader.start();
+    while (sync_index < syncs.size())
+    {
+        // Wait for `max_threads` jobs to start executing
+        int idx = sync_index;
+        while (executing.load() != max_threads_values[idx])
+        {
+            ASSERT_LE(executing, max_threads_values[idx]);
+            std::this_thread::yield();
+        }
+
+        // Allow all jobs to finish
+        syncs[idx]->arrive_and_wait(); // (A)
+        sync_index++;
+        if (sync_index < syncs.size())
+            t.loader.setMaxThreads(max_threads_values[sync_index]);
+        syncs[idx]->arrive_and_wait(); // (B) this sync point is required to allow `executing` value to go back down to zero after we change number of workers
+    }
+}

From 800efd5df82491ed0e6decfe72cdff5670c5ac0e Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sun, 16 Apr 2023 12:07:46 +0000
Subject: [PATCH 046/689] add a good comment

---
 src/Common/AsyncLoader.h | 42 +++++++++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index d639c1031cd..9e54b3ae75f 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -135,7 +135,45 @@ LoadJobPtr makeLoadJob(LoadJobSet && dependencies, const String & name, Func &&
     return std::make_shared<LoadJob>(std::move(dependencies), name, std::forward<Func>(func));
 }
 
-// TODO(serxa): write good comment
+// `AsyncLoader` is a scheduler for DAG of `LoadJob`s. It tracks dependencies and priorities of jobs.
+// Basic usage example:
+//     auto job_func = [&] (const LoadJobPtr & self) {
+//         LOG_TRACE(log, "Executing load job '{}' with priority '{}'", self->name, self->priority);
+//     };
+//     auto job1 = makeLoadJob({}, "job1", job_func);
+//     auto job2 = makeLoadJob({ job1 }, "job2", job_func);
+//     auto job3 = makeLoadJob({ job1 }, "job3", job_func);
+//     auto task = async_loader.schedule({ job1, job2, job3 }, /* priority = */ 0);
+// Here we have created and scheduled a task consisting of two jobs. Job1 has no dependencies and is run first.
+// Job2 and job3 depends on job1 and are run only after job1 completion. Another thread may prioritize a job and wait for it:
+//     async_loader->prioritize(job3, 1); // higher priority jobs are run first
+//     job3->wait(); // blocks until job completion or cancelation and rethrow an exception (if any)
+//
+// AsyncLoader tracks state of all scheduled jobs. Job lifecycle is the following:
+// 1)  Job is constructed with PENDING status.
+// 2)  Job is scheduled and placed into a task. Scheduled job may be ready (i.e. have all its dependencies finished) or blocked.
+// 3a) When all dependencies are successfully executed, job became ready. Ready job is enqueued into the ready queue.
+// 3b) If at least one of job dependencies is failed or canceled, then this job is canceled (with all it's dependent jobs as well).
+//     On cancelation an ASYNC_LOAD_CANCELED exception is generated and saved inside LoadJob object. Job status is changed to CANCELED.
+//     Exception is rethrown by any existing or new `wait()` call. Job is moved to the set of finished jobs.
+// 4)  Scheduled pending ready job starts execution by a worker. Job is dequeuedCallback `job_func` is called.
+//     Status of an executing job is PENDING. And it is still considered as scheduled job by AsyncLoader.
+//     Note that `job_func` of a CANCELED job is never executed.
+// 5a) On successful execution job status is changed to OK and all existing and new `wait()` calls finish w/o exceptions.
+// 5b) Any exception thrown out of `job_func` is wrapped into ASYNC_LOAD_FAILED exception and save inside LoadJob.
+//     Job status is changed to FAILED. All dependent jobs are canceled. The exception is rethrown from all existing and new `wait()` calls.
+// 6)  Job is no longer considered as scheduled and is instead moved to finished jobs set. This is required for introspection of finished jobs.
+// 7)  Task object containing this job is destructed or `remove()` is explicitly called. Job is removed from the finished job set.
+// 8)  Job is destructed.
+//
+// Every job has a priority associated with it. AsyncLoader runs higher priority (greater `priority` value) jobs first. Job priority can be elevated
+// (a) if either it has a dependent job with higher priority (in this case priority of a dependent job is inherited);
+// (b) or job was explicitly prioritized by `prioritize(job, higher_priority)` call (this also leads to a priority inheritance for all the dependencies).
+// Note that to avoid priority inversion `job_func` should use `self->priority` to schedule new jobs in AsyncLoader or any other pool.
+// Value stored in load job priority field is atomic and can be increased even during job execution.
+//
+// When task is scheduled. It can contain dependencies on previously scheduled jobs. These jobs can have any status.
+// The only forbidden thing is a dependency on job, that was not scheduled in AsyncLoader yet: all dependent jobs are immediately canceled.
 class AsyncLoader : private boost::noncopyable
 {
 private:
@@ -685,8 +723,6 @@ private:
     mutable std::mutex mutex;
     bool is_running = false;
 
-    // TODO(serxa): add metrics for number of jobs in every state
-
     // Full set of scheduled pending jobs along with scheduling info
     std::unordered_map<LoadJobPtr, Info> scheduled_jobs;
 

From 1a3780ed8e6fa0b0ce0a701059bc54e7d2b03d5f Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sun, 16 Apr 2023 12:13:23 +0000
Subject: [PATCH 047/689] fix

---
 src/Common/AsyncLoader.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 9e54b3ae75f..aca35b599f4 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -172,7 +172,7 @@ LoadJobPtr makeLoadJob(LoadJobSet && dependencies, const String & name, Func &&
 // Note that to avoid priority inversion `job_func` should use `self->priority` to schedule new jobs in AsyncLoader or any other pool.
 // Value stored in load job priority field is atomic and can be increased even during job execution.
 //
-// When task is scheduled. It can contain dependencies on previously scheduled jobs. These jobs can have any status.
+// When task is scheduled it can contain dependencies on previously scheduled jobs. These jobs can have any status.
 // The only forbidden thing is a dependency on job, that was not scheduled in AsyncLoader yet: all dependent jobs are immediately canceled.
 class AsyncLoader : private boost::noncopyable
 {

From 281b72c4131015a7e43340ac2ae2d78b327ba46e Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Sun, 16 Apr 2023 15:09:44 +0200
Subject: [PATCH 048/689] Update src/Common/AsyncLoader.h

---
 src/Common/AsyncLoader.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index aca35b599f4..1423b7f83c7 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -154,7 +154,7 @@ LoadJobPtr makeLoadJob(LoadJobSet && dependencies, const String & name, Func &&
 // 2)  Job is scheduled and placed into a task. Scheduled job may be ready (i.e. have all its dependencies finished) or blocked.
 // 3a) When all dependencies are successfully executed, job became ready. Ready job is enqueued into the ready queue.
 // 3b) If at least one of job dependencies is failed or canceled, then this job is canceled (with all it's dependent jobs as well).
-//     On cancelation an ASYNC_LOAD_CANCELED exception is generated and saved inside LoadJob object. Job status is changed to CANCELED.
+//     On cancellation an ASYNC_LOAD_CANCELED exception is generated and saved inside LoadJob object. Job status is changed to CANCELED.
 //     Exception is rethrown by any existing or new `wait()` call. Job is moved to the set of finished jobs.
 // 4)  Scheduled pending ready job starts execution by a worker. Job is dequeuedCallback `job_func` is called.
 //     Status of an executing job is PENDING. And it is still considered as scheduled job by AsyncLoader.

From 1efacba196e282254c1e68e08f74783bfdb6f92c Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Sun, 16 Apr 2023 15:09:55 +0200
Subject: [PATCH 049/689] Update src/Common/AsyncLoader.h

---
 src/Common/AsyncLoader.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 1423b7f83c7..34de4fe3c8b 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -147,7 +147,7 @@ LoadJobPtr makeLoadJob(LoadJobSet && dependencies, const String & name, Func &&
 // Here we have created and scheduled a task consisting of two jobs. Job1 has no dependencies and is run first.
 // Job2 and job3 depends on job1 and are run only after job1 completion. Another thread may prioritize a job and wait for it:
 //     async_loader->prioritize(job3, 1); // higher priority jobs are run first
-//     job3->wait(); // blocks until job completion or cancelation and rethrow an exception (if any)
+//     job3->wait(); // blocks until job completion or cancellation and rethrow an exception (if any)
 //
 // AsyncLoader tracks state of all scheduled jobs. Job lifecycle is the following:
 // 1)  Job is constructed with PENDING status.

From 356b93483591df41ab9fa2dfd82b98a91c26bd27 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sun, 16 Apr 2023 18:17:11 +0000
Subject: [PATCH 050/689] split AsyncLoader into .h and .cpp files + more docs

---
 src/Common/AsyncLoader.cpp              | 560 +++++++++++++++++++++
 src/Common/AsyncLoader.h                | 632 +++---------------------
 src/Common/tests/gtest_async_loader.cpp |   4 +-
 3 files changed, 641 insertions(+), 555 deletions(-)
 create mode 100644 src/Common/AsyncLoader.cpp

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
new file mode 100644
index 00000000000..09dd4f129ae
--- /dev/null
+++ b/src/Common/AsyncLoader.cpp
@@ -0,0 +1,560 @@
+#include <Common/AsyncLoader.h>
+
+#include <base/defines.h>
+#include <Common/ErrorCodes.h>
+#include <Common/Exception.h>
+#include <Common/noexcept_scope.h>
+#include <Common/setThreadName.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int ASYNC_LOAD_SCHEDULE_FAILED;
+    extern const int ASYNC_LOAD_FAILED;
+    extern const int ASYNC_LOAD_CANCELED;
+}
+
+
+LoadStatus LoadJob::status() const
+{
+    std::unique_lock lock{mutex};
+    return load_status;
+}
+
+std::exception_ptr LoadJob::exception() const
+{
+    std::unique_lock lock{mutex};
+    return load_exception;
+}
+
+ssize_t LoadJob::priority() const
+{
+    return load_priority;
+}
+
+void LoadJob::wait() const
+{
+    std::unique_lock lock{mutex};
+    waiters++;
+    finished.wait(lock, [this] { return load_status != LoadStatus::PENDING; });
+    waiters--;
+    if (load_exception)
+        std::rethrow_exception(load_exception);
+}
+
+void LoadJob::waitNoThrow() const noexcept
+{
+    std::unique_lock lock{mutex};
+    waiters++;
+    finished.wait(lock, [this] { return load_status != LoadStatus::PENDING; });
+    waiters--;
+}
+
+size_t LoadJob::waiters_count() const
+{
+    std::unique_lock lock{mutex};
+    return waiters;
+}
+
+void LoadJob::ok()
+{
+    std::unique_lock lock{mutex};
+    load_status = LoadStatus::OK;
+    if (waiters > 0)
+        finished.notify_all();
+}
+
+void LoadJob::failed(const std::exception_ptr & ptr)
+{
+    std::unique_lock lock{mutex};
+    load_status = LoadStatus::FAILED;
+    load_exception = ptr;
+    if (waiters > 0)
+        finished.notify_all();
+}
+
+void LoadJob::canceled(const std::exception_ptr & ptr)
+{
+    std::unique_lock lock{mutex};
+    load_status = LoadStatus::CANCELED;
+    load_exception = ptr;
+    if (waiters > 0)
+        finished.notify_all();
+}
+
+
+AsyncLoader::Task::Task()
+    : loader(nullptr)
+{}
+
+AsyncLoader::Task::Task(AsyncLoader * loader_, LoadJobSet && jobs_)
+    : loader(loader_)
+    , jobs(std::move(jobs_))
+{}
+
+AsyncLoader::Task::Task(AsyncLoader::Task && o) noexcept
+    : loader(std::exchange(o.loader, nullptr))
+    , jobs(std::move(o.jobs))
+{}
+
+AsyncLoader::Task::~Task()
+{
+    remove();
+}
+
+AsyncLoader::Task & AsyncLoader::Task::operator=(AsyncLoader::Task && o) noexcept
+{
+    loader = std::exchange(o.loader, nullptr);
+    jobs = std::move(o.jobs);
+    return *this;
+}
+
+void AsyncLoader::Task::merge(AsyncLoader::Task && o)
+{
+    if (!loader)
+    {
+        *this = std::move(o);
+    }
+    else
+    {
+        chassert(loader == o.loader);
+        jobs.merge(o.jobs);
+        o.loader = nullptr;
+    }
+}
+
+void AsyncLoader::Task::remove()
+{
+    if (loader)
+    {
+        loader->remove(jobs);
+        detach();
+    }
+}
+
+void AsyncLoader::Task::detach()
+{
+    loader = nullptr;
+    jobs.clear();
+}
+
+AsyncLoader::AsyncLoader(Metric metric_threads, Metric metric_active_threads, size_t max_threads_, bool log_failures_)
+    : log_failures(log_failures_)
+    , max_threads(max_threads_)
+    , pool(metric_threads, metric_active_threads, max_threads)
+{}
+
+AsyncLoader::~AsyncLoader()
+{
+    stop();
+}
+
+void AsyncLoader::start()
+{
+    std::unique_lock lock{mutex};
+    is_running = true;
+    for (size_t i = 0; workers < max_threads && i < ready_queue.size(); i++)
+        spawn(lock);
+}
+
+void AsyncLoader::wait()
+{
+    pool.wait();
+}
+
+void AsyncLoader::stop()
+{
+    {
+        std::unique_lock lock{mutex};
+        is_running = false;
+        // NOTE: there is no need to notify because workers never wait
+    }
+    pool.wait();
+}
+
+AsyncLoader::Task AsyncLoader::schedule(LoadJobSet && jobs, ssize_t priority)
+{
+    std::unique_lock lock{mutex};
+
+    // Sanity checks
+    for (const auto & job : jobs)
+    {
+        if (job->status() != LoadStatus::PENDING)
+            throw Exception(ErrorCodes::ASYNC_LOAD_SCHEDULE_FAILED, "Trying to schedule already finished load job '{}'", job->name);
+        if (scheduled_jobs.contains(job))
+            throw Exception(ErrorCodes::ASYNC_LOAD_SCHEDULE_FAILED, "Load job '{}' has been already scheduled", job->name);
+    }
+
+    // Ensure scheduled_jobs graph will have no cycles. The only way to get a cycle is to add a cycle, assuming old jobs cannot reference new ones.
+    checkCycle(jobs, lock);
+
+    // We do not want any exception to be throws after this point, because the following code is not exception-safe
+    DENY_ALLOCATIONS_IN_SCOPE;
+
+    // Schedule all incoming jobs
+    for (const auto & job : jobs)
+    {
+        NOEXCEPT_SCOPE({
+            ALLOW_ALLOCATIONS_IN_SCOPE;
+            scheduled_jobs.emplace(job, Info{.initial_priority = priority, .priority = priority});
+        });
+        job->load_priority.store(priority); // Set user-facing priority
+    }
+
+    // Process dependencies on scheduled pending jobs
+    for (const auto & job : jobs)
+    {
+        Info & info = scheduled_jobs.find(job)->second;
+        for (const auto & dep : job->dependencies)
+        {
+            // Register every dependency on scheduled job with back-link to dependent job
+            if (auto dep_info = scheduled_jobs.find(dep); dep_info != scheduled_jobs.end())
+            {
+                NOEXCEPT_SCOPE({
+                    ALLOW_ALLOCATIONS_IN_SCOPE;
+                    dep_info->second.dependent_jobs.insert(job);
+                });
+                info.dependencies_left++;
+
+                // Priority inheritance: prioritize deps to have at least given `priority` to avoid priority inversion
+                prioritize(dep, priority, lock);
+            }
+        }
+
+        // Enqueue non-blocked jobs (w/o dependencies) to ready queue
+        if (!info.is_blocked())
+            enqueue(info, job, lock);
+    }
+
+    // Process dependencies on other jobs. It is done in a separate pass to facilitate propagation of cancel signals (if any).
+    for (const auto & job : jobs)
+    {
+        if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
+        {
+            for (const auto & dep : job->dependencies)
+            {
+                if (scheduled_jobs.contains(dep))
+                    continue; // Skip dependencies on scheduled pending jobs (already processed)
+                LoadStatus dep_status = dep->status();
+                if (dep_status == LoadStatus::OK)
+                    continue; // Dependency on already successfully finished job -- it's okay.
+
+                if (dep_status == LoadStatus::PENDING)
+                {
+                    // Dependency on not scheduled pending job -- it's bad.
+                    // Probably, there is an error in `jobs` set: not all jobs were passed to `schedule()` call.
+                    // We are not going to run any dependent job, so cancel them all.
+                    std::exception_ptr e;
+                    NOEXCEPT_SCOPE({
+                        ALLOW_ALLOCATIONS_IN_SCOPE;
+                        e = std::make_exception_ptr(Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
+                            "Load job '{}' -> Load job '{}': not scheduled pending load job (it must be also scheduled), all dependent load jobs are canceled",
+                            job->name,
+                            dep->name));
+                    });
+                    finish(lock, job, LoadStatus::CANCELED, e);
+                    break; // This job is now finished, stop its dependencies processing
+                }
+                if (dep_status == LoadStatus::FAILED || dep_status == LoadStatus::CANCELED)
+                {
+                    // Dependency on already failed or canceled job -- it's okay. Cancel all dependent jobs.
+                    std::exception_ptr e;
+                    NOEXCEPT_SCOPE({
+                        ALLOW_ALLOCATIONS_IN_SCOPE;
+                        e = std::make_exception_ptr(Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
+                            "Load job '{}' -> {}",
+                            job->name,
+                            getExceptionMessage(dep->exception(), /* with_stack_trace = */ false)));
+                    });
+                    finish(lock, job, LoadStatus::CANCELED, e);
+                    break; // This job is now finished, stop its dependencies processing
+                }
+            }
+        }
+        else
+        {
+            // Job was already canceled on previous iteration of this cycle -- skip
+        }
+    }
+
+    return Task(this, std::move(jobs));
+}
+
+void AsyncLoader::prioritize(const LoadJobPtr & job, ssize_t new_priority)
+{
+    DENY_ALLOCATIONS_IN_SCOPE;
+    std::unique_lock lock{mutex};
+    prioritize(job, new_priority, lock);
+}
+
+void AsyncLoader::remove(const LoadJobSet & jobs)
+{
+    DENY_ALLOCATIONS_IN_SCOPE;
+    std::unique_lock lock{mutex};
+    // On the first pass:
+    // - cancel all not executing jobs to avoid races
+    // - do not wait executing jobs (otherwise, on unlock a worker could start executing a dependent job, that should be canceled)
+    for (const auto & job : jobs)
+    {
+        if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
+        {
+            if (info->second.is_executing())
+                continue; // Skip executing jobs on the first pass
+            std::exception_ptr e;
+            NOEXCEPT_SCOPE({
+                ALLOW_ALLOCATIONS_IN_SCOPE;
+                e = std::make_exception_ptr(Exception(ErrorCodes::ASYNC_LOAD_CANCELED, "Load job '{}' canceled", job->name));
+            });
+            finish(lock, job, LoadStatus::CANCELED, e);
+        }
+    }
+    // On the second pass wait for executing jobs to finish
+    for (const auto & job : jobs)
+    {
+        if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
+        {
+            // Job is currently executing
+            chassert(info->second.is_executing());
+            lock.unlock();
+            job->waitNoThrow(); // Wait for job to finish
+            lock.lock();
+        }
+    }
+    // On the third pass all jobs are finished - remove them all
+    // It is better to do it under one lock to avoid exposing intermediate states
+    for (const auto & job : jobs)
+        finished_jobs.erase(job);
+}
+
+void AsyncLoader::setMaxThreads(size_t value)
+{
+    std::unique_lock lock{mutex};
+    pool.setMaxThreads(value);
+    pool.setMaxFreeThreads(value);
+    pool.setQueueSize(value);
+    max_threads = value;
+    if (!is_running)
+        return;
+    for (size_t i = 0; workers < max_threads && i < ready_queue.size(); i++)
+        spawn(lock);
+}
+
+size_t AsyncLoader::getMaxThreads() const
+{
+    std::unique_lock lock{mutex};
+    return max_threads;
+}
+
+size_t AsyncLoader::getScheduledJobCount() const
+{
+    std::unique_lock lock{mutex};
+    return scheduled_jobs.size();
+}
+
+void AsyncLoader::checkCycle(const LoadJobSet & jobs, std::unique_lock<std::mutex> & lock)
+{
+    LoadJobSet left = jobs;
+    LoadJobSet visited;
+    visited.reserve(left.size());
+    while (!left.empty())
+    {
+        LoadJobPtr job = *left.begin();
+        checkCycleImpl(job, left, visited, lock);
+    }
+}
+
+String AsyncLoader::checkCycleImpl(const LoadJobPtr & job, LoadJobSet & left, LoadJobSet & visited, std::unique_lock<std::mutex> & lock)
+{
+    if (!left.contains(job))
+        return {}; // Do not consider external dependencies and already processed jobs
+    if (auto [_, inserted] = visited.insert(job); !inserted)
+    {
+        visited.erase(job); // Mark where cycle ends
+        return job->name;
+    }
+    for (const auto & dep : job->dependencies)
+    {
+        if (auto chain = checkCycleImpl(dep, left, visited, lock); !chain.empty())
+        {
+            if (!visited.contains(job)) // Check for cycle end
+                throw Exception(ErrorCodes::ASYNC_LOAD_SCHEDULE_FAILED, "Load job dependency cycle detected: {} -> {}", job->name, chain);
+            else
+                return fmt::format("{} -> {}", job->name, chain); // chain is not a cycle yet -- continue building
+        }
+    }
+    left.erase(job);
+    return {};
+}
+
+void AsyncLoader::finish(std::unique_lock<std::mutex> & lock, const LoadJobPtr & job, LoadStatus status, std::exception_ptr exception_from_job)
+{
+    if (status == LoadStatus::OK)
+    {
+        // Notify waiters
+        job->ok();
+
+        // Update dependent jobs and enqueue if ready
+        chassert(scheduled_jobs.contains(job)); // Job was pending
+        for (const auto & dep : scheduled_jobs[job].dependent_jobs)
+        {
+            chassert(scheduled_jobs.contains(dep)); // All depended jobs must be pending
+            Info & dep_info = scheduled_jobs[dep];
+            dep_info.dependencies_left--;
+            if (!dep_info.is_blocked())
+                enqueue(dep_info, dep, lock);
+        }
+    }
+    else
+    {
+        // Notify waiters
+        if (status == LoadStatus::FAILED)
+            job->failed(exception_from_job);
+        else if (status == LoadStatus::CANCELED)
+            job->canceled(exception_from_job);
+
+        chassert(scheduled_jobs.contains(job)); // Job was pending
+        Info & info = scheduled_jobs[job];
+        if (info.is_ready())
+        {
+            ready_queue.erase(info.key());
+            info.ready_seqno = 0;
+        }
+
+        // Recurse into all dependent jobs
+        LoadJobSet dependent;
+        dependent.swap(info.dependent_jobs); // To avoid container modification during recursion
+        for (const auto & dep : dependent)
+        {
+            std::exception_ptr e;
+            NOEXCEPT_SCOPE({
+                ALLOW_ALLOCATIONS_IN_SCOPE;
+                e = std::make_exception_ptr(
+                    Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
+                        "Load job '{}' -> {}",
+                        dep->name,
+                        getExceptionMessage(exception_from_job, /* with_stack_trace = */ false)));
+            });
+            finish(lock, dep, LoadStatus::CANCELED, e);
+        }
+
+        // Clean dependency graph edges
+        for (const auto & dep : job->dependencies)
+            if (auto dep_info = scheduled_jobs.find(dep); dep_info != scheduled_jobs.end())
+                dep_info->second.dependent_jobs.erase(job);
+    }
+
+    // Job became finished
+    scheduled_jobs.erase(job);
+    NOEXCEPT_SCOPE({
+        ALLOW_ALLOCATIONS_IN_SCOPE;
+        finished_jobs.insert(job);
+    });
+}
+
+void AsyncLoader::prioritize(const LoadJobPtr & job, ssize_t new_priority, std::unique_lock<std::mutex> & lock)
+{
+    if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
+    {
+        if (info->second.priority >= new_priority)
+            return; // Never lower priority
+
+        // Update priority and push job forward through ready queue if needed
+        if (info->second.ready_seqno)
+            ready_queue.erase(info->second.key());
+        info->second.priority = new_priority;
+        job->load_priority.store(new_priority); // Set user-facing priority (may affect executing jobs)
+        if (info->second.ready_seqno)
+        {
+            NOEXCEPT_SCOPE({
+                ALLOW_ALLOCATIONS_IN_SCOPE;
+                ready_queue.emplace(info->second.key(), job);
+            });
+        }
+
+        // Recurse into dependencies
+        for (const auto & dep : job->dependencies)
+            prioritize(dep, new_priority, lock);
+    }
+}
+
+void AsyncLoader::enqueue(Info & info, const LoadJobPtr & job, std::unique_lock<std::mutex> & lock)
+{
+    chassert(!info.is_blocked());
+    chassert(info.ready_seqno == 0);
+    info.ready_seqno = ++last_ready_seqno;
+    NOEXCEPT_SCOPE({
+        ALLOW_ALLOCATIONS_IN_SCOPE;
+        ready_queue.emplace(info.key(), job);
+    });
+
+    if (is_running && workers < max_threads)
+        spawn(lock);
+}
+
+void AsyncLoader::spawn(std::unique_lock<std::mutex> &)
+{
+    workers++;
+    NOEXCEPT_SCOPE({
+        ALLOW_ALLOCATIONS_IN_SCOPE;
+        pool.scheduleOrThrowOnError([this] { worker(); });
+    });
+}
+
+void AsyncLoader::worker()
+{
+    DENY_ALLOCATIONS_IN_SCOPE;
+
+    LoadJobPtr job;
+    std::exception_ptr exception_from_job;
+    while (true)
+    {
+        // This is inside the loop to also reset previous thread names set inside the jobs
+        setThreadName("AsyncLoader");
+
+        {
+            std::unique_lock lock{mutex};
+
+            // Handle just executed job
+            if (exception_from_job)
+                finish(lock, job, LoadStatus::FAILED, exception_from_job);
+            else if (job)
+                finish(lock, job, LoadStatus::OK);
+
+            if (!is_running || ready_queue.empty() || workers > max_threads)
+            {
+                workers--;
+                return;
+            }
+
+            // Take next job to be executed from the ready queue
+            auto it = ready_queue.begin();
+            job = it->second;
+            ready_queue.erase(it);
+            scheduled_jobs.find(job)->second.ready_seqno = 0; // This job is no longer in the ready queue
+        }
+
+        ALLOW_ALLOCATIONS_IN_SCOPE;
+
+        try
+        {
+            job->func(job);
+            exception_from_job = {};
+        }
+        catch (...)
+        {
+            NOEXCEPT_SCOPE({
+                if (log_failures)
+                    tryLogCurrentException(__PRETTY_FUNCTION__);
+                exception_from_job = std::make_exception_ptr(
+                    Exception(ErrorCodes::ASYNC_LOAD_FAILED,
+                        "Load job '{}' failed: {}",
+                        job->name,
+                        getCurrentExceptionMessage(/* with_stacktrace = */ true)));
+            });
+        }
+    }
+}
+
+}
diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 34de4fe3c8b..bf35b5de736 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -8,14 +8,9 @@
 #include <unordered_set>
 #include <unordered_map>
 #include <boost/noncopyable.hpp>
-#include <base/defines.h>
 #include <base/types.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/ThreadPool.h>
-#include <Common/ErrorCodes.h>
-#include <Common/Exception.h>
-#include <Common/noexcept_scope.h>
-#include <Common/setThreadName.h>
 
 namespace DB
 {
@@ -26,21 +21,18 @@ using LoadJobSet = std::unordered_set<LoadJobPtr>;
 class LoadTask;
 class AsyncLoader;
 
-namespace ErrorCodes
-{
-    extern const int ASYNC_LOAD_SCHEDULE_FAILED;
-    extern const int ASYNC_LOAD_FAILED;
-    extern const int ASYNC_LOAD_CANCELED;
-}
-
+// Execution status of a load job.
 enum class LoadStatus
 {
-    PENDING,  // Load job is not started yet
-    OK,       // Load job executed and was successful
-    FAILED,   // Load job executed and failed
-    CANCELED  // Load job is not going to be executed due to removal or dependency failure
+    PENDING,  // Load job is not started yet.
+    OK,       // Load job executed and was successful.
+    FAILED,   // Load job executed and failed.
+    CANCELED  // Load job is not going to be executed due to removal or dependency failure.
 };
 
+// Smallest indivisible part of a loading process. Load job can have multiple dependencies, thus jobs constitute a direct acyclic graph (DAG).
+// Job encapsulates a function to be executed by `AsyncLoader` as soon as job functions of all dependencies are successfully executed.
+// Job can be waited for by an arbitrary number of threads. See `AsyncLoader` class description for more details.
 class LoadJob : private boost::noncopyable
 {
 public:
@@ -51,76 +43,35 @@ public:
         , func(std::forward<Func>(func_))
     {}
 
-    LoadStatus status() const
-    {
-        std::unique_lock lock{mutex};
-        return load_status;
-    }
+    // Current job status.
+    LoadStatus status() const;
+    std::exception_ptr exception() const;
 
-    std::exception_ptr exception() const
-    {
-        std::unique_lock lock{mutex};
-        return load_exception;
-    }
+    // Returns current value of a priority of the job. May differ from initial priority passed to `AsyncLoader:::schedule()` call.
+    ssize_t priority() const;
 
-    void wait() const
-    {
-        std::unique_lock lock{mutex};
-        waiters++;
-        finished.wait(lock, [this] { return load_status != LoadStatus::PENDING; });
-        waiters--;
-        if (load_exception)
-            std::rethrow_exception(load_exception);
-    }
+    // Sync wait for a pending job to be finished: OK, FAILED or CANCELED status.
+    // Throws if job is FAILED or CANCELED. Returns or throws immediately on non-pending job.
+    void wait() const;
 
-    void waitNoThrow() const
-    {
-        std::unique_lock lock{mutex};
-        waiters++;
-        finished.wait(lock, [this] { return load_status != LoadStatus::PENDING; });
-        waiters--;
-    }
+    // Wait for a job to reach any non PENDING status.
+    void waitNoThrow() const noexcept;
 
-    size_t waiters_count() const
-    {
-        std::unique_lock lock{mutex};
-        return waiters;
-    }
+    // Returns number of threads blocked by `wait()` or `waitNoThrow()` calls.
+    size_t waiters_count() const;
 
-    const LoadJobSet dependencies; // jobs to be done before this one (with ownership), it is `const` to make creation of cycles hard
+    const LoadJobSet dependencies; // Jobs to be done before this one (with ownership), it is `const` to make creation of cycles hard
     const String name;
-    std::atomic<ssize_t> priority{0};
 
 private:
     friend class AsyncLoader;
 
-    void ok()
-    {
-        std::unique_lock lock{mutex};
-        load_status = LoadStatus::OK;
-        if (waiters > 0)
-            finished.notify_all();
-    }
-
-    void failed(const std::exception_ptr & ptr)
-    {
-        std::unique_lock lock{mutex};
-        load_status = LoadStatus::FAILED;
-        load_exception = ptr;
-        if (waiters > 0)
-            finished.notify_all();
-    }
-
-    void canceled(const std::exception_ptr & ptr)
-    {
-        std::unique_lock lock{mutex};
-        load_status = LoadStatus::CANCELED;
-        load_exception = ptr;
-        if (waiters > 0)
-            finished.notify_all();
-    }
+    void ok();
+    void failed(const std::exception_ptr & ptr);
+    void canceled(const std::exception_ptr & ptr);
 
     std::function<void(const LoadJobPtr & self)> func;
+    std::atomic<ssize_t> load_priority{0};
 
     mutable std::mutex mutex;
     mutable std::condition_variable finished;
@@ -177,7 +128,7 @@ LoadJobPtr makeLoadJob(LoadJobSet && dependencies, const String & name, Func &&
 class AsyncLoader : private boost::noncopyable
 {
 private:
-    // Key of a pending job in ready queue
+    // Key of a pending job in the ready queue.
     struct ReadyKey
     {
         ssize_t priority; // Ascending order
@@ -198,14 +149,14 @@ private:
         }
     };
 
-    // Scheduling information for a pending job
+    // Scheduling information for a pending job.
     struct Info
     {
-        ssize_t initial_priority = 0; // Initial priority passed into schedule()
-        ssize_t priority = 0; // Elevated priority, due to priority inheritance or prioritize()
-        size_t dependencies_left = 0;
-        UInt64 ready_seqno = 0; // Zero means that job is not in ready queue
-        LoadJobSet dependent_jobs;
+        ssize_t initial_priority = 0; // Initial priority passed into schedule().
+        ssize_t priority = 0; // Elevated priority, due to priority inheritance or prioritize().
+        size_t dependencies_left = 0; // Current number of dependencies on pending jobs.
+        UInt64 ready_seqno = 0; // Zero means that job is not in ready queue.
+        LoadJobSet dependent_jobs; // Set of jobs dependent on this job.
 
         // Three independent states of a non-finished jobs
         bool is_blocked() const { return dependencies_left > 0; }
@@ -222,521 +173,96 @@ private:
 public:
     using Metric = CurrentMetrics::Metric;
 
-    // Helper class that removes all not started jobs in destructor and wait all executing jobs to finish
+    // Helper class that removes all not started and finished jobs in destructor and waits for all the executing jobs to finish.
     class Task
     {
     public:
-        Task()
-            : loader(nullptr)
-        {}
-
-        Task(AsyncLoader * loader_, LoadJobSet && jobs_)
-            : loader(loader_)
-            , jobs(std::move(jobs_))
-        {}
-
+        Task();
+        Task(AsyncLoader * loader_, LoadJobSet && jobs_);
         Task(const Task & o) = delete;
+        Task(Task && o) noexcept;
         Task & operator=(const Task & o) = delete;
+        ~Task();
+        Task & operator=(Task && o) noexcept;
 
-        Task(Task && o) noexcept
-            : loader(std::exchange(o.loader, nullptr))
-            , jobs(std::move(o.jobs))
-        {}
+        // Merge all jobs from other task into this task. Useful for merging jobs with different priorities into one task.
+        void merge(Task && o);
 
-        Task & operator=(Task && o) noexcept
-        {
-            loader = std::exchange(o.loader, nullptr);
-            jobs = std::move(o.jobs);
-            return *this;
-        }
+        // Remove all jobs from AsyncLoader.
+        void remove();
 
-        void merge(Task && o)
-        {
-            if (!loader)
-            {
-                *this = std::move(o);
-            }
-            else
-            {
-                chassert(loader == o.loader);
-                jobs.merge(o.jobs);
-                o.loader = nullptr;
-            }
-        }
-
-        ~Task()
-        {
-            remove();
-        }
-
-        void remove()
-        {
-            if (loader)
-            {
-                loader->remove(jobs);
-                detach();
-            }
-        }
-
-        // Do not track jobs in this task
-        // WARNING: Jobs will never be removed() and are going to be stored as finished_jobs until ~AsyncLoader()
-        void detach()
-        {
-            loader = nullptr;
-            jobs.clear();
-        }
+        // Do not track jobs in this task.
+        // WARNING: Jobs will never be removed() and are going to be stored as finished_jobs until ~AsyncLoader().
+        void detach();
 
     private:
         AsyncLoader * loader;
         LoadJobSet jobs;
     };
 
-    AsyncLoader(Metric metric_threads, Metric metric_active_threads, size_t max_threads_, bool log_failures_)
-        : log_failures(log_failures_)
-        , max_threads(max_threads_)
-        , pool(metric_threads, metric_active_threads, max_threads)
-    {}
+    AsyncLoader(Metric metric_threads, Metric metric_active_threads, size_t max_threads_, bool log_failures_);
 
-    // WARNING: all Task instances returned by `schedule()` should be destructed before AsyncLoader
-    ~AsyncLoader()
-    {
-        stop();
-    }
+    // WARNING: all Task instances returned by `schedule()` should be destructed before AsyncLoader.
+    ~AsyncLoader();
 
-    // Start workers to execute scheduled load jobs
-    void start()
-    {
-        std::unique_lock lock{mutex};
-        is_running = true;
-        for (size_t i = 0; workers < max_threads && i < ready_queue.size(); i++)
-            spawn(lock);
-    }
+    // Start workers to execute scheduled load jobs.
+    void start();
 
     // Wait for all load jobs to finish, including all new jobs. So at first take care to stop adding new jobs.
-    void wait()
-    {
-        pool.wait();
-    }
+    void wait();
 
     // Wait for currently executing jobs to finish, but do not run any other pending jobs.
     // Not finished jobs are left in pending state:
     //  - they can be resumed by calling start() again;
     //  - or canceled using ~Task() or remove() later.
-    void stop()
-    {
-        {
-            std::unique_lock lock{mutex};
-            is_running = false;
-            // NOTE: there is no need to notify because workers never wait
-        }
-        pool.wait();
-    }
+    void stop();
 
-    [[nodiscard]] Task schedule(LoadJobSet && jobs, ssize_t priority = 0)
-    {
-        std::unique_lock lock{mutex};
-
-        // Sanity checks
-        for (const auto & job : jobs)
-        {
-            if (job->status() != LoadStatus::PENDING)
-                throw Exception(ErrorCodes::ASYNC_LOAD_SCHEDULE_FAILED, "Trying to schedule already finished load job '{}'", job->name);
-            if (scheduled_jobs.contains(job))
-                throw Exception(ErrorCodes::ASYNC_LOAD_SCHEDULE_FAILED, "Load job '{}' has been already scheduled", job->name);
-        }
-
-        // Ensure scheduled_jobs graph will have no cycles. The only way to get a cycle is to add a cycle, assuming old jobs cannot reference new ones.
-        checkCycle(jobs, lock);
-
-        // We do not want any exception to be throws after this point, because the following code is not exception-safe
-        DENY_ALLOCATIONS_IN_SCOPE;
-
-        // Schedule all incoming jobs
-        for (const auto & job : jobs)
-        {
-            NOEXCEPT_SCOPE({
-                ALLOW_ALLOCATIONS_IN_SCOPE;
-                scheduled_jobs.emplace(job, Info{.initial_priority = priority, .priority = priority});
-            });
-            job->priority.store(priority); // Set user-facing priority
-        }
-
-        // Process dependencies on scheduled pending jobs
-        for (const auto & job : jobs)
-        {
-            Info & info = scheduled_jobs.find(job)->second;
-            for (const auto & dep : job->dependencies)
-            {
-                // Register every dependency on scheduled job with back-link to dependent job
-                if (auto dep_info = scheduled_jobs.find(dep); dep_info != scheduled_jobs.end())
-                {
-                    NOEXCEPT_SCOPE({
-                        ALLOW_ALLOCATIONS_IN_SCOPE;
-                        dep_info->second.dependent_jobs.insert(job);
-                    });
-                    info.dependencies_left++;
-
-                    // Priority inheritance: prioritize deps to have at least given `priority` to avoid priority inversion
-                    prioritize(dep, priority, lock);
-                }
-            }
-
-            // Enqueue non-blocked jobs (w/o dependencies) to ready queue
-            if (!info.is_blocked())
-                enqueue(info, job, lock);
-        }
-
-        // Process dependencies on other jobs. It is done in a separate pass to facilitate propagation of cancel signals (if any).
-        for (const auto & job : jobs)
-        {
-            if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
-            {
-                for (const auto & dep : job->dependencies)
-                {
-                    if (scheduled_jobs.contains(dep))
-                        continue; // Skip dependencies on scheduled pending jobs (already processed)
-                    LoadStatus dep_status = dep->status();
-                    if (dep_status == LoadStatus::OK)
-                        continue; // Dependency on already successfully finished job -- it's okay.
-
-                    if (dep_status == LoadStatus::PENDING)
-                    {
-                        // Dependency on not scheduled pending job -- it's bad.
-                        // Probably, there is an error in `jobs` set: not all jobs were passed to `schedule()` call.
-                        // We are not going to run any dependent job, so cancel them all.
-                        std::exception_ptr e;
-                        NOEXCEPT_SCOPE({
-                            ALLOW_ALLOCATIONS_IN_SCOPE;
-                            e = std::make_exception_ptr(Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
-                                "Load job '{}' -> Load job '{}': not scheduled pending load job (it must be also scheduled), all dependent load jobs are canceled",
-                                job->name,
-                                dep->name));
-                        });
-                        finish(lock, job, LoadStatus::CANCELED, e);
-                        break; // This job is now finished, stop its dependencies processing
-                    }
-                    if (dep_status == LoadStatus::FAILED || dep_status == LoadStatus::CANCELED)
-                    {
-                        // Dependency on already failed or canceled job -- it's okay. Cancel all dependent jobs.
-                        std::exception_ptr e;
-                        NOEXCEPT_SCOPE({
-                            ALLOW_ALLOCATIONS_IN_SCOPE;
-                            e = std::make_exception_ptr(Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
-                                "Load job '{}' -> {}",
-                                job->name,
-                                getExceptionMessage(dep->exception(), /* with_stack_trace = */ false)));
-                        });
-                        finish(lock, job, LoadStatus::CANCELED, e);
-                        break; // This job is now finished, stop its dependencies processing
-                    }
-                }
-            }
-            else
-            {
-                // Job was already canceled on previous iteration of this cycle -- skip
-            }
-        }
-
-        return Task(this, std::move(jobs));
-    }
+    // Schedule all `jobs` with given `priority` and return task containing these jobs.
+    // Higher priority jobs (with greater `priority` value) are executed earlier.
+    // All dependencies of a scheduled job inherit its priority if it is higher. This way higher priority job
+    // never wait for (blocked by) lower priority jobs (no priority inversion).
+    [[nodiscard]] Task schedule(LoadJobSet && jobs, ssize_t priority = 0);
 
     // Increase priority of a job and all its dependencies recursively
-    void prioritize(const LoadJobPtr & job, ssize_t new_priority)
-    {
-        DENY_ALLOCATIONS_IN_SCOPE;
-        std::unique_lock lock{mutex};
-        prioritize(job, new_priority, lock);
-    }
+    void prioritize(const LoadJobPtr & job, ssize_t new_priority);
 
-    // Remove finished jobs, cancel scheduled jobs, wait for executing jobs to finish and remove them
-    void remove(const LoadJobSet & jobs)
-    {
-        DENY_ALLOCATIONS_IN_SCOPE;
-        std::unique_lock lock{mutex};
-        // On the first pass:
-        // - cancel all not executing jobs to avoid races
-        // - do not wait executing jobs (otherwise, on unlock a worker could start executing a dependent job, that should be canceled)
-        for (const auto & job : jobs)
-        {
-            if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
-            {
-                if (info->second.is_executing())
-                    continue; // Skip executing jobs on the first pass
-                std::exception_ptr e;
-                NOEXCEPT_SCOPE({
-                    ALLOW_ALLOCATIONS_IN_SCOPE;
-                    e = std::make_exception_ptr(Exception(ErrorCodes::ASYNC_LOAD_CANCELED, "Load job '{}' canceled", job->name));
-                });
-                finish(lock, job, LoadStatus::CANCELED, e);
-            }
-        }
-        // On the second pass wait for executing jobs to finish
-        for (const auto & job : jobs)
-        {
-            if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
-            {
-                // Job is currently executing
-                chassert(info->second.is_executing());
-                lock.unlock();
-                job->waitNoThrow(); // Wait for job to finish
-                lock.lock();
-            }
-        }
-        // On the third pass all jobs are finished - remove them all
-        // It is better to do it under one lock to avoid exposing intermediate states
-        for (const auto & job : jobs)
-            finished_jobs.erase(job);
-    }
+    // Remove finished jobs, cancel scheduled jobs, wait for executing jobs to finish and remove them.
+    void remove(const LoadJobSet & jobs);
 
-    void setMaxThreads(size_t value)
-    {
-        std::unique_lock lock{mutex};
-        pool.setMaxThreads(value);
-        pool.setMaxFreeThreads(value);
-        pool.setQueueSize(value);
-        max_threads = value;
-        if (!is_running)
-            return;
-        for (size_t i = 0; workers < max_threads && i < ready_queue.size(); i++)
-            spawn(lock);
-    }
+    // Increase or decrease maximum number of simultaneously executing jobs.
+    void setMaxThreads(size_t value);
 
-    size_t getMaxThreads() const
-    {
-        std::unique_lock lock{mutex};
-        return max_threads;
-    }
-
-    size_t getScheduledJobCount() const
-    {
-        std::unique_lock lock{mutex};
-        return scheduled_jobs.size();
-    }
+    size_t getMaxThreads() const;
+    size_t getScheduledJobCount() const;
 
 private:
-    void checkCycle(const LoadJobSet & jobs, std::unique_lock<std::mutex> & lock)
-    {
-        LoadJobSet left = jobs;
-        LoadJobSet visited;
-        visited.reserve(left.size());
-        while (!left.empty())
-        {
-            LoadJobPtr job = *left.begin();
-            checkCycleImpl(job, left, visited, lock);
-        }
-    }
+    void checkCycle(const LoadJobSet & jobs, std::unique_lock<std::mutex> & lock);
+    String checkCycleImpl(const LoadJobPtr & job, LoadJobSet & left, LoadJobSet & visited, std::unique_lock<std::mutex> & lock);
+    void finish(std::unique_lock<std::mutex> & lock, const LoadJobPtr & job, LoadStatus status, std::exception_ptr exception_from_job = {});
+    void prioritize(const LoadJobPtr & job, ssize_t new_priority, std::unique_lock<std::mutex> & lock);
+    void enqueue(Info & info, const LoadJobPtr & job, std::unique_lock<std::mutex> & lock);
+    void spawn(std::unique_lock<std::mutex> &);
+    void worker();
 
-    String checkCycleImpl(const LoadJobPtr & job, LoadJobSet & left, LoadJobSet & visited, std::unique_lock<std::mutex> & lock)
-    {
-        if (!left.contains(job))
-            return {}; // Do not consider external dependencies and already processed jobs
-        if (auto [_, inserted] = visited.insert(job); !inserted)
-        {
-            visited.erase(job); // Mark where cycle ends
-            return job->name;
-        }
-        for (const auto & dep : job->dependencies)
-        {
-            if (auto chain = checkCycleImpl(dep, left, visited, lock); !chain.empty())
-            {
-                if (!visited.contains(job)) // Check for cycle end
-                    throw Exception(ErrorCodes::ASYNC_LOAD_SCHEDULE_FAILED, "Load job dependency cycle detected: {} -> {}", job->name, chain);
-                else
-                    return fmt::format("{} -> {}", job->name, chain); // chain is not a cycle yet -- continue building
-            }
-        }
-        left.erase(job);
-        return {};
-    }
+    const bool log_failures; // Worker should log all exceptions caught from job functions.
 
-    void finish(std::unique_lock<std::mutex> & lock, const LoadJobPtr & job, LoadStatus status, std::exception_ptr exception_from_job = {})
-    {
-        if (status == LoadStatus::OK)
-        {
-            // Notify waiters
-            job->ok();
-
-            // Update dependent jobs and enqueue if ready
-            chassert(scheduled_jobs.contains(job)); // Job was pending
-            for (const auto & dep : scheduled_jobs[job].dependent_jobs)
-            {
-                chassert(scheduled_jobs.contains(dep)); // All depended jobs must be pending
-                Info & dep_info = scheduled_jobs[dep];
-                dep_info.dependencies_left--;
-                if (!dep_info.is_blocked())
-                    enqueue(dep_info, dep, lock);
-            }
-        }
-        else
-        {
-            // Notify waiters
-            if (status == LoadStatus::FAILED)
-                job->failed(exception_from_job);
-            else if (status == LoadStatus::CANCELED)
-                job->canceled(exception_from_job);
-
-            chassert(scheduled_jobs.contains(job)); // Job was pending
-            Info & info = scheduled_jobs[job];
-            if (info.is_ready())
-            {
-                ready_queue.erase(info.key());
-                info.ready_seqno = 0;
-            }
-
-            // Recurse into all dependent jobs
-            LoadJobSet dependent;
-            dependent.swap(info.dependent_jobs); // To avoid container modification during recursion
-            for (const auto & dep : dependent)
-            {
-                std::exception_ptr e;
-                NOEXCEPT_SCOPE({
-                    ALLOW_ALLOCATIONS_IN_SCOPE;
-                    e = std::make_exception_ptr(
-                        Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
-                            "Load job '{}' -> {}",
-                            dep->name,
-                            getExceptionMessage(exception_from_job, /* with_stack_trace = */ false)));
-                });
-                finish(lock, dep, LoadStatus::CANCELED, e);
-            }
-
-            // Clean dependency graph edges
-            for (const auto & dep : job->dependencies)
-                if (auto dep_info = scheduled_jobs.find(dep); dep_info != scheduled_jobs.end())
-                    dep_info->second.dependent_jobs.erase(job);
-        }
-
-        // Job became finished
-        scheduled_jobs.erase(job);
-        NOEXCEPT_SCOPE({
-            ALLOW_ALLOCATIONS_IN_SCOPE;
-            finished_jobs.insert(job);
-        });
-    }
-
-    void prioritize(const LoadJobPtr & job, ssize_t new_priority, std::unique_lock<std::mutex> & lock)
-    {
-        if (auto info = scheduled_jobs.find(job); info != scheduled_jobs.end())
-        {
-            if (info->second.priority >= new_priority)
-                return; // Never lower priority
-
-            // Update priority and push job forward through ready queue if needed
-            if (info->second.ready_seqno)
-                ready_queue.erase(info->second.key());
-            info->second.priority = new_priority;
-            job->priority.store(new_priority); // Set user-facing priority (may affect executing jobs)
-            if (info->second.ready_seqno)
-            {
-                NOEXCEPT_SCOPE({
-                    ALLOW_ALLOCATIONS_IN_SCOPE;
-                    ready_queue.emplace(info->second.key(), job);
-                });
-            }
-
-            // Recurse into dependencies
-            for (const auto & dep : job->dependencies)
-                prioritize(dep, new_priority, lock);
-        }
-    }
-
-    void enqueue(Info & info, const LoadJobPtr & job, std::unique_lock<std::mutex> & lock)
-    {
-        chassert(!info.is_blocked());
-        chassert(info.ready_seqno == 0);
-        info.ready_seqno = ++last_ready_seqno;
-        NOEXCEPT_SCOPE({
-            ALLOW_ALLOCATIONS_IN_SCOPE;
-            ready_queue.emplace(info.key(), job);
-        });
-
-        if (is_running && workers < max_threads)
-            spawn(lock);
-    }
-
-    void spawn(std::unique_lock<std::mutex> &)
-    {
-        workers++;
-        NOEXCEPT_SCOPE({
-            ALLOW_ALLOCATIONS_IN_SCOPE;
-            pool.scheduleOrThrowOnError([this] { worker(); });
-        });
-    }
-
-    void worker()
-    {
-        DENY_ALLOCATIONS_IN_SCOPE;
-
-        LoadJobPtr job;
-        std::exception_ptr exception_from_job;
-        while (true)
-        {
-            /// This is inside the loop to also reset previous thread names set inside the jobs
-            setThreadName("AsyncLoader");
-
-            {
-                std::unique_lock lock{mutex};
-
-                // Handle just executed job
-                if (exception_from_job)
-                    finish(lock, job, LoadStatus::FAILED, exception_from_job);
-                else if (job)
-                    finish(lock, job, LoadStatus::OK);
-
-                if (!is_running || ready_queue.empty() || workers > max_threads)
-                {
-                    workers--;
-                    return;
-                }
-
-                // Take next job to be executed from the ready queue
-                auto it = ready_queue.begin();
-                job = it->second;
-                ready_queue.erase(it);
-                scheduled_jobs.find(job)->second.ready_seqno = 0; // This job is no longer in the ready queue
-            }
-
-            ALLOW_ALLOCATIONS_IN_SCOPE;
-
-            try
-            {
-                job->func(job);
-                exception_from_job = {};
-            }
-            catch (...)
-            {
-                NOEXCEPT_SCOPE({
-                    if (log_failures)
-                        tryLogCurrentException(__PRETTY_FUNCTION__);
-                    exception_from_job = std::make_exception_ptr(
-                        Exception(ErrorCodes::ASYNC_LOAD_FAILED,
-                            "Load job '{}' failed: {}",
-                            job->name,
-                            getCurrentExceptionMessage(/* with_stacktrace = */ true)));
-                });
-            }
-        }
-    }
-
-    const bool log_failures;
-
-    mutable std::mutex mutex;
+    mutable std::mutex mutex; // Guards all the fields below.
     bool is_running = false;
 
-    // Full set of scheduled pending jobs along with scheduling info
+    // Full set of scheduled pending jobs along with scheduling info.
     std::unordered_map<LoadJobPtr, Info> scheduled_jobs;
 
-    // Subset of scheduled pending jobs with resolved dependencies (waiting for a thread to be executed)
-    // Represent a queue of jobs in order of decreasing priority and FIFO for jobs with equal priorities
+    // Subset of scheduled pending non-blocked jobs (waiting for a worker to be executed).
+    // Represent a queue of jobs in order of decreasing priority and FIFO for jobs with equal priorities.
     std::map<ReadyKey, LoadJobPtr> ready_queue;
 
-    // Set of finished jobs (for introspection only, until job is removed)
+    // Set of finished jobs (for introspection only, until jobs are removed).
     LoadJobSet finished_jobs;
 
-    // Increasing counter for ReadyKey assignment (to preserve FIFO order of jobs with equal priority)
+    // Increasing counter for `ReadyKey` assignment (to preserve FIFO order of the jobs with equal priorities).
     UInt64 last_ready_seqno = 0;
 
-    // For executing jobs. Note that we avoid using an internal queue of the pool to be able to prioritize jobs
+    // For executing jobs. Note that we avoid using an internal queue of the pool to be able to prioritize jobs.
     size_t max_threads;
     size_t workers = 0;
     ThreadPool pool;
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 52cc3436595..e6baa87c182 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -115,7 +115,7 @@ TEST(AsyncLoader, Smoke)
 
     auto job_func = [&] (const LoadJobPtr & self) {
         jobs_done++;
-        if (self->priority == low_priority)
+        if (self->priority() == low_priority)
             low_priority_jobs_done++;
     };
 
@@ -560,7 +560,7 @@ TEST(AsyncLoader, StaticPriorities)
 
     auto job_func = [&] (const LoadJobPtr & self)
     {
-        schedule += fmt::format("{}{}", self->name, self->priority);
+        schedule += fmt::format("{}{}", self->name, self->priority());
     };
 
     std::vector<LoadJobPtr> jobs;

From 0d7e3d410075560c22dd53a2d20b39d8f602428b Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sun, 16 Apr 2023 18:18:50 +0000
Subject: [PATCH 051/689] fix comment

---
 src/Common/AsyncLoader.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index bf35b5de736..4b05dd40bf1 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -89,7 +89,7 @@ LoadJobPtr makeLoadJob(LoadJobSet && dependencies, const String & name, Func &&
 // `AsyncLoader` is a scheduler for DAG of `LoadJob`s. It tracks dependencies and priorities of jobs.
 // Basic usage example:
 //     auto job_func = [&] (const LoadJobPtr & self) {
-//         LOG_TRACE(log, "Executing load job '{}' with priority '{}'", self->name, self->priority);
+//         LOG_TRACE(log, "Executing load job '{}' with priority '{}'", self->name, self->priority());
 //     };
 //     auto job1 = makeLoadJob({}, "job1", job_func);
 //     auto job2 = makeLoadJob({ job1 }, "job2", job_func);
@@ -120,7 +120,7 @@ LoadJobPtr makeLoadJob(LoadJobSet && dependencies, const String & name, Func &&
 // Every job has a priority associated with it. AsyncLoader runs higher priority (greater `priority` value) jobs first. Job priority can be elevated
 // (a) if either it has a dependent job with higher priority (in this case priority of a dependent job is inherited);
 // (b) or job was explicitly prioritized by `prioritize(job, higher_priority)` call (this also leads to a priority inheritance for all the dependencies).
-// Note that to avoid priority inversion `job_func` should use `self->priority` to schedule new jobs in AsyncLoader or any other pool.
+// Note that to avoid priority inversion `job_func` should use `self->priority()` to schedule new jobs in AsyncLoader or any other pool.
 // Value stored in load job priority field is atomic and can be increased even during job execution.
 //
 // When task is scheduled it can contain dependencies on previously scheduled jobs. These jobs can have any status.

From 911c46aa3298d7ee32fd745e56261924ec67cf11 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sun, 16 Apr 2023 18:40:24 +0000
Subject: [PATCH 052/689] fix double finish() during cancellation of a job with
 multiple paths

---
 src/Common/AsyncLoader.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
index 09dd4f129ae..8dbfe27e5cc 100644
--- a/src/Common/AsyncLoader.cpp
+++ b/src/Common/AsyncLoader.cpp
@@ -427,6 +427,8 @@ void AsyncLoader::finish(std::unique_lock<std::mutex> & lock, const LoadJobPtr &
         dependent.swap(info.dependent_jobs); // To avoid container modification during recursion
         for (const auto & dep : dependent)
         {
+            if (!scheduled_jobs.contains(dep))
+                continue; // Job has already been canceled
             std::exception_ptr e;
             NOEXCEPT_SCOPE({
                 ALLOW_ALLOCATIONS_IN_SCOPE;
@@ -439,7 +441,7 @@ void AsyncLoader::finish(std::unique_lock<std::mutex> & lock, const LoadJobPtr &
             finish(lock, dep, LoadStatus::CANCELED, e);
         }
 
-        // Clean dependency graph edges
+        // Clean dependency graph edges pointing to canceled jobs
         for (const auto & dep : job->dependencies)
             if (auto dep_info = scheduled_jobs.find(dep); dep_info != scheduled_jobs.end())
                 dep_info->second.dependent_jobs.erase(job);

From 54c5b1083bf2aa123cb16f122f570faed62be6c5 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sun, 16 Apr 2023 19:28:14 +0000
Subject: [PATCH 053/689] add test for dynamic prioritization

---
 src/Common/AsyncLoader.cpp              |  2 +
 src/Common/tests/gtest_async_loader.cpp | 56 +++++++++++++++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
index 8dbfe27e5cc..0734741f469 100644
--- a/src/Common/AsyncLoader.cpp
+++ b/src/Common/AsyncLoader.cpp
@@ -284,6 +284,8 @@ AsyncLoader::Task AsyncLoader::schedule(LoadJobSet && jobs, ssize_t priority)
 
 void AsyncLoader::prioritize(const LoadJobPtr & job, ssize_t new_priority)
 {
+    if (!job)
+        return;
     DENY_ALLOCATIONS_IN_SCOPE;
     std::unique_lock lock{mutex};
     prioritize(job, new_priority, lock);
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index e6baa87c182..21a0ba1f41e 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -587,6 +587,62 @@ TEST(AsyncLoader, StaticPriorities)
     ASSERT_EQ(schedule, "A9E9D9F9G9H9C4B3");
 }
 
+TEST(AsyncLoader, DynamicPriorities)
+{
+    AsyncLoaderTest t(1);
+
+    for (bool prioritize : {false, true})
+    {
+        std::string schedule;
+
+        LoadJobPtr job_to_prioritize;
+
+        auto job_func = [&] (const LoadJobPtr & self)
+        {
+            if (prioritize && self->name == "C")
+                t.loader.prioritize(job_to_prioritize, 9); // dynamic prioritization
+            schedule += fmt::format("{}{}", self->name, self->priority());
+        };
+
+        // Job DAG with initial priorities. During execution of C4, job G0 priority is increased to G9, postponing B3 job executing.
+        // A0 -+-> B3
+        //     |
+        //     `-> C4
+        //     |
+        //     `-> D1 -.
+        //     |       +-> F0 --> G0 --> H0
+        //     `-> E2 -'
+        std::vector<LoadJobPtr> jobs;
+        jobs.push_back(makeLoadJob({}, "A", job_func)); // 0
+        jobs.push_back(makeLoadJob({ jobs[0] }, "B", job_func)); // 1
+        jobs.push_back(makeLoadJob({ jobs[0] }, "C", job_func)); // 2
+        jobs.push_back(makeLoadJob({ jobs[0] }, "D", job_func)); // 3
+        jobs.push_back(makeLoadJob({ jobs[0] }, "E", job_func)); // 4
+        jobs.push_back(makeLoadJob({ jobs[3], jobs[4] }, "F", job_func)); // 5
+        jobs.push_back(makeLoadJob({ jobs[5] }, "G", job_func)); // 6
+        jobs.push_back(makeLoadJob({ jobs[6] }, "H", job_func)); // 7
+        auto task = t.loader.schedule({ jobs[0] }, 0);
+        task.merge(t.loader.schedule({ jobs[1] }, 3));
+        task.merge(t.loader.schedule({ jobs[2] }, 4));
+        task.merge(t.loader.schedule({ jobs[3] }, 1));
+        task.merge(t.loader.schedule({ jobs[4] }, 2));
+        task.merge(t.loader.schedule({ jobs[5] }, 0));
+        task.merge(t.loader.schedule({ jobs[6] }, 0));
+        task.merge(t.loader.schedule({ jobs[7] }, 0));
+
+        job_to_prioritize = jobs[6];
+
+        t.loader.start();
+        t.loader.wait();
+        t.loader.stop();
+
+        if (prioritize)
+            ASSERT_EQ(schedule, "A4C4E9D9F9G9B3H0");
+        else
+            ASSERT_EQ(schedule, "A4C4B3E2D1F0G0H0");
+    }
+}
+
 TEST(AsyncLoader, RandomIndependentTasks)
 {
     AsyncLoaderTest t(16);

From 326c733be6908c415ca1094f36ce6982d2e34af9 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sun, 16 Apr 2023 21:15:44 +0000
Subject: [PATCH 054/689] fix tidy build

---
 src/Common/AsyncLoader.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
index 0734741f469..8da41a03d3d 100644
--- a/src/Common/AsyncLoader.cpp
+++ b/src/Common/AsyncLoader.cpp
@@ -266,7 +266,7 @@ AsyncLoader::Task AsyncLoader::schedule(LoadJobSet && jobs, ssize_t priority)
                         e = std::make_exception_ptr(Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
                             "Load job '{}' -> {}",
                             job->name,
-                            getExceptionMessage(dep->exception(), /* with_stack_trace = */ false)));
+                            getExceptionMessage(dep->exception(), /* with_stacktrace = */ false)));
                     });
                     finish(lock, job, LoadStatus::CANCELED, e);
                     break; // This job is now finished, stop its dependencies processing
@@ -438,7 +438,7 @@ void AsyncLoader::finish(std::unique_lock<std::mutex> & lock, const LoadJobPtr &
                     Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
                         "Load job '{}' -> {}",
                         dep->name,
-                        getExceptionMessage(exception_from_job, /* with_stack_trace = */ false)));
+                        getExceptionMessage(exception_from_job, /* with_stacktrace = */ false)));
             });
             finish(lock, dep, LoadStatus::CANCELED, e);
         }

From baced2c66ecb37f645d2ef89740dd11e525aa4dd Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Mon, 17 Apr 2023 09:17:51 +0000
Subject: [PATCH 055/689] fix data race in test

---
 src/Common/tests/gtest_async_loader.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 21a0ba1f41e..d6039162994 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -751,4 +751,5 @@ TEST(AsyncLoader, SetMaxThreads)
             t.loader.setMaxThreads(max_threads_values[sync_index]);
         syncs[idx]->arrive_and_wait(); // (B) this sync point is required to allow `executing` value to go back down to zero after we change number of workers
     }
+    t.loader.wait();
 }

From d4daea5f0906d559657f102cf6332fd3e3982bf2 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Mon, 17 Apr 2023 09:47:16 +0000
Subject: [PATCH 056/689] fix typos

---
 src/Common/AsyncLoader.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 4b05dd40bf1..383d4b5ef24 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -95,9 +95,9 @@ LoadJobPtr makeLoadJob(LoadJobSet && dependencies, const String & name, Func &&
 //     auto job2 = makeLoadJob({ job1 }, "job2", job_func);
 //     auto job3 = makeLoadJob({ job1 }, "job3", job_func);
 //     auto task = async_loader.schedule({ job1, job2, job3 }, /* priority = */ 0);
-// Here we have created and scheduled a task consisting of two jobs. Job1 has no dependencies and is run first.
-// Job2 and job3 depends on job1 and are run only after job1 completion. Another thread may prioritize a job and wait for it:
-//     async_loader->prioritize(job3, 1); // higher priority jobs are run first
+// Here we have created and scheduled a task consisting of three jobs. Job1 has no dependencies and is run first.
+// Job2 and job3 depend on job1 and are run only after job1 completion. Another thread may prioritize a job and wait for it:
+//     async_loader->prioritize(job3, /* priority = */ 1); // higher priority jobs are run first
 //     job3->wait(); // blocks until job completion or cancellation and rethrow an exception (if any)
 //
 // AsyncLoader tracks state of all scheduled jobs. Job lifecycle is the following:

From 8cc9b32af1a80dd25f2c4f480949090a0d454967 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Mon, 17 Apr 2023 10:03:54 +0000
Subject: [PATCH 057/689] fix more comments

---
 src/Common/AsyncLoader.h | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 383d4b5ef24..d3de883f757 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -158,7 +158,7 @@ private:
         UInt64 ready_seqno = 0; // Zero means that job is not in ready queue.
         LoadJobSet dependent_jobs; // Set of jobs dependent on this job.
 
-        // Three independent states of a non-finished jobs
+        // Three independent states of a non-finished job.
         bool is_blocked() const { return dependencies_left > 0; }
         bool is_ready() const { return dependencies_left == 0 && ready_seqno > 0; }
         bool is_executing() const { return dependencies_left == 0 && ready_seqno == 0; }
@@ -188,11 +188,11 @@ public:
         // Merge all jobs from other task into this task. Useful for merging jobs with different priorities into one task.
         void merge(Task && o);
 
-        // Remove all jobs from AsyncLoader.
+        // Remove all jobs of this task from AsyncLoader.
         void remove();
 
         // Do not track jobs in this task.
-        // WARNING: Jobs will never be removed() and are going to be stored as finished_jobs until ~AsyncLoader().
+        // WARNING: Jobs will never be removed() and are going to be stored as finished jobs until ~AsyncLoader().
         void detach();
 
     private:
@@ -213,17 +213,19 @@ public:
 
     // Wait for currently executing jobs to finish, but do not run any other pending jobs.
     // Not finished jobs are left in pending state:
-    //  - they can be resumed by calling start() again;
+    //  - they can be executed by calling start() again;
     //  - or canceled using ~Task() or remove() later.
     void stop();
 
-    // Schedule all `jobs` with given `priority` and return task containing these jobs.
+    // Schedule all `jobs` with given `priority` and return a task containing these jobs.
     // Higher priority jobs (with greater `priority` value) are executed earlier.
     // All dependencies of a scheduled job inherit its priority if it is higher. This way higher priority job
     // never wait for (blocked by) lower priority jobs (no priority inversion).
+    // Returned task destructor ensures that all the `jobs` are finished (OK, FAILED or CANCELED)
+    // and are removed from AsyncLoader, so it is thread-safe to destroy them.
     [[nodiscard]] Task schedule(LoadJobSet && jobs, ssize_t priority = 0);
 
-    // Increase priority of a job and all its dependencies recursively
+    // Increase priority of a job and all its dependencies recursively.
     void prioritize(const LoadJobPtr & job, ssize_t new_priority);
 
     // Remove finished jobs, cancel scheduled jobs, wait for executing jobs to finish and remove them.

From c672d117ea78c48d036dcf40b7c3c5774644fb74 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Mon, 17 Apr 2023 10:11:59 +0000
Subject: [PATCH 058/689] better job shutdown

---
 src/Common/AsyncLoader.cpp | 12 ++++++++----
 src/Common/AsyncLoader.h   |  1 +
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
index 8da41a03d3d..d382100a203 100644
--- a/src/Common/AsyncLoader.cpp
+++ b/src/Common/AsyncLoader.cpp
@@ -62,8 +62,7 @@ void LoadJob::ok()
 {
     std::unique_lock lock{mutex};
     load_status = LoadStatus::OK;
-    if (waiters > 0)
-        finished.notify_all();
+    finish();
 }
 
 void LoadJob::failed(const std::exception_ptr & ptr)
@@ -71,8 +70,7 @@ void LoadJob::failed(const std::exception_ptr & ptr)
     std::unique_lock lock{mutex};
     load_status = LoadStatus::FAILED;
     load_exception = ptr;
-    if (waiters > 0)
-        finished.notify_all();
+    finish();
 }
 
 void LoadJob::canceled(const std::exception_ptr & ptr)
@@ -80,6 +78,12 @@ void LoadJob::canceled(const std::exception_ptr & ptr)
     std::unique_lock lock{mutex};
     load_status = LoadStatus::CANCELED;
     load_exception = ptr;
+    finish();
+}
+
+void LoadJob::finish()
+{
+    func = {}; // To ensure job function is destructed before `AsyncLoader::wait()` and `task->wait()` return
     if (waiters > 0)
         finished.notify_all();
 }
diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index d3de883f757..2e2c605cfe6 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -69,6 +69,7 @@ private:
     void ok();
     void failed(const std::exception_ptr & ptr);
     void canceled(const std::exception_ptr & ptr);
+    void finish();
 
     std::function<void(const LoadJobPtr & self)> func;
     std::atomic<ssize_t> load_priority{0};

From ff7444f0772eb201dcb4f591a6578eb745ba5aff Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Mon, 17 Apr 2023 10:16:01 +0000
Subject: [PATCH 059/689] typo

---
 src/Common/AsyncLoader.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
index d382100a203..7fe88ce7212 100644
--- a/src/Common/AsyncLoader.cpp
+++ b/src/Common/AsyncLoader.cpp
@@ -83,7 +83,7 @@ void LoadJob::canceled(const std::exception_ptr & ptr)
 
 void LoadJob::finish()
 {
-    func = {}; // To ensure job function is destructed before `AsyncLoader::wait()` and `task->wait()` return
+    func = {}; // To ensure job function is destructed before `AsyncLoader::wait()` and `LoadJob::wait()` return
     if (waiters > 0)
         finished.notify_all();
 }

From 31e3a3bc0c1824c0c1b65b33596ee967b6bc9c6b Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Wed, 19 Apr 2023 13:35:58 +0200
Subject: [PATCH 060/689] Update src/Common/AsyncLoader.h

Co-authored-by: Antonio Andelic <antonio2368@users.noreply.github.com>
---
 src/Common/AsyncLoader.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 2e2c605cfe6..5f87601e753 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -37,9 +37,9 @@ class LoadJob : private boost::noncopyable
 {
 public:
     template <class Func>
-    LoadJob(LoadJobSet && dependencies_, const String & name_, Func && func_)
+    LoadJob(LoadJobSet && dependencies_, String name_, Func && func_)
         : dependencies(std::move(dependencies_))
-        , name(name_)
+        , name(std::move(name_))
         , func(std::forward<Func>(func_))
     {}
 

From 2e31140ca4e2740d6b9b1dd3ed2a585ab1b2ec92 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Wed, 19 Apr 2023 11:46:56 +0000
Subject: [PATCH 061/689] review fixes

---
 src/Common/AsyncLoader.cpp              | 2 +-
 src/Common/AsyncLoader.h                | 2 +-
 src/Common/tests/gtest_async_loader.cpp | 9 +++++----
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
index 7fe88ce7212..1d428b3601f 100644
--- a/src/Common/AsyncLoader.cpp
+++ b/src/Common/AsyncLoader.cpp
@@ -52,7 +52,7 @@ void LoadJob::waitNoThrow() const noexcept
     waiters--;
 }
 
-size_t LoadJob::waiters_count() const
+size_t LoadJob::waitersCount() const
 {
     std::unique_lock lock{mutex};
     return waiters;
diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 2e2c605cfe6..26a192d56c8 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -58,7 +58,7 @@ public:
     void waitNoThrow() const noexcept;
 
     // Returns number of threads blocked by `wait()` or `waitNoThrow()` calls.
-    size_t waiters_count() const;
+    size_t waitersCount() const;
 
     const LoadJobSet dependencies; // Jobs to be done before this one (with ownership), it is `const` to make creation of cycles hard
     const String name;
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index d6039162994..c05dbb751ca 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -3,6 +3,7 @@
 #include <barrier>
 #include <chrono>
 #include <mutex>
+#include <stdexcept>
 #include <string_view>
 #include <vector>
 #include <thread>
@@ -313,7 +314,7 @@ TEST(AsyncLoader, CancelExecutingJob)
     {
         task.remove(); // waits for (C)
     });
-    while (job->waiters_count() == 0)
+    while (job->waitersCount() == 0)
         std::this_thread::yield();
     ASSERT_EQ(job->status(), LoadStatus::PENDING);
     sync.arrive_and_wait(); // (B) sync with job
@@ -362,7 +363,7 @@ TEST(AsyncLoader, CancelExecutingTask)
         {
             task1.remove(); // waits for (C)
         });
-        while (blocker_job->waiters_count() == 0)
+        while (blocker_job->waitersCount() == 0)
             std::this_thread::yield();
         ASSERT_EQ(blocker_job->status(), LoadStatus::PENDING);
         sync.arrive_and_wait(); // (B) sync with job
@@ -384,10 +385,10 @@ TEST(AsyncLoader, JobFailure)
     AsyncLoaderTest t;
     t.loader.start();
 
-    std::string_view error_message = "test job failure";
+    std::string error_message = "test job failure";
 
     auto job_func = [&] (const LoadJobPtr &) {
-        throw Exception(ErrorCodes::ASYNC_LOAD_FAILED, "{}", error_message);
+        throw std::runtime_error(error_message);
     };
 
     auto job = makeLoadJob({}, "job", job_func);

From 4b51f12479b28e584735d747756eea79ba4b9471 Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Wed, 19 Apr 2023 13:48:38 +0200
Subject: [PATCH 062/689] Update src/Common/AsyncLoader.h

Co-authored-by: Antonio Andelic <antonio2368@users.noreply.github.com>
---
 src/Common/AsyncLoader.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index e393fbde7d6..6f49b9b2d72 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -82,9 +82,9 @@ private:
 };
 
 template <class Func>
-LoadJobPtr makeLoadJob(LoadJobSet && dependencies, const String & name, Func && func)
+LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, Func && func)
 {
-    return std::make_shared<LoadJob>(std::move(dependencies), name, std::forward<Func>(func));
+    return std::make_shared<LoadJob>(std::move(dependencies), std::move(name), std::forward<Func>(func));
 }
 
 // `AsyncLoader` is a scheduler for DAG of `LoadJob`s. It tracks dependencies and priorities of jobs.

From 966902fdfda744bfb386cd555c21ae168af02d53 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Thu, 16 Feb 2023 16:25:07 +0800
Subject: [PATCH 063/689] extend firt_value/last_value to accept null

---
 .../AggregateFunctionAny.cpp                  |  14 +-
 .../AggregateFunctionFactory.cpp              |   2 +-
 .../AggregateFunctionMinMaxAny.h              | 165 +++++++++++++++---
 src/AggregateFunctions/HelpersMinMaxAny.h     |  48 ++++-
 4 files changed, 199 insertions(+), 30 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionAny.cpp b/src/AggregateFunctions/AggregateFunctionAny.cpp
index 9bc6e6af14f..219d530c58d 100644
--- a/src/AggregateFunctions/AggregateFunctionAny.cpp
+++ b/src/AggregateFunctions/AggregateFunctionAny.cpp
@@ -14,11 +14,21 @@ AggregateFunctionPtr createAggregateFunctionAny(const std::string & name, const
     return AggregateFunctionPtr(createAggregateFunctionSingleValue<AggregateFunctionsSingleValue, AggregateFunctionAnyData>(name, argument_types, parameters, settings));
 }
 
+AggregateFunctionPtr createAggregateFunctionNullableAny(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
+{
+    return AggregateFunctionPtr(createAggregateFunctionSingleNullableValue<AggregateFunctionsSingleValue, AggregateFunctionAnyData>(name, argument_types, parameters, settings));
+}
+
 AggregateFunctionPtr createAggregateFunctionAnyLast(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
 {
     return AggregateFunctionPtr(createAggregateFunctionSingleValue<AggregateFunctionsSingleValue, AggregateFunctionAnyLastData>(name, argument_types, parameters, settings));
 }
 
+AggregateFunctionPtr createAggregateFunctionNullableAnyLast(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
+{
+    return AggregateFunctionPtr(createAggregateFunctionSingleNullableValue<AggregateFunctionsSingleValue, AggregateFunctionAnyLastData>(name, argument_types, parameters, settings));
+}
+
 AggregateFunctionPtr createAggregateFunctionAnyHeavy(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
 {
     return AggregateFunctionPtr(createAggregateFunctionSingleValue<AggregateFunctionsSingleValue, AggregateFunctionAnyHeavyData>(name, argument_types, parameters, settings));
@@ -36,10 +46,10 @@ void registerAggregateFunctionsAny(AggregateFunctionFactory & factory)
 
     // Synonyms for use as window functions.
     factory.registerFunction("first_value",
-        { createAggregateFunctionAny, properties },
+        { createAggregateFunctionNullableAny, properties },
         AggregateFunctionFactory::CaseInsensitive);
     factory.registerFunction("last_value",
-        { createAggregateFunctionAnyLast, properties },
+        { createAggregateFunctionNullableAnyLast, properties },
         AggregateFunctionFactory::CaseInsensitive);
 }
 
diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp
index 6cacf66500f..dc8b410dd9a 100644
--- a/src/AggregateFunctions/AggregateFunctionFactory.cpp
+++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp
@@ -106,7 +106,7 @@ AggregateFunctionPtr AggregateFunctionFactory::get(
         // nullability themselves. Another special case is functions from Nothing
         // that are rewritten to AggregateFunctionNothing, in this case
         // nested_function is nullptr.
-        if (!nested_function || !nested_function->isOnlyWindowFunction())
+        if (!nested_function->getResultType()->isNullable() && (!nested_function || !nested_function->isOnlyWindowFunction()))
             return combinator->transformAggregateFunction(nested_function, out_properties, types_without_low_cardinality, parameters);
     }
 
diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
index b984772c8ea..76cdd5690a5 100644
--- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
+++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
@@ -766,19 +766,24 @@ static_assert(
 
 
 /// For any other value types.
+template <bool IS_NULLABLE = false, bool IS_NULL_GREATER = false>
 struct SingleValueDataGeneric
 {
 private:
     using Self = SingleValueDataGeneric;
 
     Field value;
+    bool has_value = false;
 
 public:
-    static constexpr bool is_nullable = false;
+    static constexpr bool is_nullable = IS_NULLABLE;
     static constexpr bool is_any = false;
+    static constexpr bool is_null_greater = IS_NULL_GREATER;
 
     bool has() const
     {
+        if constexpr (is_nullable)
+            return has_value;
         return !value.isNull();
     }
 
@@ -813,11 +818,15 @@ public:
     void change(const IColumn & column, size_t row_num, Arena *)
     {
         column.get(row_num, value);
+        if constexpr (is_nullable)
+            has_value = true;
     }
 
     void change(const Self & to, Arena *)
     {
         value = to.value;
+        if constexpr (is_nullable)
+            has_value = true;
     }
 
     bool changeFirstTime(const IColumn & column, size_t row_num, Arena * arena)
@@ -833,7 +842,7 @@ public:
 
     bool changeFirstTime(const Self & to, Arena * arena)
     {
-        if (!has() && to.has())
+        if (!has() && (is_nullable || to.has()))
         {
             change(to, arena);
             return true;
@@ -868,27 +877,86 @@ public:
         }
         else
         {
-            Field new_value;
-            column.get(row_num, new_value);
-            if (new_value < value)
+            if constexpr (is_nullable)
             {
-                value = new_value;
-                return true;
+                Field new_value;
+                column.get(row_num, new_value);
+                if constexpr (!is_null_greater)
+                {
+                    if (!value.isNull() && (new_value.isNull() || new_value < value))
+                    {
+                        value = new_value;
+                        return true;
+                    }
+                    else
+                        return false;
+                }
+                else
+                {
+                    if ((value.isNull() && !new_value.isNull()) || (!new_value.isNull() && new_value < value))
+                    {
+                        value = new_value;
+                        return true;
+                    }
+
+                    return false;
+                }
             }
             else
-                return false;
+            {
+                Field new_value;
+                column.get(row_num, new_value);
+                if (new_value < value)
+                {
+                    value = new_value;
+                    return true;
+                }
+                else
+                    return false;
+            }
         }
     }
 
     bool changeIfLess(const Self & to, Arena * arena)
     {
-        if (to.has() && (!has() || to.value < value))
+        if (!to.has())
+            return false;
+        if constexpr (is_nullable)
         {
-            change(to, arena);
-            return true;
+            if (!has())
+            {
+                change(to, arena);
+                return true;
+            }
+            if constexpr (!is_null_greater)
+            {
+                if (to.value.isNull() || (!value.isNull() && to.value < value))
+                {
+                    value = to.value;
+                    return true;
+                }
+                return false;
+            }
+            else
+            {
+                if ((value.isNull() && !to.value.isNull()) || (!to.value.isNull() || to.value < value))
+                {
+                    value = to.value;
+                    return true;
+                }
+                return false;
+            }
         }
         else
-            return false;
+        {
+            if (!has() || to.value < value)
+            {
+                change(to, arena);
+                return true;
+            }
+            else
+                return false;
+        }
     }
 
     bool changeIfGreater(const IColumn & column, size_t row_num, Arena * arena)
@@ -900,27 +968,80 @@ public:
         }
         else
         {
-            Field new_value;
-            column.get(row_num, new_value);
-            if (new_value > value)
+            if constexpr (is_nullable)
             {
-                value = new_value;
-                return true;
+                Field new_value;
+                column.get(row_num, new_value);
+                if constexpr (is_null_greater)
+                {
+                    if (!value.isNull() && (new_value.isNull() || value < new_value))
+                    {
+                        value = new_value;
+                        return true;
+                    }
+                    return false;
+                }
+                else
+                {
+                    if ((value.isNull() && !new_value.isNull()) || (!new_value.isNull() && value < new_value))
+                    {
+                        value = new_value;
+                        return true;
+                    }
+                    return false;
+                }
             }
             else
-                return false;
+            {
+                Field new_value;
+                column.get(row_num, new_value);
+                if (new_value > value)
+                {
+                    value = new_value;
+                    return true;
+                }
+                else
+                    return false;
+            }
         }
     }
 
     bool changeIfGreater(const Self & to, Arena * arena)
     {
-        if (to.has() && (!has() || to.value > value))
+        if (!to.has())
+            return false;
+        if constexpr (is_nullable)
         {
-            change(to, arena);
-            return true;
+            if constexpr (is_null_greater)
+            {
+                if (!value.isNull() && (to.value.isNull() || value < to.value))
+                {
+                    value = to.value;
+                    return true;
+                }
+                return false;
+            }
+            else
+            {
+                if ((value.isNull() && !to.value.isNull()) || (!to.value.isNull() && value < to.value))
+                {
+                    value = to.value;
+                    return true;
+                }
+                return false;
+
+            }
         }
         else
-            return false;
+        {
+            if (!has() || to.value > value)
+            {
+                change(to, arena);
+                return true;
+            }
+            else
+                return false;
+        }
     }
 
     bool isEqualTo(const IColumn & column, size_t row_num) const
diff --git a/src/AggregateFunctions/HelpersMinMaxAny.h b/src/AggregateFunctions/HelpersMinMaxAny.h
index 026a206b109..f8fac616d2c 100644
--- a/src/AggregateFunctions/HelpersMinMaxAny.h
+++ b/src/AggregateFunctions/HelpersMinMaxAny.h
@@ -9,9 +9,12 @@
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeString.h>
 
-
 namespace DB
 {
+namespace ErrorCodes
+{
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+}
 struct Settings;
 
 /// min, max, any, anyLast, anyHeavy, etc...
@@ -22,7 +25,6 @@ static IAggregateFunction * createAggregateFunctionSingleValue(const String & na
     assertUnary(name, argument_types);
 
     const DataTypePtr & argument_type = argument_types[0];
-
     WhichDataType which(argument_type);
 #define DISPATCH(TYPE) \
     if (which.idx == TypeIndex::TYPE) return new AggregateFunctionTemplate<Data<SingleValueDataFixed<TYPE>>>(argument_type); /// NOLINT
@@ -46,7 +48,43 @@ static IAggregateFunction * createAggregateFunctionSingleValue(const String & na
     if (which.idx == TypeIndex::String)
         return new AggregateFunctionTemplate<Data<SingleValueDataString>>(argument_type);
 
-    return new AggregateFunctionTemplate<Data<SingleValueDataGeneric>>(argument_type);
+    return new AggregateFunctionTemplate<Data<SingleValueDataGeneric<>>>(argument_type);
+}
+
+template <template <typename> class AggregateFunctionTemplate, template <typename> class Data>
+static IAggregateFunction * createAggregateFunctionSingleNullableValue(const String & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
+{
+    assertUnary(name, argument_types);
+    bool is_nullable = false;
+    bool is_null_greater = false;
+    if (parameters.size() > 0)
+    {
+        is_nullable = parameters[0].get<bool>();
+    }
+    if (parameters.size() > 1)
+    {
+       is_null_greater = parameters[1].get<bool>();
+    }
+    if (parameters.size() > 2)
+        throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "name's parameter size must not be larger  then 2");
+
+    const DataTypePtr & argument_type = argument_types[0];
+    WhichDataType which(argument_type);
+    // if the resule value could be null(excluding the case that no row is matched),
+    // use SingleValueDataGeneric.
+    if (!is_nullable)
+    {
+        return createAggregateFunctionSingleValue<AggregateFunctionTemplate, Data>(name, argument_types, Array(), settings);
+    }
+    else if (is_nullable && is_null_greater)
+    {
+        return new AggregateFunctionTemplate<Data<SingleValueDataGeneric<true, true>>>(argument_type);
+    }
+    else if (is_nullable && !is_null_greater)
+    {
+        return new AggregateFunctionTemplate<Data<SingleValueDataGeneric<true, false>>>(argument_type);
+    }
+    UNREACHABLE();
 }
 
 
@@ -79,7 +117,7 @@ static IAggregateFunction * createAggregateFunctionArgMinMaxSecond(const DataTyp
     if (which.idx == TypeIndex::String)
         return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataString>>>(res_type, val_type);
 
-    return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataGeneric>>>(res_type, val_type);
+    return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataGeneric<>>>>(res_type, val_type);
 }
 
 template <template <typename> class MinMaxData>
@@ -115,7 +153,7 @@ static IAggregateFunction * createAggregateFunctionArgMinMax(const String & name
     if (which.idx == TypeIndex::String)
         return createAggregateFunctionArgMinMaxSecond<MinMaxData, SingleValueDataString>(res_type, val_type);
 
-    return createAggregateFunctionArgMinMaxSecond<MinMaxData, SingleValueDataGeneric>(res_type, val_type);
+    return createAggregateFunctionArgMinMaxSecond<MinMaxData, SingleValueDataGeneric<false>>(res_type, val_type);
 }
 
 }

From cc0480c03fae447f63a7457bd6300ef2c77aa133 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Thu, 16 Feb 2023 16:58:46 +0800
Subject: [PATCH 064/689] update docs and add tests

---
 .../reference/first_value.md                  | 52 ++++++++++++++++++
 .../aggregate-functions/reference/index.md    |  2 +
 .../reference/last_value.md                   | 53 +++++++++++++++++++
 src/AggregateFunctions/HelpersMinMaxAny.h     |  2 +-
 .../02662_first_last_value.reference          | 20 +++++++
 .../0_stateless/02662_first_last_value.sql    | 16 ++++++
 6 files changed, 144 insertions(+), 1 deletion(-)
 create mode 100644 docs/en/sql-reference/aggregate-functions/reference/first_value.md
 create mode 100644 docs/en/sql-reference/aggregate-functions/reference/last_value.md
 create mode 100644 tests/queries/0_stateless/02662_first_last_value.reference
 create mode 100644 tests/queries/0_stateless/02662_first_last_value.sql

diff --git a/docs/en/sql-reference/aggregate-functions/reference/first_value.md b/docs/en/sql-reference/aggregate-functions/reference/first_value.md
new file mode 100644
index 00000000000..920aa65baf9
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/first_value.md
@@ -0,0 +1,52 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/first_value
+sidebar_position: 7
+---
+
+# first_value
+
+Selects the first encountered value, similar to `any`, but could accept NULL.
+
+## examples
+
+```sq;
+insert into test_data (a,b) values (1,null), (2,3), (4, 5), (6.null)
+```
+
+### example1
+The NULL value is ignored at default.
+```sql
+select first_value(b) from test_data
+```
+
+```text
+┌─first_value(b)─┐
+│              3 │
+└────────────────┘
+```
+
+### example2
+The NULL value is ignored.
+```sql
+select first_value(false)(b) from test_data
+```
+
+```text
+┌─first_value(false)(b)─┐
+│                     3 │
+└───────────────────────┘
+```
+
+### example3
+The NULL value is accepted.
+```sql
+select first_value(true)(b) from test_data
+```
+
+```text
+┌─first_value(true)(b)─┐
+│                 ᴺᵁᴸᴸ │
+└──────────────────────┘
+```
+
+
diff --git a/docs/en/sql-reference/aggregate-functions/reference/index.md b/docs/en/sql-reference/aggregate-functions/reference/index.md
index e25e3a54356..50208352f38 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/index.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/index.md
@@ -26,6 +26,8 @@ ClickHouse-specific aggregate functions:
 
 - [anyHeavy](../../../sql-reference/aggregate-functions/reference/anyheavy.md)
 - [anyLast](../../../sql-reference/aggregate-functions/reference/anylast.md)
+- [first_value](../../../sql-reference/aggregate-functions/reference/first_value.md)
+- [last_value](../../../sql-reference/aggregate-functions/reference/last_value.md)
 - [argMin](../../../sql-reference/aggregate-functions/reference/argmin.md)
 - [argMax](../../../sql-reference/aggregate-functions/reference/argmax.md)
 - [avgWeighted](../../../sql-reference/aggregate-functions/reference/avgweighted.md)
diff --git a/docs/en/sql-reference/aggregate-functions/reference/last_value.md b/docs/en/sql-reference/aggregate-functions/reference/last_value.md
new file mode 100644
index 00000000000..5fc631fabb3
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/last_value.md
@@ -0,0 +1,53 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/last_value
+sidebar_position: 8
+---
+
+# first_value
+
+Selects the last encountered value, similar to `anyLast`, but could accept NULL.
+
+
+## examples
+
+```sq;
+insert into test_data (a,b) values (1,null), (2,3), (4, 5), (6.null)
+```
+
+### example1
+The NULL value is ignored at default.
+```sql
+select last_value(b) from test_data
+```
+
+```text
+┌─last_value(b)─┐
+│             5 │
+└───────────────┘
+```
+
+### example2
+The NULL value is ignored.
+```sql
+select last_value(false)(b) from test_data
+```
+
+```text
+┌─last_value(false)(b)─┐
+│                    5 │
+└──────────────────────┘
+```
+
+### example3
+The NULL value is accepted.
+```sql
+select last_value(true)(b) from test_data
+```
+
+```text
+┌─last_value(true)(b)─┐
+│                ᴺᵁᴸᴸ │
+└─────────────────────┘
+```
+
+
diff --git a/src/AggregateFunctions/HelpersMinMaxAny.h b/src/AggregateFunctions/HelpersMinMaxAny.h
index f8fac616d2c..ac4cfaaeea1 100644
--- a/src/AggregateFunctions/HelpersMinMaxAny.h
+++ b/src/AggregateFunctions/HelpersMinMaxAny.h
@@ -153,7 +153,7 @@ static IAggregateFunction * createAggregateFunctionArgMinMax(const String & name
     if (which.idx == TypeIndex::String)
         return createAggregateFunctionArgMinMaxSecond<MinMaxData, SingleValueDataString>(res_type, val_type);
 
-    return createAggregateFunctionArgMinMaxSecond<MinMaxData, SingleValueDataGeneric<false>>(res_type, val_type);
+    return createAggregateFunctionArgMinMaxSecond<MinMaxData, SingleValueDataGeneric<>>(res_type, val_type);
 }
 
 }
diff --git a/tests/queries/0_stateless/02662_first_last_value.reference b/tests/queries/0_stateless/02662_first_last_value.reference
new file mode 100644
index 00000000000..14f9aede4c0
--- /dev/null
+++ b/tests/queries/0_stateless/02662_first_last_value.reference
@@ -0,0 +1,20 @@
+-- { echo }
+
+-- create table
+drop table if exists test;
+create table test(`a` Nullable(Int32), `b` Nullable(Int32)) ENGINE = Memory;
+insert into test (a,b) values (1,null), (2,3), (4, 5), (6,null);
+-- first value
+select first_value(b) from test;
+3
+select first_value(false)(b) from test;
+3
+select first_value(true)(b) from test;
+\N
+-- last value
+select last_value(b) from test;
+5
+select last_value(false)(b) from test;
+5
+select last_value(true)(b) from test;
+\N
diff --git a/tests/queries/0_stateless/02662_first_last_value.sql b/tests/queries/0_stateless/02662_first_last_value.sql
new file mode 100644
index 00000000000..2e11808312a
--- /dev/null
+++ b/tests/queries/0_stateless/02662_first_last_value.sql
@@ -0,0 +1,16 @@
+-- { echo }
+
+-- create table
+drop table if exists test;
+create table test(`a` Nullable(Int32), `b` Nullable(Int32)) ENGINE = Memory;
+insert into test (a,b) values (1,null), (2,3), (4, 5), (6,null);
+
+-- first value
+select first_value(b) from test;
+select first_value(false)(b) from test;
+select first_value(true)(b) from test;
+
+-- last value
+select last_value(b) from test;
+select last_value(false)(b) from test;
+select last_value(true)(b) from test;

From bd84b7a87a35862624953eb55742c4c24b6d8d17 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Tue, 21 Feb 2023 10:41:58 +0800
Subject: [PATCH 065/689] support syntax, respect nulls/ignore nulls

---
 .../reference/first_value.md                  | 26 +++++++-------
 .../reference/last_value.md                   | 24 ++++++-------
 .../AggregateFunctionAny.cpp                  | 28 ++++++++++++---
 .../AggregateFunctionFactory.cpp              |  2 +-
 .../AggregateFunctionMinMaxAny.h              | 11 ++++++
 src/AggregateFunctions/HelpersMinMaxAny.h     | 19 +++--------
 src/Parsers/ExpressionListParsers.cpp         | 34 ++++++++++++++++++-
 .../02662_first_last_value.reference          |  8 ++---
 .../0_stateless/02662_first_last_value.sql    |  8 ++---
 9 files changed, 108 insertions(+), 52 deletions(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/first_value.md b/docs/en/sql-reference/aggregate-functions/reference/first_value.md
index 920aa65baf9..73847cabe2e 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/first_value.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/first_value.md
@@ -10,7 +10,7 @@ Selects the first encountered value, similar to `any`, but could accept NULL.
 ## examples
 
 ```sq;
-insert into test_data (a,b) values (1,null), (2,3), (4, 5), (6.null)
+insert into test_data (a,b) values (1,null), (2,3), (4, 5), (6,null)
 ```
 
 ### example1
@@ -20,33 +20,35 @@ select first_value(b) from test_data
 ```
 
 ```text
-┌─first_value(b)─┐
-│              3 │
-└────────────────┘
+┌─first_value_ignore_nulls(b)─┐
+│                           3 │
+└─────────────────────────────┘
+
 ```
 
 ### example2
 The NULL value is ignored.
 ```sql
-select first_value(false)(b) from test_data
+select first_value(b) ignore nulls sfrom test_data
 ```
 
 ```text
-┌─first_value(false)(b)─┐
-│                     3 │
-└───────────────────────┘
+┌─first_value_ignore_nulls(b)─┐
+│                           3 │
+└─────────────────────────────┘
+
 ```
 
 ### example3
 The NULL value is accepted.
 ```sql
-select first_value(true)(b) from test_data
+select first_value(b) respect nulls from test_data
 ```
 
 ```text
-┌─first_value(true)(b)─┐
-│                 ᴺᵁᴸᴸ │
-└──────────────────────┘
+┌─first_value_respect_nulls(b)─┐
+│                         ᴺᵁᴸᴸ │
+└──────────────────────────────┘
 ```
 
 
diff --git a/docs/en/sql-reference/aggregate-functions/reference/last_value.md b/docs/en/sql-reference/aggregate-functions/reference/last_value.md
index 5fc631fabb3..05894c333f6 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/last_value.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/last_value.md
@@ -11,7 +11,7 @@ Selects the last encountered value, similar to `anyLast`, but could accept NULL.
 ## examples
 
 ```sq;
-insert into test_data (a,b) values (1,null), (2,3), (4, 5), (6.null)
+insert into test_data (a,b) values (1,null), (2,3), (4, 5), (6,null)
 ```
 
 ### example1
@@ -21,33 +21,33 @@ select last_value(b) from test_data
 ```
 
 ```text
-┌─last_value(b)─┐
-│             5 │
-└───────────────┘
+┌─last_value_ignore_nulls(b)─┐
+│                          5 │
+└────────────────────────────┘
 ```
 
 ### example2
 The NULL value is ignored.
 ```sql
-select last_value(false)(b) from test_data
+select last_value(b) ignore nulls from test_data
 ```
 
 ```text
-┌─last_value(false)(b)─┐
-│                    5 │
-└──────────────────────┘
+┌─last_value_ignore_nulls(b)─┐
+│                          5 │
+└────────────────────────────┘
 ```
 
 ### example3
 The NULL value is accepted.
 ```sql
-select last_value(true)(b) from test_data
+select last_value(b) respect nulls from test_data
 ```
 
 ```text
-┌─last_value(true)(b)─┐
-│                ᴺᵁᴸᴸ │
-└─────────────────────┘
+┌─last_value_respect_nulls(b)─┐
+│                        ᴺᵁᴸᴸ │
+└─────────────────────────────┘
 ```
 
 
diff --git a/src/AggregateFunctions/AggregateFunctionAny.cpp b/src/AggregateFunctions/AggregateFunctionAny.cpp
index 219d530c58d..460ae3262b8 100644
--- a/src/AggregateFunctions/AggregateFunctionAny.cpp
+++ b/src/AggregateFunctions/AggregateFunctionAny.cpp
@@ -14,9 +14,12 @@ AggregateFunctionPtr createAggregateFunctionAny(const std::string & name, const
     return AggregateFunctionPtr(createAggregateFunctionSingleValue<AggregateFunctionsSingleValue, AggregateFunctionAnyData>(name, argument_types, parameters, settings));
 }
 
+template <bool RespectNulls = false, bool NullIsGreater = false>
 AggregateFunctionPtr createAggregateFunctionNullableAny(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
 {
-    return AggregateFunctionPtr(createAggregateFunctionSingleNullableValue<AggregateFunctionsSingleValue, AggregateFunctionAnyData>(name, argument_types, parameters, settings));
+    return AggregateFunctionPtr(
+        createAggregateFunctionSingleNullableValue<AggregateFunctionsSingleValue, AggregateFunctionAnyData, RespectNulls, NullIsGreater>(
+            name, argument_types, parameters, settings));
 }
 
 AggregateFunctionPtr createAggregateFunctionAnyLast(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
@@ -24,9 +27,14 @@ AggregateFunctionPtr createAggregateFunctionAnyLast(const std::string & name, co
     return AggregateFunctionPtr(createAggregateFunctionSingleValue<AggregateFunctionsSingleValue, AggregateFunctionAnyLastData>(name, argument_types, parameters, settings));
 }
 
+template <bool RespectNulls = false, bool NullIsGreater = false>
 AggregateFunctionPtr createAggregateFunctionNullableAnyLast(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
 {
-    return AggregateFunctionPtr(createAggregateFunctionSingleNullableValue<AggregateFunctionsSingleValue, AggregateFunctionAnyLastData>(name, argument_types, parameters, settings));
+    return AggregateFunctionPtr(createAggregateFunctionSingleNullableValue<
+                                AggregateFunctionsSingleValue,
+                                AggregateFunctionAnyLastData,
+                                RespectNulls,
+                                NullIsGreater>(name, argument_types, parameters, settings));
 }
 
 AggregateFunctionPtr createAggregateFunctionAnyHeavy(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
@@ -46,10 +54,22 @@ void registerAggregateFunctionsAny(AggregateFunctionFactory & factory)
 
     // Synonyms for use as window functions.
     factory.registerFunction("first_value",
-        { createAggregateFunctionNullableAny, properties },
+        { createAggregateFunctionAny, properties },
+        AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("first_value_respect_nulls",
+        { createAggregateFunctionNullableAny<true>, properties },
+        AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("first_value_ignore_nulls",
+        { createAggregateFunctionNullableAny<false>, properties },
         AggregateFunctionFactory::CaseInsensitive);
     factory.registerFunction("last_value",
-        { createAggregateFunctionNullableAnyLast, properties },
+        { createAggregateFunctionAnyLast, properties },
+        AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("last_value_respect_nulls",
+        { createAggregateFunctionNullableAnyLast<true>, properties },
+        AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("last_value_ignore_nulls",
+        { createAggregateFunctionNullableAnyLast<false>, properties },
         AggregateFunctionFactory::CaseInsensitive);
 }
 
diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp
index dc8b410dd9a..9c60160f3ee 100644
--- a/src/AggregateFunctions/AggregateFunctionFactory.cpp
+++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp
@@ -106,7 +106,7 @@ AggregateFunctionPtr AggregateFunctionFactory::get(
         // nullability themselves. Another special case is functions from Nothing
         // that are rewritten to AggregateFunctionNothing, in this case
         // nested_function is nullptr.
-        if (!nested_function->getResultType()->isNullable() && (!nested_function || !nested_function->isOnlyWindowFunction()))
+        if ((!nested_function || !nested_function->isOnlyWindowFunction()))
             return combinator->transformAggregateFunction(nested_function, out_properties, types_without_low_cardinality, parameters);
     }
 
diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
index 76cdd5690a5..dc6a822d85c 100644
--- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
+++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
@@ -1478,6 +1478,17 @@ public:
         this->data(place).insertResultInto(to);
     }
 
+    AggregateFunctionPtr getOwnNullAdapter(
+        const AggregateFunctionPtr & nested_function,
+        const DataTypes & /*arguments*/,
+        const Array & /*params*/,
+        const AggregateFunctionProperties & /*properties*/) const override
+    {
+        if (Data::is_nullable)
+            return nested_function;
+        return nullptr;
+    }
+
 #if USE_EMBEDDED_COMPILER
 
     bool isCompilable() const override
diff --git a/src/AggregateFunctions/HelpersMinMaxAny.h b/src/AggregateFunctions/HelpersMinMaxAny.h
index ac4cfaaeea1..ea67e2f072e 100644
--- a/src/AggregateFunctions/HelpersMinMaxAny.h
+++ b/src/AggregateFunctions/HelpersMinMaxAny.h
@@ -51,20 +51,11 @@ static IAggregateFunction * createAggregateFunctionSingleValue(const String & na
     return new AggregateFunctionTemplate<Data<SingleValueDataGeneric<>>>(argument_type);
 }
 
-template <template <typename> class AggregateFunctionTemplate, template <typename> class Data>
+template <template <typename> class AggregateFunctionTemplate, template <typename> class Data, bool RespectNulls = false, bool NullIsGreater = false>
 static IAggregateFunction * createAggregateFunctionSingleNullableValue(const String & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
 {
+    assertNoParameters(name, parameters);
     assertUnary(name, argument_types);
-    bool is_nullable = false;
-    bool is_null_greater = false;
-    if (parameters.size() > 0)
-    {
-        is_nullable = parameters[0].get<bool>();
-    }
-    if (parameters.size() > 1)
-    {
-       is_null_greater = parameters[1].get<bool>();
-    }
     if (parameters.size() > 2)
         throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "name's parameter size must not be larger  then 2");
 
@@ -72,15 +63,15 @@ static IAggregateFunction * createAggregateFunctionSingleNullableValue(const Str
     WhichDataType which(argument_type);
     // if the resule value could be null(excluding the case that no row is matched),
     // use SingleValueDataGeneric.
-    if (!is_nullable)
+    if constexpr (!RespectNulls)
     {
         return createAggregateFunctionSingleValue<AggregateFunctionTemplate, Data>(name, argument_types, Array(), settings);
     }
-    else if (is_nullable && is_null_greater)
+    else if constexpr (NullIsGreater)
     {
         return new AggregateFunctionTemplate<Data<SingleValueDataGeneric<true, true>>>(argument_type);
     }
-    else if (is_nullable && !is_null_greater)
+    else
     {
         return new AggregateFunctionTemplate<Data<SingleValueDataGeneric<true, false>>>(argument_type);
     }
diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp
index 18ed2ed223a..7fec7f4c5db 100644
--- a/src/Parsers/ExpressionListParsers.cpp
+++ b/src/Parsers/ExpressionListParsers.cpp
@@ -1,4 +1,5 @@
 #include <string_view>
+#include <unordered_map>
 
 #include <Parsers/ExpressionListParsers.h>
 #include <Parsers/ParserSetQuery.h>
@@ -25,6 +26,7 @@
 
 #include <Common/logger_useful.h>
 #include <Parsers/queryToString.h>
+#include <Parsers/CommonParsers.h>
 
 using namespace std::literals;
 
@@ -1085,6 +1087,8 @@ public:
 
             ParserKeyword filter("FILTER");
             ParserKeyword over("OVER");
+            ParserKeyword respect_nulls("RESPECT NULLS");
+            ParserKeyword ignore_nulls("IGNORE NULLS");
 
             if (filter.ignore(pos, expected))
             {
@@ -1100,6 +1104,17 @@ public:
                     return false;
             }
 
+            NullsAction nulls_action = NullsAction::DEFAULT;
+            if (respect_nulls.ignore(pos, expected))
+            {
+                nulls_action = NullsAction::RESPECT_NULLS;
+            }
+            if (ignore_nulls.ignore(pos, expected))
+            {
+                nulls_action = NullsAction::IGNORE_NULLS;
+            }
+            function_node->name = transformFunctionNameForRepectNulls(function_node->name, nulls_action);
+
             if (over.ignore(pos, expected))
             {
                 function_node->is_window_function = true;
@@ -1132,6 +1147,24 @@ private:
 
     bool allow_function_parameters;
     bool is_compound_name;
+
+    enum NullsAction
+    {
+        DEFAULT = 0,
+        RESPECT_NULLS = 1,
+        IGNORE_NULLS = 2,
+    };
+    static String transformFunctionNameForRepectNulls(const String & original_function_name, NullsAction nulls_action)
+    {
+        static std::unordered_map<String, std::vector<String>> renamed_functions_with_nulls = {
+            {"first_value", {"first_value_ignore_nulls", "first_value_respect_nulls", "first_value_ignore_nulls"}},
+            {"last_value", {"last_value_ignore_nulls", "last_value_respect_nulls", "last_value_ignore_nulls"}},
+        };
+        auto it = renamed_functions_with_nulls.find(original_function_name);
+        if (it == renamed_functions_with_nulls.end())
+            return original_function_name;
+        return it->second[nulls_action];
+    }
 };
 
 /// Layer for priority brackets and tuple function
@@ -2353,7 +2386,6 @@ bool ParserExpressionImpl::parse(std::unique_ptr<Layer> start, IParser::Pos & po
         {
             if (!layers.back()->parse(pos, expected, next))
                 break;
-
             if (layers.back()->isFinished())
             {
                 if (layers.size() == 1)
diff --git a/tests/queries/0_stateless/02662_first_last_value.reference b/tests/queries/0_stateless/02662_first_last_value.reference
index 14f9aede4c0..46ee9e0ed1e 100644
--- a/tests/queries/0_stateless/02662_first_last_value.reference
+++ b/tests/queries/0_stateless/02662_first_last_value.reference
@@ -7,14 +7,14 @@ insert into test (a,b) values (1,null), (2,3), (4, 5), (6,null);
 -- first value
 select first_value(b) from test;
 3
-select first_value(false)(b) from test;
+select first_value(b) ignore nulls from test;
 3
-select first_value(true)(b) from test;
+select first_value(b) respect nulls from test;
 \N
 -- last value
 select last_value(b) from test;
 5
-select last_value(false)(b) from test;
+select last_value(b) ignore nulls from test;
 5
-select last_value(true)(b) from test;
+select last_value(true)(b) respect nulls from test;
 \N
diff --git a/tests/queries/0_stateless/02662_first_last_value.sql b/tests/queries/0_stateless/02662_first_last_value.sql
index 2e11808312a..42fbd1a11ce 100644
--- a/tests/queries/0_stateless/02662_first_last_value.sql
+++ b/tests/queries/0_stateless/02662_first_last_value.sql
@@ -7,10 +7,10 @@ insert into test (a,b) values (1,null), (2,3), (4, 5), (6,null);
 
 -- first value
 select first_value(b) from test;
-select first_value(false)(b) from test;
-select first_value(true)(b) from test;
+select first_value(b) ignore nulls from test;
+select first_value(b) respect nulls from test;
 
 -- last value
 select last_value(b) from test;
-select last_value(false)(b) from test;
-select last_value(true)(b) from test;
+select last_value(b) ignore nulls from test;
+select last_value(true)(b) respect nulls from test;

From f6a532d30901e2f07481880cda7e94638c378d7d Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Tue, 21 Feb 2023 13:59:45 +0800
Subject: [PATCH 066/689] update

---
 .../sql-reference/aggregate-functions/reference/first_value.md   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/first_value.md b/docs/en/sql-reference/aggregate-functions/reference/first_value.md
index 73847cabe2e..12dcf179f84 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/first_value.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/first_value.md
@@ -46,6 +46,7 @@ select first_value(b) respect nulls from test_data
 ```
 
 ```text
+
 ┌─first_value_respect_nulls(b)─┐
 │                         ᴺᵁᴸᴸ │
 └──────────────────────────────┘

From 8e16db706009d2be13a4b930d80b208152368a78 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Tue, 21 Feb 2023 16:06:05 +0800
Subject: [PATCH 067/689] fixed tests

---
 tests/queries/0_stateless/02662_first_last_value.reference | 2 +-
 tests/queries/0_stateless/02662_first_last_value.sql       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02662_first_last_value.reference b/tests/queries/0_stateless/02662_first_last_value.reference
index 46ee9e0ed1e..0c4503d61ac 100644
--- a/tests/queries/0_stateless/02662_first_last_value.reference
+++ b/tests/queries/0_stateless/02662_first_last_value.reference
@@ -16,5 +16,5 @@ select last_value(b) from test;
 5
 select last_value(b) ignore nulls from test;
 5
-select last_value(true)(b) respect nulls from test;
+select last_value(b) respect nulls from test;
 \N
diff --git a/tests/queries/0_stateless/02662_first_last_value.sql b/tests/queries/0_stateless/02662_first_last_value.sql
index 42fbd1a11ce..126e89f5325 100644
--- a/tests/queries/0_stateless/02662_first_last_value.sql
+++ b/tests/queries/0_stateless/02662_first_last_value.sql
@@ -13,4 +13,4 @@ select first_value(b) respect nulls from test;
 -- last value
 select last_value(b) from test;
 select last_value(b) ignore nulls from test;
-select last_value(true)(b) respect nulls from test;
+select last_value(b) respect nulls from test;

From 10daacb18f51974f89739067ace1d862afc1ff28 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Wed, 19 Apr 2023 19:22:36 +0800
Subject: [PATCH 068/689] updated

---
 .../reference/first_value.md                  |  2 +-
 .../reference/last_value.md                   |  2 +-
 .../AggregateFunctionAny.cpp                  | 18 ++--
 .../AggregateFunctionFactory.cpp              |  2 +-
 .../AggregateFunctionMinMaxAny.h              | 83 ++++---------------
 src/AggregateFunctions/HelpersMinMaxAny.h     | 16 +---
 src/Parsers/ExpressionListParsers.cpp         | 16 ++--
 7 files changed, 39 insertions(+), 100 deletions(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/first_value.md b/docs/en/sql-reference/aggregate-functions/reference/first_value.md
index 12dcf179f84..e163bd62a45 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/first_value.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/first_value.md
@@ -9,7 +9,7 @@ Selects the first encountered value, similar to `any`, but could accept NULL.
 
 ## examples
 
-```sq;
+```sql
 insert into test_data (a,b) values (1,null), (2,3), (4, 5), (6,null)
 ```
 
diff --git a/docs/en/sql-reference/aggregate-functions/reference/last_value.md b/docs/en/sql-reference/aggregate-functions/reference/last_value.md
index 05894c333f6..ebf002e6ae2 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/last_value.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/last_value.md
@@ -10,7 +10,7 @@ Selects the last encountered value, similar to `anyLast`, but could accept NULL.
 
 ## examples
 
-```sq;
+```sql
 insert into test_data (a,b) values (1,null), (2,3), (4, 5), (6,null)
 ```
 
diff --git a/src/AggregateFunctions/AggregateFunctionAny.cpp b/src/AggregateFunctions/AggregateFunctionAny.cpp
index 460ae3262b8..7f57062126b 100644
--- a/src/AggregateFunctions/AggregateFunctionAny.cpp
+++ b/src/AggregateFunctions/AggregateFunctionAny.cpp
@@ -14,11 +14,12 @@ AggregateFunctionPtr createAggregateFunctionAny(const std::string & name, const
     return AggregateFunctionPtr(createAggregateFunctionSingleValue<AggregateFunctionsSingleValue, AggregateFunctionAnyData>(name, argument_types, parameters, settings));
 }
 
-template <bool RespectNulls = false, bool NullIsGreater = false>
-AggregateFunctionPtr createAggregateFunctionNullableAny(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
+template <bool RespectNulls = false>
+AggregateFunctionPtr createAggregateFunctionNullableAny(
+    const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
 {
     return AggregateFunctionPtr(
-        createAggregateFunctionSingleNullableValue<AggregateFunctionsSingleValue, AggregateFunctionAnyData, RespectNulls, NullIsGreater>(
+        createAggregateFunctionSingleNullableValue<AggregateFunctionsSingleValue, AggregateFunctionAnyData, RespectNulls>(
             name, argument_types, parameters, settings));
 }
 
@@ -27,14 +28,13 @@ AggregateFunctionPtr createAggregateFunctionAnyLast(const std::string & name, co
     return AggregateFunctionPtr(createAggregateFunctionSingleValue<AggregateFunctionsSingleValue, AggregateFunctionAnyLastData>(name, argument_types, parameters, settings));
 }
 
-template <bool RespectNulls = false, bool NullIsGreater = false>
+template <bool RespectNulls = false>
 AggregateFunctionPtr createAggregateFunctionNullableAnyLast(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
 {
     return AggregateFunctionPtr(createAggregateFunctionSingleNullableValue<
                                 AggregateFunctionsSingleValue,
                                 AggregateFunctionAnyLastData,
-                                RespectNulls,
-                                NullIsGreater>(name, argument_types, parameters, settings));
+                                RespectNulls>(name, argument_types, parameters, settings));
 }
 
 AggregateFunctionPtr createAggregateFunctionAnyHeavy(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
@@ -59,18 +59,12 @@ void registerAggregateFunctionsAny(AggregateFunctionFactory & factory)
     factory.registerFunction("first_value_respect_nulls",
         { createAggregateFunctionNullableAny<true>, properties },
         AggregateFunctionFactory::CaseInsensitive);
-    factory.registerFunction("first_value_ignore_nulls",
-        { createAggregateFunctionNullableAny<false>, properties },
-        AggregateFunctionFactory::CaseInsensitive);
     factory.registerFunction("last_value",
         { createAggregateFunctionAnyLast, properties },
         AggregateFunctionFactory::CaseInsensitive);
     factory.registerFunction("last_value_respect_nulls",
         { createAggregateFunctionNullableAnyLast<true>, properties },
         AggregateFunctionFactory::CaseInsensitive);
-    factory.registerFunction("last_value_ignore_nulls",
-        { createAggregateFunctionNullableAnyLast<false>, properties },
-        AggregateFunctionFactory::CaseInsensitive);
 }
 
 }
diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp
index 9c60160f3ee..6cacf66500f 100644
--- a/src/AggregateFunctions/AggregateFunctionFactory.cpp
+++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp
@@ -106,7 +106,7 @@ AggregateFunctionPtr AggregateFunctionFactory::get(
         // nullability themselves. Another special case is functions from Nothing
         // that are rewritten to AggregateFunctionNothing, in this case
         // nested_function is nullptr.
-        if ((!nested_function || !nested_function->isOnlyWindowFunction()))
+        if (!nested_function || !nested_function->isOnlyWindowFunction())
             return combinator->transformAggregateFunction(nested_function, out_properties, types_without_low_cardinality, parameters);
     }
 
diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
index dc6a822d85c..51eec1689ae 100644
--- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
+++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
@@ -766,7 +766,7 @@ static_assert(
 
 
 /// For any other value types.
-template <bool IS_NULLABLE = false, bool IS_NULL_GREATER = false>
+template <bool IS_NULLABLE = false>
 struct SingleValueDataGeneric
 {
 private:
@@ -778,7 +778,6 @@ private:
 public:
     static constexpr bool is_nullable = IS_NULLABLE;
     static constexpr bool is_any = false;
-    static constexpr bool is_null_greater = IS_NULL_GREATER;
 
     bool has() const
     {
@@ -881,26 +880,13 @@ public:
             {
                 Field new_value;
                 column.get(row_num, new_value);
-                if constexpr (!is_null_greater)
+                if (!value.isNull() && (new_value.isNull() || new_value < value))
                 {
-                    if (!value.isNull() && (new_value.isNull() || new_value < value))
-                    {
-                        value = new_value;
-                        return true;
-                    }
-                    else
-                        return false;
+                    value = new_value;
+                    return true;
                 }
                 else
-                {
-                    if ((value.isNull() && !new_value.isNull()) || (!new_value.isNull() && new_value < value))
-                    {
-                        value = new_value;
-                        return true;
-                    }
-
                     return false;
-                }
             }
             else
             {
@@ -928,24 +914,12 @@ public:
                 change(to, arena);
                 return true;
             }
-            if constexpr (!is_null_greater)
+            if (to.value.isNull() || (!value.isNull() && to.value < value))
             {
-                if (to.value.isNull() || (!value.isNull() && to.value < value))
-                {
-                    value = to.value;
-                    return true;
-                }
-                return false;
-            }
-            else
-            {
-                if ((value.isNull() && !to.value.isNull()) || (!to.value.isNull() || to.value < value))
-                {
-                    value = to.value;
-                    return true;
-                }
-                return false;
+                value = to.value;
+                return true;
             }
+            return false;
         }
         else
         {
@@ -972,24 +946,12 @@ public:
             {
                 Field new_value;
                 column.get(row_num, new_value);
-                if constexpr (is_null_greater)
+                if (!value.isNull() && (new_value.isNull() || value < new_value))
                 {
-                    if (!value.isNull() && (new_value.isNull() || value < new_value))
-                    {
-                        value = new_value;
-                        return true;
-                    }
-                    return false;
-                }
-                else
-                {
-                    if ((value.isNull() && !new_value.isNull()) || (!new_value.isNull() && value < new_value))
-                    {
-                        value = new_value;
-                        return true;
-                    }
-                    return false;
+                    value = new_value;
+                    return true;
                 }
+                return false;
             }
             else
             {
@@ -1012,25 +974,12 @@ public:
             return false;
         if constexpr (is_nullable)
         {
-            if constexpr (is_null_greater)
+            if (!value.isNull() && (to.value.isNull() || value < to.value))
             {
-                if (!value.isNull() && (to.value.isNull() || value < to.value))
-                {
-                    value = to.value;
-                    return true;
-                }
-                return false;
-            }
-            else
-            {
-                if ((value.isNull() && !to.value.isNull()) || (!to.value.isNull() && value < to.value))
-                {
-                    value = to.value;
-                    return true;
-                }
-                return false;
-
+                value = to.value;
+                return true;
             }
+            return false;
         }
         else
         {
diff --git a/src/AggregateFunctions/HelpersMinMaxAny.h b/src/AggregateFunctions/HelpersMinMaxAny.h
index ea67e2f072e..31ae5fdd59a 100644
--- a/src/AggregateFunctions/HelpersMinMaxAny.h
+++ b/src/AggregateFunctions/HelpersMinMaxAny.h
@@ -11,10 +11,6 @@
 
 namespace DB
 {
-namespace ErrorCodes
-{
-    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-}
 struct Settings;
 
 /// min, max, any, anyLast, anyHeavy, etc...
@@ -51,29 +47,23 @@ static IAggregateFunction * createAggregateFunctionSingleValue(const String & na
     return new AggregateFunctionTemplate<Data<SingleValueDataGeneric<>>>(argument_type);
 }
 
-template <template <typename> class AggregateFunctionTemplate, template <typename> class Data, bool RespectNulls = false, bool NullIsGreater = false>
+template <template <typename> class AggregateFunctionTemplate, template <typename> class Data, bool RespectNulls = false>
 static IAggregateFunction * createAggregateFunctionSingleNullableValue(const String & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings)
 {
     assertNoParameters(name, parameters);
     assertUnary(name, argument_types);
-    if (parameters.size() > 2)
-        throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "name's parameter size must not be larger  then 2");
 
     const DataTypePtr & argument_type = argument_types[0];
     WhichDataType which(argument_type);
-    // if the resule value could be null(excluding the case that no row is matched),
+    // If the result value could be null (excluding the case that no row is matched),
     // use SingleValueDataGeneric.
     if constexpr (!RespectNulls)
     {
         return createAggregateFunctionSingleValue<AggregateFunctionTemplate, Data>(name, argument_types, Array(), settings);
     }
-    else if constexpr (NullIsGreater)
-    {
-        return new AggregateFunctionTemplate<Data<SingleValueDataGeneric<true, true>>>(argument_type);
-    }
     else
     {
-        return new AggregateFunctionTemplate<Data<SingleValueDataGeneric<true, false>>>(argument_type);
+        return new AggregateFunctionTemplate<Data<SingleValueDataGeneric<true>>>(argument_type);
     }
     UNREACHABLE();
 }
diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp
index 7fec7f4c5db..1477f3d000d 100644
--- a/src/Parsers/ExpressionListParsers.cpp
+++ b/src/Parsers/ExpressionListParsers.cpp
@@ -1104,7 +1104,7 @@ public:
                     return false;
             }
 
-            NullsAction nulls_action = NullsAction::DEFAULT;
+            NullsAction nulls_action = NullsAction::EMPTY;
             if (respect_nulls.ignore(pos, expected))
             {
                 nulls_action = NullsAction::RESPECT_NULLS;
@@ -1150,19 +1150,25 @@ private:
 
     enum NullsAction
     {
-        DEFAULT = 0,
+        EMPTY = 0,
         RESPECT_NULLS = 1,
         IGNORE_NULLS = 2,
     };
     static String transformFunctionNameForRepectNulls(const String & original_function_name, NullsAction nulls_action)
     {
         static std::unordered_map<String, std::vector<String>> renamed_functions_with_nulls = {
-            {"first_value", {"first_value_ignore_nulls", "first_value_respect_nulls", "first_value_ignore_nulls"}},
-            {"last_value", {"last_value_ignore_nulls", "last_value_respect_nulls", "last_value_ignore_nulls"}},
+            {"first_value", {"first_value", "first_value_respect_nulls", "first_value"}},
+            {"last_value", {"last_value", "last_value_respect_nulls", "last_value"}},
         };
         auto it = renamed_functions_with_nulls.find(original_function_name);
         if (it == renamed_functions_with_nulls.end())
-            return original_function_name;
+        {
+            if (nulls_action == NullsAction::EMPTY)
+                return original_function_name;
+            else
+                throw Exception(
+                    ErrorCodes::SYNTAX_ERROR, "Function {} does not support RESPECT NULLS or IGNORE NULLS", original_function_name);
+        }
         return it->second[nulls_action];
     }
 };

From 64fb8da812654c155a539463ddba72d6a8a6eee3 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sat, 22 Apr 2023 17:28:53 +0000
Subject: [PATCH 069/689] improve AsyncLoader interface

---
 src/Common/AsyncLoader.cpp              |  85 +++++++--------
 src/Common/AsyncLoader.h                | 132 +++++++++++++++---------
 src/Common/tests/gtest_async_loader.cpp | 127 +++++++++++------------
 3 files changed, 184 insertions(+), 160 deletions(-)

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
index 1d428b3601f..804b8ea71dc 100644
--- a/src/Common/AsyncLoader.cpp
+++ b/src/Common/AsyncLoader.cpp
@@ -88,59 +88,40 @@ void LoadJob::finish()
         finished.notify_all();
 }
 
-
-AsyncLoader::Task::Task()
-    : loader(nullptr)
-{}
-
-AsyncLoader::Task::Task(AsyncLoader * loader_, LoadJobSet && jobs_)
+LoadTask::LoadTask(AsyncLoader & loader_, LoadJobSet && jobs_, LoadJobSet && goal_jobs_)
     : loader(loader_)
     , jobs(std::move(jobs_))
+    , goal_jobs(std::move(goal_jobs_))
 {}
 
-AsyncLoader::Task::Task(AsyncLoader::Task && o) noexcept
-    : loader(std::exchange(o.loader, nullptr))
-    , jobs(std::move(o.jobs))
-{}
-
-AsyncLoader::Task::~Task()
+LoadTask::~LoadTask()
 {
     remove();
 }
 
-AsyncLoader::Task & AsyncLoader::Task::operator=(AsyncLoader::Task && o) noexcept
+void LoadTask::merge(const LoadTaskPtr & task)
 {
-    loader = std::exchange(o.loader, nullptr);
-    jobs = std::move(o.jobs);
-    return *this;
+    chassert(&loader == &task->loader);
+    jobs.merge(task->jobs);
+    goal_jobs.merge(task->goal_jobs);
 }
 
-void AsyncLoader::Task::merge(AsyncLoader::Task && o)
+void LoadTask::schedule()
 {
-    if (!loader)
+    loader.schedule(*this);
+}
+
+void LoadTask::remove()
+{
+    if (!jobs.empty())
     {
-        *this = std::move(o);
-    }
-    else
-    {
-        chassert(loader == o.loader);
-        jobs.merge(o.jobs);
-        o.loader = nullptr;
+        loader.remove(jobs);
+        jobs.clear();
     }
 }
 
-void AsyncLoader::Task::remove()
+void LoadTask::detach()
 {
-    if (loader)
-    {
-        loader->remove(jobs);
-        detach();
-    }
-}
-
-void AsyncLoader::Task::detach()
-{
-    loader = nullptr;
     jobs.clear();
 }
 
@@ -178,7 +159,30 @@ void AsyncLoader::stop()
     pool.wait();
 }
 
-AsyncLoader::Task AsyncLoader::schedule(LoadJobSet && jobs, ssize_t priority)
+void AsyncLoader::schedule(LoadTask & task)
+{
+    chassert(this == &task.loader);
+    scheduleImpl(task.jobs);
+}
+
+void AsyncLoader::schedule(const LoadTaskPtr & task)
+{
+    chassert(this == &task->loader);
+    scheduleImpl(task->jobs);
+}
+
+void AsyncLoader::schedule(const std::vector<LoadTaskPtr> & tasks)
+{
+    LoadJobSet all_jobs;
+    for (const auto & task : tasks)
+    {
+        chassert(this == &task->loader);
+        all_jobs.insert(task->jobs.begin(), task->jobs.end());
+    }
+    scheduleImpl(all_jobs);
+}
+
+void AsyncLoader::scheduleImpl(const LoadJobSet & jobs)
 {
     std::unique_lock lock{mutex};
 
@@ -202,9 +206,8 @@ AsyncLoader::Task AsyncLoader::schedule(LoadJobSet && jobs, ssize_t priority)
     {
         NOEXCEPT_SCOPE({
             ALLOW_ALLOCATIONS_IN_SCOPE;
-            scheduled_jobs.emplace(job, Info{.initial_priority = priority, .priority = priority});
+            scheduled_jobs.emplace(job, Info{.initial_priority = job->load_priority, .priority = job->load_priority});
         });
-        job->load_priority.store(priority); // Set user-facing priority
     }
 
     // Process dependencies on scheduled pending jobs
@@ -223,7 +226,7 @@ AsyncLoader::Task AsyncLoader::schedule(LoadJobSet && jobs, ssize_t priority)
                 info.dependencies_left++;
 
                 // Priority inheritance: prioritize deps to have at least given `priority` to avoid priority inversion
-                prioritize(dep, priority, lock);
+                prioritize(dep, info.priority, lock);
             }
         }
 
@@ -282,8 +285,6 @@ AsyncLoader::Task AsyncLoader::schedule(LoadJobSet && jobs, ssize_t priority)
             // Job was already canceled on previous iteration of this cycle -- skip
         }
     }
-
-    return Task(this, std::move(jobs));
 }
 
 void AsyncLoader::prioritize(const LoadJobPtr & job, ssize_t new_priority)
diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 6f49b9b2d72..abeb456cacb 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -5,6 +5,7 @@
 #include <memory>
 #include <map>
 #include <mutex>
+#include <vector>
 #include <unordered_set>
 #include <unordered_map>
 #include <boost/noncopyable.hpp>
@@ -19,6 +20,7 @@ class LoadJob;
 using LoadJobPtr = std::shared_ptr<LoadJob>;
 using LoadJobSet = std::unordered_set<LoadJobPtr>;
 class LoadTask;
+using LoadTaskPtr = std::shared_ptr<LoadTask>;
 class AsyncLoader;
 
 // Execution status of a load job.
@@ -37,17 +39,23 @@ class LoadJob : private boost::noncopyable
 {
 public:
     template <class Func>
-    LoadJob(LoadJobSet && dependencies_, String name_, Func && func_)
+    LoadJob(LoadJobSet && dependencies_, String name_, ssize_t priority_, Func && func_)
         : dependencies(std::move(dependencies_))
         , name(std::move(name_))
         , func(std::forward<Func>(func_))
+        , load_priority(priority_)
+    {}
+
+    template <class Func>
+    LoadJob(LoadJobSet && dependencies_, String name_, Func && func_)
+        : LoadJob(std::move(dependencies_), std::move(name_), 0, std::forward<Func>(func_))
     {}
 
     // Current job status.
     LoadStatus status() const;
     std::exception_ptr exception() const;
 
-    // Returns current value of a priority of the job. May differ from initial priority passed to `AsyncLoader:::schedule()` call.
+    // Returns current value of a priority of the job. May differ from initial priority.
     ssize_t priority() const;
 
     // Sync wait for a pending job to be finished: OK, FAILED or CANCELED status.
@@ -72,7 +80,7 @@ private:
     void finish();
 
     std::function<void(const LoadJobPtr & self)> func;
-    std::atomic<ssize_t> load_priority{0};
+    std::atomic<ssize_t> load_priority;
 
     mutable std::mutex mutex;
     mutable std::condition_variable finished;
@@ -87,6 +95,49 @@ LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, Func && func)
     return std::make_shared<LoadJob>(std::move(dependencies), std::move(name), std::forward<Func>(func));
 }
 
+template <class Func>
+LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, ssize_t priority, Func && func)
+{
+    return std::make_shared<LoadJob>(std::move(dependencies), std::move(name), priority, std::forward<Func>(func));
+}
+
+// Represents a logically connected set of LoadJobs required to achieve some goals (final LoadJob in the set).
+class LoadTask : private boost::noncopyable
+{
+public:
+    LoadTask(AsyncLoader & loader_, LoadJobSet && jobs_, LoadJobSet && goals_ = {});
+    ~LoadTask();
+
+    // Merge all jobs from other task into this task.
+    void merge(const LoadTaskPtr & o);
+
+    // Schedule all jobs with AsyncLoader.
+    void schedule();
+
+    // Remove all jobs of this task from AsyncLoader.
+    void remove();
+
+    // Do not track jobs in this task.
+    // WARNING: Jobs will never be removed() and are going to be stored as finished jobs until ~AsyncLoader().
+    void detach();
+
+    // Return the final jobs in this tasks. This jobs subset should be used as `dependencies` for dependent jobs or tasks.
+    const LoadJobSet & goals() const { return goal_jobs.empty() ? jobs : goal_jobs; }
+
+private:
+    friend class AsyncLoader;
+
+    AsyncLoader & loader;
+    LoadJobSet jobs;
+    LoadJobSet goal_jobs;
+};
+
+inline LoadTaskPtr makeLoadTask(AsyncLoader & loader, LoadJobSet && jobs, LoadJobSet && goals = {})
+{
+    return std::make_shared<LoadTask>(loader, std::move(jobs), std::move(goals));
+}
+
+
 // `AsyncLoader` is a scheduler for DAG of `LoadJob`s. It tracks dependencies and priorities of jobs.
 // Basic usage example:
 //     auto job_func = [&] (const LoadJobPtr & self) {
@@ -95,28 +146,29 @@ LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, Func && func)
 //     auto job1 = makeLoadJob({}, "job1", job_func);
 //     auto job2 = makeLoadJob({ job1 }, "job2", job_func);
 //     auto job3 = makeLoadJob({ job1 }, "job3", job_func);
-//     auto task = async_loader.schedule({ job1, job2, job3 }, /* priority = */ 0);
+//     auto task = makeLoadTask(async_loader, { job1, job2, job3 });
+//     task.schedule();
 // Here we have created and scheduled a task consisting of three jobs. Job1 has no dependencies and is run first.
 // Job2 and job3 depend on job1 and are run only after job1 completion. Another thread may prioritize a job and wait for it:
-//     async_loader->prioritize(job3, /* priority = */ 1); // higher priority jobs are run first
+//     async_loader->prioritize(job3, /* priority = */ 1); // higher priority jobs are run first, default priority is zero.
 //     job3->wait(); // blocks until job completion or cancellation and rethrow an exception (if any)
 //
 // AsyncLoader tracks state of all scheduled jobs. Job lifecycle is the following:
-// 1)  Job is constructed with PENDING status.
-// 2)  Job is scheduled and placed into a task. Scheduled job may be ready (i.e. have all its dependencies finished) or blocked.
-// 3a) When all dependencies are successfully executed, job became ready. Ready job is enqueued into the ready queue.
-// 3b) If at least one of job dependencies is failed or canceled, then this job is canceled (with all it's dependent jobs as well).
-//     On cancellation an ASYNC_LOAD_CANCELED exception is generated and saved inside LoadJob object. Job status is changed to CANCELED.
-//     Exception is rethrown by any existing or new `wait()` call. Job is moved to the set of finished jobs.
-// 4)  Scheduled pending ready job starts execution by a worker. Job is dequeuedCallback `job_func` is called.
-//     Status of an executing job is PENDING. And it is still considered as scheduled job by AsyncLoader.
+// 1)  Job is constructed with PENDING status and initial priority. The job is placed into a task.
+// 2)  The task is scheduled with all its jobs. A scheduled job may be ready (i.e. have all its dependencies finished) or blocked.
+// 3a) When all dependencies are successfully executed, the job became ready. A ready job is enqueued into the ready queue.
+// 3b) If at least one of the job dependencies is failed or canceled, then this job is canceled (with all it's dependent jobs as well).
+//     On cancellation an ASYNC_LOAD_CANCELED exception is generated and saved inside LoadJob object. The job status is changed to CANCELED.
+//     Exception is rethrown by any existing or new `wait()` call. The job is moved to the set of the finished jobs.
+// 4)  The scheduled pending ready job starts execution by a worker. The job is dequeuedCallback `job_func` is called.
+//     Status of an executing job is PENDING. And it is still considered as a scheduled job by AsyncLoader.
 //     Note that `job_func` of a CANCELED job is never executed.
-// 5a) On successful execution job status is changed to OK and all existing and new `wait()` calls finish w/o exceptions.
-// 5b) Any exception thrown out of `job_func` is wrapped into ASYNC_LOAD_FAILED exception and save inside LoadJob.
-//     Job status is changed to FAILED. All dependent jobs are canceled. The exception is rethrown from all existing and new `wait()` calls.
-// 6)  Job is no longer considered as scheduled and is instead moved to finished jobs set. This is required for introspection of finished jobs.
-// 7)  Task object containing this job is destructed or `remove()` is explicitly called. Job is removed from the finished job set.
-// 8)  Job is destructed.
+// 5a) On successful execution the job status is changed to OK and all existing and new `wait()` calls finish w/o exceptions.
+// 5b) Any exception thrown out of `job_func` is wrapped into an ASYNC_LOAD_FAILED exception and save inside LoadJob.
+//     The job status is changed to FAILED. All the dependent jobs are canceled. The exception is rethrown from all existing and new `wait()` calls.
+// 6)  The job is no longer considered as scheduled and is instead moved to the finished jobs set. This is just for introspection of the finished jobs.
+// 7)  The task containing this job is destructed or `remove()` is explicitly called. The job is removed from the finished job set.
+// 8)  The job is destructed.
 //
 // Every job has a priority associated with it. AsyncLoader runs higher priority (greater `priority` value) jobs first. Job priority can be elevated
 // (a) if either it has a dependent job with higher priority (in this case priority of a dependent job is inherited);
@@ -125,7 +177,7 @@ LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, Func && func)
 // Value stored in load job priority field is atomic and can be increased even during job execution.
 //
 // When task is scheduled it can contain dependencies on previously scheduled jobs. These jobs can have any status.
-// The only forbidden thing is a dependency on job, that was not scheduled in AsyncLoader yet: all dependent jobs are immediately canceled.
+// The only forbidden thing is a dependency on job, that was not scheduled in AsyncLoader yet: all the dependent jobs are immediately canceled.
 class AsyncLoader : private boost::noncopyable
 {
 private:
@@ -174,36 +226,9 @@ private:
 public:
     using Metric = CurrentMetrics::Metric;
 
-    // Helper class that removes all not started and finished jobs in destructor and waits for all the executing jobs to finish.
-    class Task
-    {
-    public:
-        Task();
-        Task(AsyncLoader * loader_, LoadJobSet && jobs_);
-        Task(const Task & o) = delete;
-        Task(Task && o) noexcept;
-        Task & operator=(const Task & o) = delete;
-        ~Task();
-        Task & operator=(Task && o) noexcept;
-
-        // Merge all jobs from other task into this task. Useful for merging jobs with different priorities into one task.
-        void merge(Task && o);
-
-        // Remove all jobs of this task from AsyncLoader.
-        void remove();
-
-        // Do not track jobs in this task.
-        // WARNING: Jobs will never be removed() and are going to be stored as finished jobs until ~AsyncLoader().
-        void detach();
-
-    private:
-        AsyncLoader * loader;
-        LoadJobSet jobs;
-    };
-
     AsyncLoader(Metric metric_threads, Metric metric_active_threads, size_t max_threads_, bool log_failures_);
 
-    // WARNING: all Task instances returned by `schedule()` should be destructed before AsyncLoader.
+    // WARNING: all LoadTask instances returned by `schedule()` should be destructed before AsyncLoader.
     ~AsyncLoader();
 
     // Start workers to execute scheduled load jobs.
@@ -218,13 +243,17 @@ public:
     //  - or canceled using ~Task() or remove() later.
     void stop();
 
-    // Schedule all `jobs` with given `priority` and return a task containing these jobs.
+    // Schedule all jobs of given task.
     // Higher priority jobs (with greater `priority` value) are executed earlier.
     // All dependencies of a scheduled job inherit its priority if it is higher. This way higher priority job
     // never wait for (blocked by) lower priority jobs (no priority inversion).
-    // Returned task destructor ensures that all the `jobs` are finished (OK, FAILED or CANCELED)
+    // Task destructor ensures that all the `jobs` are finished (OK, FAILED or CANCELED)
     // and are removed from AsyncLoader, so it is thread-safe to destroy them.
-    [[nodiscard]] Task schedule(LoadJobSet && jobs, ssize_t priority = 0);
+    void schedule(LoadTask & task);
+    void schedule(const LoadTaskPtr & task);
+
+    // Schedule all tasks atomically. To ensure only highest priority jobs among all tasks are run first.
+    void schedule(const std::vector<LoadTaskPtr> & tasks);
 
     // Increase priority of a job and all its dependencies recursively.
     void prioritize(const LoadJobPtr & job, ssize_t new_priority);
@@ -242,6 +271,7 @@ private:
     void checkCycle(const LoadJobSet & jobs, std::unique_lock<std::mutex> & lock);
     String checkCycleImpl(const LoadJobPtr & job, LoadJobSet & left, LoadJobSet & visited, std::unique_lock<std::mutex> & lock);
     void finish(std::unique_lock<std::mutex> & lock, const LoadJobPtr & job, LoadStatus status, std::exception_ptr exception_from_job = {});
+    void scheduleImpl(const LoadJobSet & jobs);
     void prioritize(const LoadJobPtr & job, ssize_t new_priority, std::unique_lock<std::mutex> & lock);
     void enqueue(Info & info, const LoadJobPtr & job, std::unique_lock<std::mutex> & lock);
     void spawn(std::unique_lock<std::mutex> &);
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index c05dbb751ca..7bd80c28284 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -19,8 +19,8 @@ using namespace DB;
 
 namespace CurrentMetrics
 {
-    extern const Metric TablesLoaderThreads;
-    extern const Metric TablesLoaderThreadsActive;
+    extern const Metric AsyncLoaderThreads;
+    extern const Metric AsyncLoaderThreadsActive;
 }
 
 namespace DB::ErrorCodes
@@ -38,7 +38,7 @@ struct AsyncLoaderTest
     pcg64 rng{randomSeed()};
 
     explicit AsyncLoaderTest(size_t max_threads = 1)
-        : loader(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, max_threads, /* log_failures = */ false)
+        : loader(CurrentMetrics::AsyncLoaderThreads, CurrentMetrics::AsyncLoaderThreadsActive, max_threads, /* log_failures = */ false)
     {}
 
     template <typename T>
@@ -103,6 +103,13 @@ struct AsyncLoaderTest
             jobs.push_back(makeLoadJob({ jobs[j - 1] }, fmt::format("{}{}", name_prefix, j), job_func));
         return {jobs.begin(), jobs.end()};
     }
+
+    LoadTaskPtr schedule(LoadJobSet && jobs)
+    {
+        LoadTaskPtr task = makeLoadTask(loader, std::move(jobs));
+        task->schedule();
+        return task;
+    }
 };
 
 TEST(AsyncLoader, Smoke)
@@ -123,13 +130,13 @@ TEST(AsyncLoader, Smoke)
     {
         auto job1 = makeLoadJob({}, "job1", job_func);
         auto job2 = makeLoadJob({ job1 }, "job2", job_func);
-        auto task1 = t.loader.schedule({ job1, job2 });
+        auto task1 = t.schedule({ job1, job2 });
 
         auto job3 = makeLoadJob({ job2 }, "job3", job_func);
         auto job4 = makeLoadJob({ job2 }, "job4", job_func);
-        auto task2 = t.loader.schedule({ job3, job4 });
-        auto job5 = makeLoadJob({ job3, job4 }, "job5", job_func);
-        task2.merge(t.loader.schedule({ job5 }, low_priority));
+        auto task2 = t.schedule({ job3, job4 });
+        auto job5 = makeLoadJob({ job3, job4 }, "job5", low_priority, job_func);
+        task2->merge(t.schedule({ job5 }));
 
         std::thread waiter_thread([=] { job5->wait(); });
 
@@ -183,7 +190,7 @@ TEST(AsyncLoader, CycleDetection)
         jobs.push_back(makeLoadJob({}, "job9", job_func));
         jobs.push_back(makeLoadJob({ jobs[9] }, "job10", job_func));
 
-        auto task1 = t.loader.schedule({ jobs.begin(), jobs.end()});
+        auto task1 = t.schedule({ jobs.begin(), jobs.end()});
         FAIL();
     }
     catch (Exception & e)
@@ -203,9 +210,9 @@ TEST(AsyncLoader, CancelPendingJob)
     auto job_func = [&] (const LoadJobPtr &) {};
 
     auto job = makeLoadJob({}, "job", job_func);
-    auto task = t.loader.schedule({ job });
+    auto task = t.schedule({ job });
 
-    task.remove(); // this cancels pending the job (async loader was not started to execute it)
+    task->remove(); // this cancels pending the job (async loader was not started to execute it)
 
     ASSERT_EQ(job->status(), LoadStatus::CANCELED);
     try
@@ -227,9 +234,9 @@ TEST(AsyncLoader, CancelPendingTask)
 
     auto job1 = makeLoadJob({}, "job1", job_func);
     auto job2 = makeLoadJob({ job1 }, "job2", job_func);
-    auto task = t.loader.schedule({ job1, job2 });
+    auto task = t.schedule({ job1, job2 });
 
-    task.remove(); // this cancels both jobs (async loader was not started to execute it)
+    task->remove(); // this cancels both jobs (async loader was not started to execute it)
 
     ASSERT_EQ(job1->status(), LoadStatus::CANCELED);
     ASSERT_EQ(job2->status(), LoadStatus::CANCELED);
@@ -263,10 +270,10 @@ TEST(AsyncLoader, CancelPendingDependency)
 
     auto job1 = makeLoadJob({}, "job1", job_func);
     auto job2 = makeLoadJob({ job1 }, "job2", job_func);
-    auto task1 = t.loader.schedule({ job1 });
-    auto task2 = t.loader.schedule({ job2 });
+    auto task1 = t.schedule({ job1 });
+    auto task2 = t.schedule({ job2 });
 
-    task1.remove(); // this cancels both jobs, due to dependency (async loader was not started to execute it)
+    task1->remove(); // this cancels both jobs, due to dependency (async loader was not started to execute it)
 
     ASSERT_EQ(job1->status(), LoadStatus::CANCELED);
     ASSERT_EQ(job2->status(), LoadStatus::CANCELED);
@@ -307,12 +314,12 @@ TEST(AsyncLoader, CancelExecutingJob)
     };
 
     auto job = makeLoadJob({}, "job", job_func);
-    auto task = t.loader.schedule({ job });
+    auto task = t.schedule({ job });
 
     sync.arrive_and_wait(); // (A) wait for job to start executing
     std::thread canceler([&]
     {
-        task.remove(); // waits for (C)
+        task->remove(); // waits for (C)
     });
     while (job->waitersCount() == 0)
         std::this_thread::yield();
@@ -354,14 +361,14 @@ TEST(AsyncLoader, CancelExecutingTask)
         task1_jobs.push_back(blocker_job);
         for (int i = 0; i < 100; i++)
             task1_jobs.push_back(makeLoadJob({ blocker_job }, "job_to_cancel", job_to_cancel_func));
-        auto task1 = t.loader.schedule({ task1_jobs.begin(), task1_jobs.end() });
+        auto task1 = t.schedule({ task1_jobs.begin(), task1_jobs.end() });
         auto job_to_succeed = makeLoadJob({ blocker_job }, "job_to_succeed", job_to_succeed_func);
-        auto task2 = t.loader.schedule({ job_to_succeed });
+        auto task2 = t.schedule({ job_to_succeed });
 
         sync.arrive_and_wait(); // (A) wait for job to start executing
         std::thread canceler([&]
         {
-            task1.remove(); // waits for (C)
+            task1->remove(); // waits for (C)
         });
         while (blocker_job->waitersCount() == 0)
             std::this_thread::yield();
@@ -392,7 +399,7 @@ TEST(AsyncLoader, JobFailure)
     };
 
     auto job = makeLoadJob({}, "job", job_func);
-    auto task = t.loader.schedule({ job });
+    auto task = t.schedule({ job });
 
     t.loader.wait();
 
@@ -421,7 +428,7 @@ TEST(AsyncLoader, ScheduleJobWithFailedDependencies)
     };
 
     auto failed_job = makeLoadJob({}, "failed_job", failed_job_func);
-    auto failed_task = t.loader.schedule({ failed_job });
+    auto failed_task = t.schedule({ failed_job });
 
     t.loader.wait();
 
@@ -429,7 +436,7 @@ TEST(AsyncLoader, ScheduleJobWithFailedDependencies)
 
     auto job1 = makeLoadJob({ failed_job }, "job1", job_func);
     auto job2 = makeLoadJob({ job1 }, "job2", job_func);
-    auto task = t.loader.schedule({ job1, job2 });
+    auto task = t.schedule({ job1, job2 });
 
     t.loader.wait();
 
@@ -463,15 +470,15 @@ TEST(AsyncLoader, ScheduleJobWithCanceledDependencies)
 
     auto canceled_job_func = [&] (const LoadJobPtr &) {};
     auto canceled_job = makeLoadJob({}, "canceled_job", canceled_job_func);
-    auto canceled_task = t.loader.schedule({ canceled_job });
-    canceled_task.remove();
+    auto canceled_task = t.schedule({ canceled_job });
+    canceled_task->remove();
 
     t.loader.start();
 
     auto job_func = [&] (const LoadJobPtr &) {};
     auto job1 = makeLoadJob({ canceled_job }, "job1", job_func);
     auto job2 = makeLoadJob({ job1 }, "job2", job_func);
-    auto task = t.loader.schedule({ job1, job2 });
+    auto task = t.schedule({ job1, job2 });
 
     t.loader.wait();
 
@@ -515,10 +522,10 @@ TEST(AsyncLoader, TestConcurrency)
             executing--;
         };
 
-        std::vector<AsyncLoader::Task> tasks;
+        std::vector<LoadTaskPtr> tasks;
         tasks.reserve(concurrency);
         for (int i = 0; i < concurrency; i++)
-            tasks.push_back(t.loader.schedule(t.chainJobSet(5, job_func)));
+            tasks.push_back(t.schedule(t.chainJobSet(5, job_func)));
         t.loader.wait();
         ASSERT_EQ(executing, 0);
     }
@@ -543,10 +550,10 @@ TEST(AsyncLoader, TestOverload)
         };
 
         t.loader.stop();
-        std::vector<AsyncLoader::Task> tasks;
+        std::vector<LoadTaskPtr> tasks;
         tasks.reserve(concurrency);
         for (int i = 0; i < concurrency; i++)
-            tasks.push_back(t.loader.schedule(t.chainJobSet(5, job_func)));
+            tasks.push_back(t.schedule(t.chainJobSet(5, job_func)));
         t.loader.start();
         t.loader.wait();
         ASSERT_EQ(executing, 0);
@@ -565,22 +572,15 @@ TEST(AsyncLoader, StaticPriorities)
     };
 
     std::vector<LoadJobPtr> jobs;
-    jobs.push_back(makeLoadJob({}, "A", job_func)); // 0
-    jobs.push_back(makeLoadJob({ jobs[0] }, "B", job_func)); // 1
-    jobs.push_back(makeLoadJob({ jobs[0] }, "C", job_func)); // 2
-    jobs.push_back(makeLoadJob({ jobs[0] }, "D", job_func)); // 3
-    jobs.push_back(makeLoadJob({ jobs[0] }, "E", job_func)); // 4
-    jobs.push_back(makeLoadJob({ jobs[3], jobs[4] }, "F", job_func)); // 5
-    jobs.push_back(makeLoadJob({ jobs[5] }, "G", job_func)); // 6
-    jobs.push_back(makeLoadJob({ jobs[6] }, "H", job_func)); // 7
-    auto task = t.loader.schedule({ jobs[0] }, 0);
-    task.merge(t.loader.schedule({ jobs[1] }, 3));
-    task.merge(t.loader.schedule({ jobs[2] }, 4));
-    task.merge(t.loader.schedule({ jobs[3] }, 1));
-    task.merge(t.loader.schedule({ jobs[4] }, 2));
-    task.merge(t.loader.schedule({ jobs[5] }, 0));
-    task.merge(t.loader.schedule({ jobs[6] }, 0));
-    task.merge(t.loader.schedule({ jobs[7] }, 9));
+    jobs.push_back(makeLoadJob({}, "A", 0, job_func)); // 0
+    jobs.push_back(makeLoadJob({ jobs[0] }, "B", 3, job_func)); // 1
+    jobs.push_back(makeLoadJob({ jobs[0] }, "C", 4, job_func)); // 2
+    jobs.push_back(makeLoadJob({ jobs[0] }, "D", 1, job_func)); // 3
+    jobs.push_back(makeLoadJob({ jobs[0] }, "E", 2, job_func)); // 4
+    jobs.push_back(makeLoadJob({ jobs[3], jobs[4] }, "F", 0, job_func)); // 5
+    jobs.push_back(makeLoadJob({ jobs[5] }, "G", 0, job_func)); // 6
+    jobs.push_back(makeLoadJob({ jobs[6] }, "H", 9, job_func)); // 7
+    auto task = t.schedule({ jobs.begin(), jobs.end() });
 
     t.loader.start();
     t.loader.wait();
@@ -614,22 +614,15 @@ TEST(AsyncLoader, DynamicPriorities)
         //     |       +-> F0 --> G0 --> H0
         //     `-> E2 -'
         std::vector<LoadJobPtr> jobs;
-        jobs.push_back(makeLoadJob({}, "A", job_func)); // 0
-        jobs.push_back(makeLoadJob({ jobs[0] }, "B", job_func)); // 1
-        jobs.push_back(makeLoadJob({ jobs[0] }, "C", job_func)); // 2
-        jobs.push_back(makeLoadJob({ jobs[0] }, "D", job_func)); // 3
-        jobs.push_back(makeLoadJob({ jobs[0] }, "E", job_func)); // 4
-        jobs.push_back(makeLoadJob({ jobs[3], jobs[4] }, "F", job_func)); // 5
-        jobs.push_back(makeLoadJob({ jobs[5] }, "G", job_func)); // 6
-        jobs.push_back(makeLoadJob({ jobs[6] }, "H", job_func)); // 7
-        auto task = t.loader.schedule({ jobs[0] }, 0);
-        task.merge(t.loader.schedule({ jobs[1] }, 3));
-        task.merge(t.loader.schedule({ jobs[2] }, 4));
-        task.merge(t.loader.schedule({ jobs[3] }, 1));
-        task.merge(t.loader.schedule({ jobs[4] }, 2));
-        task.merge(t.loader.schedule({ jobs[5] }, 0));
-        task.merge(t.loader.schedule({ jobs[6] }, 0));
-        task.merge(t.loader.schedule({ jobs[7] }, 0));
+        jobs.push_back(makeLoadJob({}, "A", 0, job_func)); // 0
+        jobs.push_back(makeLoadJob({ jobs[0] }, "B", 3, job_func)); // 1
+        jobs.push_back(makeLoadJob({ jobs[0] }, "C", 4, job_func)); // 2
+        jobs.push_back(makeLoadJob({ jobs[0] }, "D", 1, job_func)); // 3
+        jobs.push_back(makeLoadJob({ jobs[0] }, "E", 2, job_func)); // 4
+        jobs.push_back(makeLoadJob({ jobs[3], jobs[4] }, "F", 0, job_func)); // 5
+        jobs.push_back(makeLoadJob({ jobs[5] }, "G", 0, job_func)); // 6
+        jobs.push_back(makeLoadJob({ jobs[6] }, "H", 0, job_func)); // 7
+        auto task = t.schedule({ jobs.begin(), jobs.end() });
 
         job_to_prioritize = jobs[6];
 
@@ -656,12 +649,12 @@ TEST(AsyncLoader, RandomIndependentTasks)
         t.randomSleepUs(100, 500, 5);
     };
 
-    std::vector<AsyncLoader::Task> tasks;
+    std::vector<LoadTaskPtr> tasks;
     tasks.reserve(512);
     for (int i = 0; i < 512; i++)
     {
         int job_count = t.randomInt(1, 32);
-        tasks.push_back(t.loader.schedule(t.randomJobSet(job_count, 5, job_func)));
+        tasks.push_back(t.schedule(t.randomJobSet(job_count, 5, job_func)));
         t.randomSleepUs(100, 900, 20); // avg=100us
     }
 }
@@ -673,7 +666,7 @@ TEST(AsyncLoader, RandomDependentTasks)
 
     std::mutex mutex;
     std::condition_variable cv;
-    std::vector<AsyncLoader::Task> tasks;
+    std::vector<LoadTaskPtr> tasks;
     std::vector<LoadJobPtr> all_jobs;
 
     auto job_func = [&] (const LoadJobPtr & self)
@@ -695,7 +688,7 @@ TEST(AsyncLoader, RandomDependentTasks)
         int job_count = t.randomInt(1, 32);
         LoadJobSet jobs = t.randomJobSet(job_count, 5, all_jobs, job_func);
         all_jobs.insert(all_jobs.end(), jobs.begin(), jobs.end());
-        tasks.push_back(t.loader.schedule(std::move(jobs)));
+        tasks.push_back(t.schedule(std::move(jobs)));
 
         // Cancel random old task
         if (tasks.size() > 100)
@@ -732,7 +725,7 @@ TEST(AsyncLoader, SetMaxThreads)
 
     // Generate enough independent jobs
     for (int i = 0; i < 1000; i++)
-        t.loader.schedule({makeLoadJob({}, "job", job_func)}).detach();
+        t.schedule({makeLoadJob({}, "job", job_func)})->detach();
 
     t.loader.start();
     while (sync_index < syncs.size())

From 056268f90b82acc83171cdadd8edce6f0c221f85 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sat, 22 Apr 2023 17:43:36 +0000
Subject: [PATCH 070/689] fix

---
 src/Common/tests/gtest_async_loader.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 7bd80c28284..0a1eddf54ab 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -19,8 +19,8 @@ using namespace DB;
 
 namespace CurrentMetrics
 {
-    extern const Metric AsyncLoaderThreads;
-    extern const Metric AsyncLoaderThreadsActive;
+    extern const Metric TablesLoaderThreads;
+    extern const Metric TablesLoaderThreadsActive;
 }
 
 namespace DB::ErrorCodes
@@ -38,7 +38,7 @@ struct AsyncLoaderTest
     pcg64 rng{randomSeed()};
 
     explicit AsyncLoaderTest(size_t max_threads = 1)
-        : loader(CurrentMetrics::AsyncLoaderThreads, CurrentMetrics::AsyncLoaderThreadsActive, max_threads, /* log_failures = */ false)
+        : loader(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, max_threads, /* log_failures = */ false)
     {}
 
     template <typename T>

From 34d82625b58e157d937d48a9660079d63deab785 Mon Sep 17 00:00:00 2001
From: FFFFFFFHHHHHHH <916677625@qq.com>
Date: Sun, 23 Apr 2023 02:58:00 +0800
Subject: [PATCH 071/689] feat: add dotProduct for array

---
 src/Functions/array/arrayAUC.cpp         |  2 +-
 src/Functions/array/arrayDotProduct.cpp  | 75 ++++++++++++++++++++++++
 src/Functions/array/arrayScalarProduct.h | 71 +++++++++++++---------
 src/Functions/vectorFunctions.cpp        | 16 ++++-
 4 files changed, 133 insertions(+), 31 deletions(-)
 create mode 100644 src/Functions/array/arrayDotProduct.cpp

diff --git a/src/Functions/array/arrayAUC.cpp b/src/Functions/array/arrayAUC.cpp
index 2890ae55886..297394822d9 100644
--- a/src/Functions/array/arrayAUC.cpp
+++ b/src/Functions/array/arrayAUC.cpp
@@ -91,7 +91,7 @@ public:
         return std::make_shared<DataTypeNumber<ResultType>>();
     }
 
-    template <typename T, typename U>
+    template <typename ResultType, typename T, typename U>
     static ResultType apply(
         const T * scores,
         const U * labels,
diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp
new file mode 100644
index 00000000000..4b9433f683d
--- /dev/null
+++ b/src/Functions/array/arrayDotProduct.cpp
@@ -0,0 +1,75 @@
+#include <DataTypes/DataTypesNumber.h>
+#include <Functions/FunctionFactory.h>
+#include <DataTypes/getLeastSupertype.h>
+#include "arrayScalarProduct.h"
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+}
+
+struct NameArrayDotProduct
+{
+    static constexpr auto name = "arrayDotProduct";
+};
+
+
+class ArrayDotProductImpl
+{
+public:
+
+    static DataTypePtr getReturnType(const DataTypePtr left_type, const DataTypePtr & right_type)
+    {
+        const auto & common_type = getLeastSupertype(DataTypes{left_type, right_type});
+        switch (common_type->getTypeId())
+        {
+            case TypeIndex::UInt8:
+            case TypeIndex::UInt16:
+            case TypeIndex::UInt32:
+            case TypeIndex::Int8:
+            case TypeIndex::Int16:
+            case TypeIndex::Int32:
+            case TypeIndex::UInt64:
+            case TypeIndex::Int64:
+            case TypeIndex::Float64:
+                return std::make_shared<DataTypeFloat64>();
+            case TypeIndex::Float32:
+                return std::make_shared<DataTypeFloat32>();
+           default:
+                throw Exception(
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Arguments of function {} has nested type {}. "
+                    "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.",
+                    std::string(NameArrayDotProduct::name),
+                    common_type->getName());
+        }
+    }
+
+    template <typename ResultType, typename T, typename U>
+    static ResultType apply(
+        const T * left,
+        const U * right,
+        size_t size)
+    {
+        ResultType result = 0;
+        for (size_t i = 0; i < size; ++i) 
+            result += static_cast<ResultType>(left[i]) * static_cast<ResultType>(right[i]);
+        return result;
+    }
+
+};
+
+using FunctionArrayDotProduct = FunctionArrayScalarProduct<ArrayDotProductImpl, NameArrayDotProduct>;
+
+REGISTER_FUNCTION(ArrayDotProduct)
+{
+    factory.registerFunction<FunctionArrayDotProduct>();
+}
+
+/// These functions are used by TupleOrArrayFunction in Function/vectorFunctions.cpp
+FunctionPtr createFunctionArrayDotProduct(ContextPtr context_) { return FunctionArrayDotProduct::create(context_); }
+}
diff --git a/src/Functions/array/arrayScalarProduct.h b/src/Functions/array/arrayScalarProduct.h
index 94ce1bc533c..ded6ec8ae29 100644
--- a/src/Functions/array/arrayScalarProduct.h
+++ b/src/Functions/array/arrayScalarProduct.h
@@ -29,29 +29,28 @@ public:
     static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionArrayScalarProduct>(); }
 
 private:
-    using ResultColumnType = ColumnVector<typename Method::ResultType>;
 
-    template <typename T>
+    template <typename ResultType, typename T>
     ColumnPtr executeNumber(const ColumnsWithTypeAndName & arguments) const
     {
         ColumnPtr res;
-        if (   (res = executeNumberNumber<T, UInt8>(arguments))
-            || (res = executeNumberNumber<T, UInt16>(arguments))
-            || (res = executeNumberNumber<T, UInt32>(arguments))
-            || (res = executeNumberNumber<T, UInt64>(arguments))
-            || (res = executeNumberNumber<T, Int8>(arguments))
-            || (res = executeNumberNumber<T, Int16>(arguments))
-            || (res = executeNumberNumber<T, Int32>(arguments))
-            || (res = executeNumberNumber<T, Int64>(arguments))
-            || (res = executeNumberNumber<T, Float32>(arguments))
-            || (res = executeNumberNumber<T, Float64>(arguments)))
+        if (   (res = executeNumberNumber<ResultType, T, UInt8>(arguments))
+            || (res = executeNumberNumber<ResultType, T, UInt16>(arguments))
+            || (res = executeNumberNumber<ResultType, T, UInt32>(arguments))
+            || (res = executeNumberNumber<ResultType, T, UInt64>(arguments))
+            || (res = executeNumberNumber<ResultType, T, Int8>(arguments))
+            || (res = executeNumberNumber<ResultType, T, Int16>(arguments))
+            || (res = executeNumberNumber<ResultType, T, Int32>(arguments))
+            || (res = executeNumberNumber<ResultType, T, Int64>(arguments))
+            || (res = executeNumberNumber<ResultType, T, Float32>(arguments))
+            || (res = executeNumberNumber<ResultType, T, Float64>(arguments)))
             return res;
 
        return nullptr;
     }
 
 
-    template <typename T, typename U>
+    template <typename ResultType, typename T, typename U>
     ColumnPtr executeNumberNumber(const ColumnsWithTypeAndName & arguments) const
     {
         ColumnPtr col1 = arguments[0].column->convertToFullColumnIfConst();
@@ -72,7 +71,7 @@ private:
         if (!col_nested1 || !col_nested2)
             return nullptr;
 
-        auto col_res = ResultColumnType::create();
+        auto col_res = ColumnVector<ResultType>::create();
 
         vector(
             col_nested1->getData(),
@@ -83,12 +82,12 @@ private:
         return col_res;
     }
 
-    template <typename T, typename U>
+    template <typename ResultType, typename T, typename U>
     static NO_INLINE void vector(
         const PaddedPODArray<T> & data1,
         const PaddedPODArray<U> & data2,
         const ColumnArray::Offsets & offsets,
-        PaddedPODArray<typename Method::ResultType> & result)
+        PaddedPODArray<ResultType> & result)
     {
         size_t size = offsets.size();
         result.resize(size);
@@ -97,7 +96,7 @@ private:
         for (size_t i = 0; i < size; ++i)
         {
             size_t array_size = offsets[i] - current_offset;
-            result[i] = Method::apply(&data1[current_offset], &data2[current_offset], array_size);
+            result[i] = Method::template apply<ResultType, T, U>(&data1[current_offset], &data2[current_offset], array_size);
             current_offset = offsets[i];
         }
     }
@@ -130,24 +129,40 @@ public:
         return Method::getReturnType(nested_types[0], nested_types[1]);
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /* input_rows_count */) const override
+    template <typename ResultType>
+    ColumnPtr executeWithResultType(const ColumnsWithTypeAndName & arguments) const
     {
         ColumnPtr res;
-        if (!((res = executeNumber<UInt8>(arguments))
-            || (res = executeNumber<UInt16>(arguments))
-            || (res = executeNumber<UInt32>(arguments))
-            || (res = executeNumber<UInt64>(arguments))
-            || (res = executeNumber<Int8>(arguments))
-            || (res = executeNumber<Int16>(arguments))
-            || (res = executeNumber<Int32>(arguments))
-            || (res = executeNumber<Int64>(arguments))
-            || (res = executeNumber<Float32>(arguments))
-            || (res = executeNumber<Float64>(arguments))))
+        if ( !((res = executeNumber<ResultType, UInt8>(arguments))
+            || (res = executeNumber<ResultType, UInt16>(arguments))
+            || (res = executeNumber<ResultType, UInt32>(arguments))
+            || (res = executeNumber<ResultType, UInt64>(arguments))
+            || (res = executeNumber<ResultType, Int8>(arguments))
+            || (res = executeNumber<ResultType, Int16>(arguments))
+            || (res = executeNumber<ResultType, Int32>(arguments))
+            || (res = executeNumber<ResultType, Int64>(arguments))
+            || (res = executeNumber<ResultType, Float32>(arguments))
+            || (res = executeNumber<ResultType, Float64>(arguments))))
             throw Exception(ErrorCodes::ILLEGAL_COLUMN,
                 "Illegal column {} of first argument of function {}", arguments[0].column->getName(), getName());
 
         return res;
     }
+
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /* input_rows_count */) const override
+    {
+        switch (result_type->getTypeId())
+        {
+            case TypeIndex::Float32:
+                return executeWithResultType<Float32>(arguments);
+                break;
+            case TypeIndex::Float64:
+                return executeWithResultType<Float64>(arguments);
+                break;
+            default:
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected result type {}", result_type->getName());
+        }
+    }
 };
 
 }
diff --git a/src/Functions/vectorFunctions.cpp b/src/Functions/vectorFunctions.cpp
index a7ab09612cf..b52def28755 100644
--- a/src/Functions/vectorFunctions.cpp
+++ b/src/Functions/vectorFunctions.cpp
@@ -1429,6 +1429,8 @@ private:
     FunctionPtr array_function;
 };
 
+extern FunctionPtr createFunctionArrayDotProduct(ContextPtr context_);
+
 extern FunctionPtr createFunctionArrayL1Norm(ContextPtr context_);
 extern FunctionPtr createFunctionArrayL2Norm(ContextPtr context_);
 extern FunctionPtr createFunctionArrayL2SquaredNorm(ContextPtr context_);
@@ -1442,6 +1444,14 @@ extern FunctionPtr createFunctionArrayLpDistance(ContextPtr context_);
 extern FunctionPtr createFunctionArrayLinfDistance(ContextPtr context_);
 extern FunctionPtr createFunctionArrayCosineDistance(ContextPtr context_);
 
+struct DotProduct
+{
+    static constexpr auto name = "dotProduct";
+
+    static constexpr auto CreateTupleFunction = FunctionDotProduct::create;
+    static constexpr auto CreateArrayFunction = createFunctionArrayDotProduct;
+};
+
 struct L1NormTraits
 {
     static constexpr auto name = "L1Norm";
@@ -1530,6 +1540,8 @@ struct CosineDistanceTraits
     static constexpr auto CreateArrayFunction = createFunctionArrayCosineDistance;
 };
 
+using TupleOrArrayFunctionDotProduct = TupleOrArrayFunction<DotProduct>;
+
 using TupleOrArrayFunctionL1Norm = TupleOrArrayFunction<L1NormTraits>;
 using TupleOrArrayFunctionL2Norm = TupleOrArrayFunction<L2NormTraits>;
 using TupleOrArrayFunctionL2SquaredNorm = TupleOrArrayFunction<L2SquaredNormTraits>;
@@ -1615,8 +1627,8 @@ If the types of the first interval (or the interval in the tuple) and the second
     factory.registerFunction<FunctionTupleMultiplyByNumber>();
     factory.registerFunction<FunctionTupleDivideByNumber>();
 
-    factory.registerFunction<FunctionDotProduct>();
-    factory.registerAlias("scalarProduct", FunctionDotProduct::name, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<TupleOrArrayFunctionDotProduct>();
+    factory.registerAlias("scalarProduct", TupleOrArrayFunctionDotProduct::name, FunctionFactory::CaseInsensitive);
 
     factory.registerFunction<TupleOrArrayFunctionL1Norm>();
     factory.registerFunction<TupleOrArrayFunctionL2Norm>();

From 81fa4701aaa00974aeb8173c9847dfbe220c2a27 Mon Sep 17 00:00:00 2001
From: FFFFFFFHHHHHHH <916677625@qq.com>
Date: Sun, 23 Apr 2023 03:23:14 +0800
Subject: [PATCH 072/689] feat: add dotProduct for array

---
 .../0_stateless/02708_dot_product.reference   | 10 ++++++
 .../queries/0_stateless/02708_dot_product.sql | 34 +++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 tests/queries/0_stateless/02708_dot_product.reference
 create mode 100644 tests/queries/0_stateless/02708_dot_product.sql

diff --git a/tests/queries/0_stateless/02708_dot_product.reference b/tests/queries/0_stateless/02708_dot_product.reference
new file mode 100644
index 00000000000..7106b870fab
--- /dev/null
+++ b/tests/queries/0_stateless/02708_dot_product.reference
@@ -0,0 +1,10 @@
+3881.304
+3881.304
+3881.304
+376.5
+230
+0
+Float64
+Float32
+Float64
+Float64
diff --git a/tests/queries/0_stateless/02708_dot_product.sql b/tests/queries/0_stateless/02708_dot_product.sql
new file mode 100644
index 00000000000..46450ae6394
--- /dev/null
+++ b/tests/queries/0_stateless/02708_dot_product.sql
@@ -0,0 +1,34 @@
+SELECT dotProduct([12, 2.22, 302], [1.32, 231.2, 11.1]);
+SELECT scalarProduct([12, 2.22, 302], [1.32, 231.2, 11.1]);
+SELECT arrayDotProduct([12, 2.22, 302], [1.32, 231.2, 11.1]);
+
+SELECT dotProduct([1.3, 2, 3, 4, 5], [222, 12, 5.3, 2, 8]);
+
+SELECT dotProduct([1, 1, 1, 1, 1], [222, 12, 0, -12, 8]);
+
+SELECT round(dotProduct([-1, 2, 3.002], [2, 3.4, 4]) - dotProduct((-1, 2, 3.002), (2, 3.4, 4)), 2);
+
+
+DROP TABLE IF EXISTS product_fp64_fp64;
+CREATE TABLE product_fp64_fp64 (x Array(Float64), y Array(Float64)) engine = MergeTree() order by x;
+INSERT INTO TABLE product_fp64_fp64 (x, y) values ([1, 2], [3, 4]);
+SELECT toTypeName(dotProduct(x, y)) from product_fp64_fp64;
+DROP TABLE product_fp64_fp64;
+
+DROP TABLE IF EXISTS product_fp32_fp32;
+CREATE TABLE product_fp32_fp32 (x Array(Float32), y Array(Float32)) engine = MergeTree() order by x;
+INSERT INTO TABLE product_fp32_fp32 (x, y) values ([1, 2], [3, 4]);
+SELECT toTypeName(dotProduct(x, y)) from product_fp32_fp32;
+DROP TABLE product_fp32_fp32;
+
+DROP TABLE IF EXISTS product_fp32_fp64;
+CREATE TABLE product_fp32_fp64 (x Array(Float32), y Array(Float64)) engine = MergeTree() order by x;
+INSERT INTO TABLE product_fp32_fp64 (x, y) values ([1, 2], [3, 4]);
+SELECT toTypeName(dotProduct(x, y)) from product_fp32_fp64;
+DROP TABLE product_fp32_fp64;
+
+DROP TABLE IF EXISTS product_uint8_fp64;
+CREATE TABLE product_uint8_fp64 (x Array(UInt8), y Array(Float64)) engine = MergeTree() order by x;
+INSERT INTO TABLE product_uint8_fp64 (x, y) values ([1, 2], [3, 4]);
+SELECT toTypeName(dotProduct(x, y)) from product_uint8_fp64;
+DROP TABLE product_uint8_fp64;

From 79ae949b566e5023500d037db6327a050dc7a11f Mon Sep 17 00:00:00 2001
From: FFFFFFFHHHHHHH <916677625@qq.com>
Date: Sun, 23 Apr 2023 03:35:09 +0800
Subject: [PATCH 073/689] fix style

---
 src/Functions/array/arrayDotProduct.cpp  | 7 ++-----
 src/Functions/array/arrayScalarProduct.h | 2 +-
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp
index 4b9433f683d..0d21bae90e3 100644
--- a/src/Functions/array/arrayDotProduct.cpp
+++ b/src/Functions/array/arrayDotProduct.cpp
@@ -17,12 +17,10 @@ struct NameArrayDotProduct
     static constexpr auto name = "arrayDotProduct";
 };
 
-
 class ArrayDotProductImpl
 {
 public:
-
-    static DataTypePtr getReturnType(const DataTypePtr left_type, const DataTypePtr & right_type)
+    static DataTypePtr getReturnType(const DataTypePtr & left_type, const DataTypePtr & right_type)
     {
         const auto & common_type = getLeastSupertype(DataTypes{left_type, right_type});
         switch (common_type->getTypeId())
@@ -56,11 +54,10 @@ public:
         size_t size)
     {
         ResultType result = 0;
-        for (size_t i = 0; i < size; ++i) 
+        for (size_t i = 0; i < size; ++i)
             result += static_cast<ResultType>(left[i]) * static_cast<ResultType>(right[i]);
         return result;
     }
-
 };
 
 using FunctionArrayDotProduct = FunctionArrayScalarProduct<ArrayDotProductImpl, NameArrayDotProduct>;
diff --git a/src/Functions/array/arrayScalarProduct.h b/src/Functions/array/arrayScalarProduct.h
index ded6ec8ae29..0d1bf44a3e7 100644
--- a/src/Functions/array/arrayScalarProduct.h
+++ b/src/Functions/array/arrayScalarProduct.h
@@ -133,7 +133,7 @@ public:
     ColumnPtr executeWithResultType(const ColumnsWithTypeAndName & arguments) const
     {
         ColumnPtr res;
-        if ( !((res = executeNumber<ResultType, UInt8>(arguments))
+        if (!((res = executeNumber<ResultType, UInt8>(arguments))
             || (res = executeNumber<ResultType, UInt16>(arguments))
             || (res = executeNumber<ResultType, UInt32>(arguments))
             || (res = executeNumber<ResultType, UInt64>(arguments))

From 4145abf547c3f09b849aa8e3429a318a340ef668 Mon Sep 17 00:00:00 2001
From: FFFFFFFHHHHHHH <916677625@qq.com>
Date: Sun, 23 Apr 2023 04:04:05 +0800
Subject: [PATCH 074/689] fix style

---
 src/Functions/array/arrayScalarProduct.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Functions/array/arrayScalarProduct.h b/src/Functions/array/arrayScalarProduct.h
index 0d1bf44a3e7..5c36f2492c6 100644
--- a/src/Functions/array/arrayScalarProduct.h
+++ b/src/Functions/array/arrayScalarProduct.h
@@ -18,6 +18,7 @@ namespace ErrorCodes
     extern const int ILLEGAL_COLUMN;
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int BAD_ARGUMENTS;
+    extern const int LOGICAL_ERROR;
 }
 
 

From 9e1e94648e4849d15785e00b34fdbf388b639686 Mon Sep 17 00:00:00 2001
From: Larry Luo <larry.luo@ca.ibm.com>
Date: Sun, 23 Apr 2023 21:59:51 -0700
Subject: [PATCH 075/689] support saving interval types to tables

---
 .../data-types/special-data-types/interval.md            | 7 +++----
 src/DataTypes/DataTypeInterval.h                         | 4 ----
 .../0_stateless/02724_persist_interval_type.reference    | 4 ++++
 .../queries/0_stateless/02724_persist_interval_type.sql  | 9 +++++++++
 4 files changed, 16 insertions(+), 8 deletions(-)
 create mode 100644 tests/queries/0_stateless/02724_persist_interval_type.reference
 create mode 100644 tests/queries/0_stateless/02724_persist_interval_type.sql

diff --git a/docs/en/sql-reference/data-types/special-data-types/interval.md b/docs/en/sql-reference/data-types/special-data-types/interval.md
index c89c2e78752..bedbcf0bd28 100644
--- a/docs/en/sql-reference/data-types/special-data-types/interval.md
+++ b/docs/en/sql-reference/data-types/special-data-types/interval.md
@@ -8,10 +8,6 @@ sidebar_label: Interval
 
 The family of data types representing time and date intervals. The resulting types of the [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) operator.
 
-:::note    
-`Interval` data type values can’t be stored in tables.
-:::
-
 Structure:
 
 - Time interval as an unsigned integer value.
@@ -19,6 +15,9 @@ Structure:
 
 Supported interval types:
 
+- `NANOSECOND`
+- `MICROSECOND`
+- `MILLISECOND`
 - `SECOND`
 - `MINUTE`
 - `HOUR`
diff --git a/src/DataTypes/DataTypeInterval.h b/src/DataTypes/DataTypeInterval.h
index 83d89a73460..05abe1d9b24 100644
--- a/src/DataTypes/DataTypeInterval.h
+++ b/src/DataTypes/DataTypeInterval.h
@@ -11,9 +11,6 @@ namespace DB
   *
   * Mostly the same as Int64.
   * But also tagged with interval kind.
-  *
-  * Intended usage is for temporary elements in expressions,
-  *  not for storing values in tables.
   */
 class DataTypeInterval final : public DataTypeNumberBase<Int64>
 {
@@ -34,7 +31,6 @@ public:
     bool equals(const IDataType & rhs) const override;
 
     bool isParametric() const override { return true; }
-    bool cannotBeStoredInTables() const override { return true; }
     bool isCategorial() const override { return false; }
     bool canBeInsideNullable() const override { return true; }
 };
diff --git a/tests/queries/0_stateless/02724_persist_interval_type.reference b/tests/queries/0_stateless/02724_persist_interval_type.reference
new file mode 100644
index 00000000000..9139216d644
--- /dev/null
+++ b/tests/queries/0_stateless/02724_persist_interval_type.reference
@@ -0,0 +1,4 @@
+2023-01-01 00:00:01.000000001	2023-01-01 02:00:00.000000001	2023-01-01 00:00:00.000000004	1	2	0
+2023-01-01 00:00:02.000000001	2023-01-01 03:00:00.000000001	2023-01-01 00:00:00.000000005	2	3	0
+2023-01-01 00:00:01.000000001	2023-01-01 02:00:00.000000001	2023-01-01 00:00:00.000000004	1	2	0
+2023-01-01 00:00:02.000000001	2023-01-01 03:00:00.000000001	2023-01-01 00:00:00.000000005	2	3	0
diff --git a/tests/queries/0_stateless/02724_persist_interval_type.sql b/tests/queries/0_stateless/02724_persist_interval_type.sql
new file mode 100644
index 00000000000..72133aceb00
--- /dev/null
+++ b/tests/queries/0_stateless/02724_persist_interval_type.sql
@@ -0,0 +1,9 @@
+
+DROP TABLE IF EXISTS saved_intervals_tmp;
+DROP TABLE IF EXISTS saved_intervals_mgt;
+create table saved_intervals_tmp Engine=Memory as SELECT number as EventID, toIntervalSecond(number+1) as v1, toIntervalHour(number+2) as v2, toIntervalNanosecond(number+3) as v3 from numbers(2);
+create table saved_intervals_mgt Engine=MergeTree() ORDER BY EventID as SELECT number as EventID, toIntervalSecond(number+1) as v1, toIntervalHour(number+2) as v2, toIntervalNanosecond(number+3) as v3 from numbers(2);
+with toDateTime64('2023-01-01 00:00:00.000000001', 9, 'US/Eastern') as c select c+v1 as c_v1, c+v2 as c_v2, c+v3 as c_v3, date_diff(second, c, c_v1), date_diff(hour, c, c_v2), date_diff(second, c, c_v3) from saved_intervals_tmp;
+with toDateTime64('2023-01-01 00:00:00.000000001', 9, 'US/Eastern') as c select c+v1 as c_v1, c+v2 as c_v2, c+v3 as c_v3, date_diff(second, c, c_v1), date_diff(hour, c, c_v2), date_diff(second, c, c_v3) from saved_intervals_mgt;
+DROP TABLE IF EXISTS saved_intervals_tmp;
+DROP TABLE IF EXISTS saved_intervals_mgt;
\ No newline at end of file

From 69e39aba80b4fadea79bfdf0f83a23848520948e Mon Sep 17 00:00:00 2001
From: FFFFFFFHHHHHHH <916677625@qq.com>
Date: Mon, 24 Apr 2023 22:13:13 +0800
Subject: [PATCH 076/689] fix test

---
 .../02415_all_new_functions_must_be_documented.reference         | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
index 4ff8c2d3af1..98cae995b47 100644
--- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
+++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
@@ -97,6 +97,7 @@ arrayCumSum
 arrayCumSumNonNegative
 arrayDifference
 arrayDistinct
+arrayDotProduct
 arrayElement
 arrayEnumerate
 arrayEnumerateDense

From 2d0812e3c745c1c589a489c824db2f21785733fb Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Mon, 24 Apr 2023 14:55:31 +0000
Subject: [PATCH 077/689] Refactor ColumnLowCardinality::cutAndCompact to avoid
 calling IColumn::assumeMutable.

---
 src/Columns/ColumnLowCardinality.cpp | 27 ++++++++++++---------------
 src/Columns/ColumnLowCardinality.h   |  8 +++++---
 2 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp
index 11d02b023d6..4f9ab8215be 100644
--- a/src/Columns/ColumnLowCardinality.cpp
+++ b/src/Columns/ColumnLowCardinality.cpp
@@ -485,13 +485,8 @@ void ColumnLowCardinality::setSharedDictionary(const ColumnPtr & column_unique)
 ColumnLowCardinality::MutablePtr ColumnLowCardinality::cutAndCompact(size_t start, size_t length) const
 {
     auto sub_positions = IColumn::mutate(idx.getPositions()->cut(start, length));
-    /// Create column with new indexes and old dictionary.
-    /// Dictionary is shared, but will be recreated after compactInplace call.
-    auto column = ColumnLowCardinality::create(getDictionary().assumeMutable(), std::move(sub_positions));
-    /// Will create new dictionary.
-    column->compactInplace();
-
-    return column;
+    auto new_column_unique = Dictionary::compact(dictionary.getColumnUnique(), sub_positions);
+    return ColumnLowCardinality::create(std::move(new_column_unique), std::move(sub_positions));
 }
 
 void ColumnLowCardinality::compactInplace()
@@ -589,7 +584,7 @@ size_t ColumnLowCardinality::Index::getSizeOfIndexType(const IColumn & column, s
                     column.getName());
 }
 
-void ColumnLowCardinality::Index::attachPositions(ColumnPtr positions_)
+void ColumnLowCardinality::Index::attachPositions(MutableColumnPtr positions_)
 {
     positions = std::move(positions_);
     updateSizeOfType();
@@ -820,21 +815,23 @@ void ColumnLowCardinality::Dictionary::setShared(const ColumnPtr & column_unique
     shared = true;
 }
 
-void ColumnLowCardinality::Dictionary::compact(ColumnPtr & positions)
+void ColumnLowCardinality::Dictionary::compact(MutableColumnPtr & positions)
 {
-    auto new_column_unique = column_unique->cloneEmpty();
+    column_unique = compact(getColumnUnique(), positions);
+    shared = false;
+}
 
-    auto & unique = getColumnUnique();
+MutableColumnPtr ColumnLowCardinality::Dictionary::compact(const IColumnUnique & unique, MutableColumnPtr & positions)
+{
+    auto new_column_unique = unique.cloneEmpty();
     auto & new_unique = static_cast<IColumnUnique &>(*new_column_unique);
 
-    auto indexes = mapUniqueIndex(positions->assumeMutableRef());
+    auto indexes = mapUniqueIndex(*positions);
     auto sub_keys = unique.getNestedColumn()->index(*indexes, 0);
     auto new_indexes = new_unique.uniqueInsertRangeFrom(*sub_keys, 0, sub_keys->size());
 
     positions = IColumn::mutate(new_indexes->index(*positions, 0));
-    column_unique = std::move(new_column_unique);
-
-    shared = false;
+    return new_column_unique;
 }
 
 ColumnPtr ColumnLowCardinality::cloneWithDefaultOnNull() const
diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h
index e7f4b92d733..df707039b03 100644
--- a/src/Columns/ColumnLowCardinality.h
+++ b/src/Columns/ColumnLowCardinality.h
@@ -301,8 +301,8 @@ public:
 
         void checkSizeOfType();
 
-        ColumnPtr detachPositions() { return std::move(positions); }
-        void attachPositions(ColumnPtr positions_);
+        MutableColumnPtr detachPositions() { return IColumn::mutate(std::move(positions)); }
+        void attachPositions(MutableColumnPtr positions_);
 
         void countKeys(ColumnUInt64::Container & counts) const;
 
@@ -350,7 +350,9 @@ private:
         bool isShared() const { return shared; }
 
         /// Create new dictionary with only keys that are mentioned in positions.
-        void compact(ColumnPtr & positions);
+        void compact(MutableColumnPtr & positions);
+
+        static MutableColumnPtr compact(const IColumnUnique & column_unique, MutableColumnPtr & positions);
 
     private:
         WrappedPtr column_unique;

From b565257a39355996ff271ace9c0f45462c427e37 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Thu, 20 Apr 2023 13:23:08 +0000
Subject: [PATCH 078/689] Support totals and extremes in query cache

---
 src/Interpreters/Cache/QueryCache.cpp         | 147 +++++++++++++-----
 src/Interpreters/Cache/QueryCache.h           |  28 +++-
 src/Interpreters/executeQuery.cpp             |  17 +-
 src/Processors/ISource.cpp                    |   2 +-
 src/Processors/Sources/SourceFromChunks.cpp   |  86 ++++++----
 src/Processors/Sources/SourceFromChunks.h     |  25 ++-
 .../StreamInQueryCacheTransform.cpp           |  15 +-
 .../Transforms/StreamInQueryCacheTransform.h  |  11 +-
 src/QueryPipeline/QueryPipeline.cpp           |  33 +++-
 src/QueryPipeline/QueryPipeline.h             |   4 +-
 .../System/StorageSystemQueryCache.cpp        |   2 +-
 ...2494_query_cache_totals_extremes.reference |  43 +++++
 .../02494_query_cache_totals_extremes.sql     |  48 ++++++
 13 files changed, 348 insertions(+), 113 deletions(-)
 create mode 100644 tests/queries/0_stateless/02494_query_cache_totals_extremes.reference
 create mode 100644 tests/queries/0_stateless/02494_query_cache_totals_extremes.sql

diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp
index 6337fd9b04c..512abc82529 100644
--- a/src/Interpreters/Cache/QueryCache.cpp
+++ b/src/Interpreters/Cache/QueryCache.cpp
@@ -157,11 +157,13 @@ size_t QueryCache::KeyHasher::operator()(const Key & key) const
     return res;
 }
 
-size_t QueryCache::QueryResultWeight::operator()(const Chunks & chunks) const
+size_t QueryCache::QueryCacheEntryWeight::operator()(const Entry & entry) const
 {
     size_t res = 0;
-    for (const auto & chunk : chunks)
+    for (const auto & chunk : entry.chunks)
         res += chunk.allocatedBytes();
+    res += entry.totals.has_value() ? entry.totals->allocatedBytes() : 0;
+    res += entry.extremes.has_value() ? entry.extremes->allocatedBytes() : 0;
     return res;
 }
 
@@ -191,25 +193,48 @@ QueryCache::Writer::Writer(
     }
 }
 
-void QueryCache::Writer::buffer(Chunk && partial_query_result)
+QueryCache::Writer::Writer(const Writer & other)
+    : cache(other.cache)
+    , key(other.key)
+    , max_entry_size_in_bytes(other.max_entry_size_in_bytes)
+    , max_entry_size_in_rows(other.max_entry_size_in_rows)
+    , min_query_runtime(other.min_query_runtime)
+    , squash_partial_results(other.squash_partial_results)
+    , max_block_size(other.max_block_size)
+{
+}
+
+void QueryCache::Writer::buffer(Chunk && chunk, Type type)
 {
     if (skip_insert)
         return;
 
     std::lock_guard lock(mutex);
 
-    auto & chunks = *query_result;
-
-    chunks.emplace_back(std::move(partial_query_result));
-
-    new_entry_size_in_bytes += chunks.back().allocatedBytes();
-    new_entry_size_in_rows += chunks.back().getNumRows();
-
-    if ((new_entry_size_in_bytes > max_entry_size_in_bytes) || (new_entry_size_in_rows > max_entry_size_in_rows))
+    switch (type)
     {
-        chunks.clear(); /// eagerly free some space
-        skip_insert = true;
-        LOG_TRACE(&Poco::Logger::get("QueryCache"), "Skipped insert (query result too big), new_entry_size_in_bytes: {} ({}), new_entry_size_in_rows: {} ({}), query: {}", new_entry_size_in_bytes, max_entry_size_in_bytes, new_entry_size_in_rows, max_entry_size_in_rows, key.queryStringFromAst());
+        case Type::Result:
+        {
+            /// Normal query result chunks are simply buffered. They are squashed and compressed later in finalizeWrite().
+            query_result->chunks.emplace_back(std::move(chunk));
+            break;
+        }
+        case Type::Totals:
+        case Type::Extremes:
+        {
+            /// For simplicity, totals and extremes chunks are immediately squashed (totals/extremes are obscure and even if enabled, few
+            /// such chunks are expected).
+            auto & buffered_chunk = (type == Type::Totals) ? query_result->totals : query_result->extremes;
+
+            convertToFullIfSparse(chunk);
+
+            if (!buffered_chunk.has_value())
+                buffered_chunk = std::move(chunk);
+            else
+                buffered_chunk->append(chunk);
+
+            break;
+        }
     }
 }
 
@@ -222,6 +247,8 @@ void QueryCache::Writer::finalizeWrite()
 
     chassert(!was_finalized);
 
+    /// Check some reasons why the entry must not be cached:
+
     if (std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - query_start_time) < min_query_runtime)
     {
         LOG_TRACE(&Poco::Logger::get("QueryCache"), "Skipped insert (query not expensive enough), query: {}", key.queryStringFromAst());
@@ -244,7 +271,7 @@ void QueryCache::Writer::finalizeWrite()
         Chunks squashed_chunks;
         size_t rows_remaining_in_squashed = 0; /// how many further rows can the last squashed chunk consume until it reaches max_block_size
 
-        for (auto & chunk : *query_result)
+        for (auto & chunk : query_result->chunks)
         {
             convertToFullIfSparse(chunk);
 
@@ -272,26 +299,49 @@ void QueryCache::Writer::finalizeWrite()
             }
         }
 
-        *query_result = std::move(squashed_chunks);
+        query_result->chunks = std::move(squashed_chunks);
     }
 
     if (key.is_compressed)
     {
+        /// Compress result chunks. Reduces the space consumption of the cache but means reading from it will be slower due to decompression.
+
         Chunks compressed_chunks;
-        const Chunks & decompressed_chunks = *query_result;
-        for (const auto & decompressed_chunk : decompressed_chunks)
+
+        for (const auto & chunk : query_result->chunks)
         {
-            const Columns & decompressed_columns = decompressed_chunk.getColumns();
+            const Columns & columns = chunk.getColumns();
             Columns compressed_columns;
-            for (const auto & decompressed_column : decompressed_columns)
+            for (const auto & column : columns)
             {
-                auto compressed_column = decompressed_column->compress();
+                auto compressed_column = column->compress();
                 compressed_columns.push_back(compressed_column);
             }
-            Chunk compressed_chunk(compressed_columns, decompressed_chunk.getNumRows());
+            Chunk compressed_chunk(compressed_columns, chunk.getNumRows());
             compressed_chunks.push_back(std::move(compressed_chunk));
         }
-        *query_result = std::move(compressed_chunks);
+        query_result->chunks = std::move(compressed_chunks);
+    }
+
+    /// Check more reasons why the entry must not be cached.
+
+    auto count_rows_in_chunks = [](const Entry & entry)
+    {
+        size_t res = 0;
+        for (const auto & chunk : entry.chunks)
+            res += chunk.getNumRows();
+        res += entry.totals.has_value() ? entry.totals->getNumRows() : 0;
+        res += entry.extremes.has_value() ? entry.extremes->getNumRows() : 0;
+        return res;
+    };
+
+    size_t new_entry_size_in_bytes = QueryCacheEntryWeight()(*query_result);
+    size_t new_entry_size_in_rows = count_rows_in_chunks(*query_result);
+
+    if ((new_entry_size_in_bytes > max_entry_size_in_bytes) || (new_entry_size_in_rows > max_entry_size_in_rows))
+    {
+        LOG_TRACE(&Poco::Logger::get("QueryCache"), "Skipped insert (query result too big), new_entry_size_in_bytes: {} ({}), new_entry_size_in_rows: {} ({}), query: {}", new_entry_size_in_bytes, max_entry_size_in_bytes, new_entry_size_in_rows, max_entry_size_in_rows, key.queryStringFromAst());
+        return;
     }
 
     cache.set(key, query_result);
@@ -299,6 +349,21 @@ void QueryCache::Writer::finalizeWrite()
     was_finalized = true;
 }
 
+/// Constructs a pipe with just one processor: SourceFromChunks which serves data from the query cache
+void QueryCache::Reader::buildPipe(Block header, Chunks && chunks, const std::optional<Chunk> & totals, const std::optional<Chunk> & extremes)
+{
+    auto source_from_chunks = std::make_shared<SourceFromChunks>(
+            header, std::move(chunks),
+            totals.has_value() ? std::optional(totals->clone()) : std::nullopt,
+            extremes.has_value() ? std::optional(extremes->clone()) : std::nullopt);
+
+    OutputPort * out = &source_from_chunks->getPort();
+    OutputPort * out_totals = source_from_chunks->getTotalsPort();
+    OutputPort * out_extremes = source_from_chunks->getExtremesPort();
+
+    pipe = Pipe(source_from_chunks, out, out_totals, out_extremes);
+}
+
 QueryCache::Reader::Reader(Cache & cache_, const Key & key, const std::lock_guard<std::mutex> &)
 {
     auto entry = cache_.getWithKey(key);
@@ -322,25 +387,37 @@ QueryCache::Reader::Reader(Cache & cache_, const Key & key, const std::lock_guar
     }
 
     if (!entry->key.is_compressed)
-        pipe = Pipe(std::make_shared<SourceFromChunks>(entry->key.header, entry->mapped));
+    {
+        // Cloning chunks isn't exactly great. It could be avoided by another indirection, i.e. wrapping Entry's members chunks, totals and
+        // extremes into shared_ptrs and assuming that the lifecycle of these shared_ptrs coincides with the lifecycle of the Entry
+        // shared_ptr. This is not done 1. to keep things simple 2. this case (uncompressed chunks) is the exceptional case, in the other
+        // case (the default case aka. compressed chunks) we need to decompress the entry anyways and couldn't apply the potential
+        // optimization.
+
+        Chunks cloned_chunks;
+        for (const auto & chunk : entry->mapped->chunks)
+            cloned_chunks.push_back(chunk.clone());
+
+        buildPipe(entry->key.header, std::move(cloned_chunks), entry->mapped->totals, entry->mapped->extremes);
+    }
     else
     {
-        auto decompressed_chunks = std::make_shared<Chunks>();
-        const Chunks & compressed_chunks = *entry->mapped;
-        for (const auto & compressed_chunk : compressed_chunks)
+        Chunks decompressed_chunks;
+        const Chunks & chunks = entry->mapped->chunks;
+        for (const auto & chunk : chunks)
         {
-            const Columns & compressed_chunk_columns = compressed_chunk.getColumns();
+            const Columns & columns = chunk.getColumns();
             Columns decompressed_columns;
-            for (const auto & compressed_column : compressed_chunk_columns)
+            for (const auto & column : columns)
             {
-                auto column = compressed_column->decompress();
-                decompressed_columns.push_back(column);
+                auto decompressed_column = column->decompress();
+                decompressed_columns.push_back(decompressed_column);
             }
-            Chunk decompressed_chunk(decompressed_columns, compressed_chunk.getNumRows());
-            decompressed_chunks->push_back(std::move(decompressed_chunk));
+            Chunk decompressed_chunk(decompressed_columns, chunk.getNumRows());
+            decompressed_chunks.push_back(std::move(decompressed_chunk));
         }
 
-        pipe = Pipe(std::make_shared<SourceFromChunks>(entry->key.header, decompressed_chunks));
+        buildPipe(entry->key.header, std::move(decompressed_chunks), entry->mapped->totals, entry->mapped->extremes);
     }
 
     LOG_TRACE(&Poco::Logger::get("QueryCache"), "Entry found for query {}", key.queryStringFromAst());
@@ -406,7 +483,7 @@ std::vector<QueryCache::Cache::KeyMapped> QueryCache::dump() const
 }
 
 QueryCache::QueryCache()
-    : cache(std::make_unique<TTLCachePolicy<Key, Chunks, KeyHasher, QueryResultWeight, IsStale>>(std::make_unique<PerUserTTLCachePolicyUserQuota>()))
+    : cache(std::make_unique<TTLCachePolicy<Key, Entry, KeyHasher, QueryCacheEntryWeight, IsStale>>(std::make_unique<PerUserTTLCachePolicyUserQuota>()))
 {
 }
 
diff --git a/src/Interpreters/Cache/QueryCache.h b/src/Interpreters/Cache/QueryCache.h
index 64753974f1c..f614ca248d4 100644
--- a/src/Interpreters/Cache/QueryCache.h
+++ b/src/Interpreters/Cache/QueryCache.h
@@ -53,7 +53,8 @@ public:
         /// When does the entry expire?
         const std::chrono::time_point<std::chrono::system_clock> expires_at;
 
-        /// Is the entry compressed?
+        /// Are the chunks in the entry compressed?
+        /// (we could theoretically apply compression also to the totals and extremes but it's an obscure use case)
         const bool is_compressed;
 
         Key(ASTPtr ast_,
@@ -66,15 +67,22 @@ public:
         String queryStringFromAst() const;
     };
 
+    struct Entry
+    {
+        Chunks chunks;
+        std::optional<Chunk> totals = std::nullopt;
+        std::optional<Chunk> extremes = std::nullopt;
+    };
+
 private:
     struct KeyHasher
     {
         size_t operator()(const Key & key) const;
     };
 
-    struct QueryResultWeight
+    struct QueryCacheEntryWeight
     {
-        size_t operator()(const Chunks & chunks) const;
+        size_t operator()(const Entry & entry) const;
     };
 
     struct IsStale
@@ -83,7 +91,7 @@ private:
     };
 
     /// query --> query result
-    using Cache = CacheBase<Key, Chunks, KeyHasher, QueryResultWeight>;
+    using Cache = CacheBase<Key, Entry, KeyHasher, QueryCacheEntryWeight>;
 
     /// query --> query execution count
     using TimesExecuted = std::unordered_map<Key, size_t, KeyHasher>;
@@ -103,21 +111,24 @@ public:
     class Writer
     {
     public:
-        void buffer(Chunk && partial_query_result);
+
+        Writer(const Writer & other);
+
+        enum class Type {Result, Totals, Extremes};
+        void buffer(Chunk && chunk, Type type);
+
         void finalizeWrite();
     private:
         std::mutex mutex;
         Cache & cache;
         const Key key;
-        size_t new_entry_size_in_bytes TSA_GUARDED_BY(mutex) = 0;
         const size_t max_entry_size_in_bytes;
-        size_t new_entry_size_in_rows TSA_GUARDED_BY(mutex) = 0;
         const size_t max_entry_size_in_rows;
         const std::chrono::time_point<std::chrono::system_clock> query_start_time = std::chrono::system_clock::now(); /// Writer construction and finalizeWrite() coincide with query start/end
         const std::chrono::milliseconds min_query_runtime;
         const bool squash_partial_results;
         const size_t max_block_size;
-        std::shared_ptr<Chunks> query_result TSA_GUARDED_BY(mutex) = std::make_shared<Chunks>();
+        Cache::MappedPtr query_result TSA_GUARDED_BY(mutex) = std::make_shared<Entry>();
         std::atomic<bool> skip_insert = false;
         bool was_finalized = false;
 
@@ -138,6 +149,7 @@ public:
         Pipe && getPipe(); /// must be called only if hasCacheEntryForKey() returns true
     private:
         Reader(Cache & cache_, const Key & key, const std::lock_guard<std::mutex> &);
+        void buildPipe(Block header, Chunks && chunks, const std::optional<Chunk> & totals, const std::optional<Chunk> & extremes);
         Pipe pipe;
         friend class QueryCache; /// for createReader()
     };
diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index 00a5d0ed1d8..b94287d1d69 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -755,15 +755,14 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
                     const size_t num_query_runs = query_cache->recordQueryRun(key);
                     if (num_query_runs > settings.query_cache_min_query_runs)
                     {
-                        auto stream_in_query_cache_transform =
-                                    std::make_shared<StreamInQueryCacheTransform>(
-                                        res.pipeline.getHeader(), query_cache, key,
-                                        std::chrono::milliseconds(context->getSettings().query_cache_min_query_duration.totalMilliseconds()),
-                                        context->getSettings().query_cache_squash_partial_results,
-                                        context->getSettings().max_block_size,
-                                        context->getSettings().query_cache_max_size_in_bytes,
-                                        context->getSettings().query_cache_max_entries);
-                        res.pipeline.streamIntoQueryCache(stream_in_query_cache_transform);
+                        auto query_cache_writer = std::make_shared<QueryCache::Writer>(query_cache->createWriter(
+                                         key,
+                                         std::chrono::milliseconds(settings.query_cache_min_query_duration.totalMilliseconds()),
+                                         settings.query_cache_squash_partial_results,
+                                         settings.max_block_size,
+                                         settings.query_cache_max_size_in_bytes,
+                                         settings.query_cache_max_entries));
+                        res.pipeline.streamIntoQueryCache(query_cache_writer);
                     }
                 }
 
diff --git a/src/Processors/ISource.cpp b/src/Processors/ISource.cpp
index e6aeffa4807..6a88d3973a1 100644
--- a/src/Processors/ISource.cpp
+++ b/src/Processors/ISource.cpp
@@ -124,7 +124,7 @@ std::optional<Chunk> ISource::tryGenerate()
 {
     auto chunk = generate();
     if (!chunk)
-        return {};
+        return std::nullopt;
 
     return chunk;
 }
diff --git a/src/Processors/Sources/SourceFromChunks.cpp b/src/Processors/Sources/SourceFromChunks.cpp
index 7b73b877d2e..54a6e4b54e2 100644
--- a/src/Processors/Sources/SourceFromChunks.cpp
+++ b/src/Processors/Sources/SourceFromChunks.cpp
@@ -3,20 +3,61 @@
 namespace DB
 {
 
-SourceFromChunks::SourceFromChunks(Block header, Chunks chunks_)
-    : SourceFromChunks(header, std::make_shared<Chunks>(std::move(chunks_)), true)
-{}
-
-SourceFromChunks::SourceFromChunks(Block header, std::shared_ptr<Chunks> chunks_)
-    : SourceFromChunks(header, chunks_, false)
-{}
-
-SourceFromChunks::SourceFromChunks(Block header, std::shared_ptr<Chunks> chunks_, bool move_from_chunks_)
-    : ISource(std::move(header))
-    , chunks(chunks_)
-    , it(chunks->begin())
-    , move_from_chunks(move_from_chunks_)
+SourceFromChunks::SourceFromChunks(Block header, Chunks && chunks_, std::optional<Chunk> && chunk_totals_, std::optional<Chunk> && chunk_extremes_)
+    : ISource(header)
+    , chunks(std::move(chunks_))
+    , it(chunks.begin())
 {
+    if (chunk_totals_ != std::nullopt)
+    {
+        outputs.emplace_back(header, this);
+        chunk_totals = std::move(chunk_totals_);
+        output_totals = &outputs.back();
+    }
+
+    if (chunk_extremes_ != std::nullopt)
+    {
+        outputs.emplace_back(header, this);
+        chunk_extremes = std::move(chunk_extremes_);
+        output_extremes = &outputs.back();
+    }
+}
+
+SourceFromChunks::Status SourceFromChunks::prepare()
+{
+    if (!finished_chunks)
+    {
+        Status status = ISource::prepare();
+
+        if (status != Status::Finished)
+            return status;
+
+        finished_chunks = true;
+    }
+
+    if (getTotalsPort())
+    {
+        /// This logic force-pushes the data into the port instead of checking the port status first (isFinished(), canPush()). Seems to
+        /// work but should be improved (TODO).
+        chassert(chunk_totals.has_value());
+        output_totals->push(std::move(*chunk_totals));
+        output_totals->finish();
+    }
+
+    if (getExtremesPort())
+    {
+        chassert(chunk_extremes.has_value());
+        output_extremes->push(std::move(*chunk_extremes));
+        output_extremes->finish();
+    }
+
+    return Status::Finished;
+}
+
+void SourceFromChunks::work()
+{
+    if (!finished_chunks)
+        ISource::work();
 }
 
 String SourceFromChunks::getName() const
@@ -26,19 +67,12 @@ String SourceFromChunks::getName() const
 
 Chunk SourceFromChunks::generate()
 {
-    if (it != chunks->end())
-        if (move_from_chunks)
-        {
-            Chunk && chunk = std::move(*it);
-            it++;
-            return chunk;
-        }
-        else
-        {
-            Chunk chunk = it->clone();
-            it++;
-            return chunk;
-        }
+    if (it != chunks.end())
+    {
+        Chunk && chunk = std::move(*it);
+        it++;
+        return chunk;
+    }
     else
         return {};
 }
diff --git a/src/Processors/Sources/SourceFromChunks.h b/src/Processors/Sources/SourceFromChunks.h
index d41999208a0..58616a8058c 100644
--- a/src/Processors/Sources/SourceFromChunks.h
+++ b/src/Processors/Sources/SourceFromChunks.h
@@ -7,24 +7,33 @@
 namespace DB
 {
 
+/// The big brother of SourceFromSingleChunk. Supports multiple chunks and totals/extremes.
 class SourceFromChunks : public ISource
 {
 public:
-    SourceFromChunks(Block header, Chunks chunks_);
-    SourceFromChunks(Block header, std::shared_ptr<Chunks> chunks_);
+    SourceFromChunks(Block header, Chunks && chunks_, std::optional<Chunk> && chunk_totals_, std::optional<Chunk> && chunk_extremes_);
 
     String getName() const override;
 
-protected:
+    Status prepare() override;
+    void work() override;
+
+    OutputPort * getTotalsPort() const { return output_totals; }
+    OutputPort * getExtremesPort() const { return output_extremes; }
+
     Chunk generate() override;
 
 private:
-    SourceFromChunks(Block header, std::shared_ptr<Chunks> chunks_, bool move_from_chunks_);
-
-    const std::shared_ptr<Chunks> chunks;
+    Chunks chunks;
     Chunks::iterator it;
-    /// Optimization: if the chunks are exclusively owned by SourceFromChunks, then generate() can move from them
-    const bool move_from_chunks;
+
+    std::optional<Chunk> chunk_totals = std::nullopt;
+    std::optional<Chunk> chunk_extremes = std::nullopt;
+
+    OutputPort * output_totals = nullptr;
+    OutputPort * output_extremes = nullptr;
+
+    bool finished_chunks = false;
 };
 
 }
diff --git a/src/Processors/Transforms/StreamInQueryCacheTransform.cpp b/src/Processors/Transforms/StreamInQueryCacheTransform.cpp
index 34dec20ac8e..c4210d7afa5 100644
--- a/src/Processors/Transforms/StreamInQueryCacheTransform.cpp
+++ b/src/Processors/Transforms/StreamInQueryCacheTransform.cpp
@@ -5,26 +5,23 @@ namespace DB
 
 StreamInQueryCacheTransform::StreamInQueryCacheTransform(
     const Block & header_,
-    QueryCachePtr cache,
-    const QueryCache::Key & cache_key,
-    std::chrono::milliseconds min_query_duration,
-    bool squash_partial_results,
-    size_t max_block_size,
-    size_t max_query_cache_size_in_bytes_quota, size_t max_query_cache_entries_quota)
+    std::shared_ptr<QueryCache::Writer> query_cache_writer_,
+    QueryCache::Writer::Type type_)
     : ISimpleTransform(header_, header_, false)
-    , cache_writer(cache->createWriter(cache_key, min_query_duration, squash_partial_results, max_block_size, max_query_cache_size_in_bytes_quota, max_query_cache_entries_quota))
+    , query_cache_writer(query_cache_writer_)
+    , type(type_)
 {
 }
 
 void StreamInQueryCacheTransform::transform(Chunk & chunk)
 {
-    cache_writer.buffer(chunk.clone());
+    query_cache_writer->buffer(chunk.clone(), type);
 }
 
 void StreamInQueryCacheTransform::finalizeWriteInQueryCache()
 {
     if (!isCancelled())
-        cache_writer.finalizeWrite();
+        query_cache_writer->finalizeWrite();
 }
 
 };
diff --git a/src/Processors/Transforms/StreamInQueryCacheTransform.h b/src/Processors/Transforms/StreamInQueryCacheTransform.h
index 2989ff5d1a5..e26f343290d 100644
--- a/src/Processors/Transforms/StreamInQueryCacheTransform.h
+++ b/src/Processors/Transforms/StreamInQueryCacheTransform.h
@@ -11,12 +11,8 @@ class StreamInQueryCacheTransform : public ISimpleTransform
 public:
     StreamInQueryCacheTransform(
         const Block & header_,
-        QueryCachePtr cache,
-        const QueryCache::Key & cache_key,
-        std::chrono::milliseconds min_query_duration,
-        bool squash_partial_results,
-        size_t max_block_size,
-        size_t max_query_cache_size_in_bytes_quota, size_t max_query_cache_entries_quota);
+        std::shared_ptr<QueryCache::Writer> query_cache_writer,
+        QueryCache::Writer::Type type);
 
 protected:
     void transform(Chunk & chunk) override;
@@ -26,7 +22,8 @@ public:
     String getName() const override { return "StreamInQueryCacheTransform"; }
 
 private:
-    QueryCache::Writer cache_writer;
+    const std::shared_ptr<QueryCache::Writer> query_cache_writer;
+    const QueryCache::Writer::Type type;
 };
 
 }
diff --git a/src/QueryPipeline/QueryPipeline.cpp b/src/QueryPipeline/QueryPipeline.cpp
index f060f2f508f..1149cef0094 100644
--- a/src/QueryPipeline/QueryPipeline.cpp
+++ b/src/QueryPipeline/QueryPipeline.cpp
@@ -1,3 +1,5 @@
+#include <QueryPipeline/QueryPipeline.h>
+
 #include <queue>
 #include <QueryPipeline/Chain.h>
 #include <Processors/Formats/IOutputFormat.h>
@@ -7,7 +9,6 @@
 #include <Interpreters/ExpressionActions.h>
 #include <QueryPipeline/ReadProgressCallback.h>
 #include <QueryPipeline/Pipe.h>
-#include <QueryPipeline/QueryPipeline.h>
 #include <Processors/Sinks/EmptySink.h>
 #include <Processors/Sinks/NullSink.h>
 #include <Processors/Sinks/SinkToStorage.h>
@@ -576,13 +577,31 @@ bool QueryPipeline::tryGetResultRowsAndBytes(UInt64 & result_rows, UInt64 & resu
     return true;
 }
 
-void QueryPipeline::streamIntoQueryCache(std::shared_ptr<StreamInQueryCacheTransform> transform)
+void QueryPipeline::streamIntoQueryCache(std::shared_ptr<QueryCache::Writer> query_cache_writer)
 {
     assert(pulling());
 
-    connect(*output, transform->getInputPort());
-    output = &transform->getOutputPort();
-    processors->emplace_back(transform);
+    /// Attach a special transform to all output ports (result + possibly totals/extremes). The only purpose of the transform is
+    /// to write each chunk into the query cache. All transforms hold a refcounted reference to the same query cache writer object.
+    /// This ensures that all transforms write to the single same cache entry. The writer object synchronizes internally, the
+    /// expensive stuff like cloning chunks happens outside lock scopes).
+
+    auto add_stream_in_query_cache_transform = [&](OutputPort *& out_port, QueryCache::Writer::Type type)
+    {
+        if (!out_port)
+            return;
+
+        auto transform = std::make_shared<StreamInQueryCacheTransform>(out_port->getHeader(), query_cache_writer, type);
+        connect(*out_port, transform->getInputPort());
+        out_port = &transform->getOutputPort();
+        processors->emplace_back(std::move(transform));
+    };
+
+    using enum QueryCache::Writer::Type;
+
+    add_stream_in_query_cache_transform(output, Result);
+    add_stream_in_query_cache_transform(totals, Totals);
+    add_stream_in_query_cache_transform(extremes, Extremes);
 }
 
 void QueryPipeline::finalizeWriteInQueryCache()
@@ -591,8 +610,8 @@ void QueryPipeline::finalizeWriteInQueryCache()
         processors->begin(), processors->end(),
         [](ProcessorPtr processor){ return dynamic_cast<StreamInQueryCacheTransform *>(&*processor); });
 
-    /// the pipeline should theoretically contain just one StreamInQueryCacheTransform
-
+    /// The pipeline can contain up to three StreamInQueryCacheTransforms which all point to the same query cache writer object.
+    /// We can call finalize() on any of them.
     if (it != processors->end())
         dynamic_cast<StreamInQueryCacheTransform &>(**it).finalizeWriteInQueryCache();
 }
diff --git a/src/QueryPipeline/QueryPipeline.h b/src/QueryPipeline/QueryPipeline.h
index 55c78ca78ed..297178715fd 100644
--- a/src/QueryPipeline/QueryPipeline.h
+++ b/src/QueryPipeline/QueryPipeline.h
@@ -2,6 +2,7 @@
 #include <QueryPipeline/QueryPlanResourceHolder.h>
 #include <QueryPipeline/SizeLimits.h>
 #include <QueryPipeline/StreamLocalLimits.h>
+#include <Interpreters/Cache/QueryCache.h> /// nested classes such as QC::Writer can't be fwd declared
 #include <functional>
 
 namespace DB
@@ -31,7 +32,6 @@ class SinkToStorage;
 class ISource;
 class ISink;
 class ReadProgressCallback;
-class StreamInQueryCacheTransform;
 
 struct ColumnWithTypeAndName;
 using ColumnsWithTypeAndName = std::vector<ColumnWithTypeAndName>;
@@ -105,7 +105,7 @@ public:
     void setLimitsAndQuota(const StreamLocalLimits & limits, std::shared_ptr<const EnabledQuota> quota_);
     bool tryGetResultRowsAndBytes(UInt64 & result_rows, UInt64 & result_bytes) const;
 
-    void streamIntoQueryCache(std::shared_ptr<StreamInQueryCacheTransform> transform);
+    void streamIntoQueryCache(std::shared_ptr<QueryCache::Writer> query_cache_writer);
     void finalizeWriteInQueryCache();
 
     void setQuota(std::shared_ptr<const EnabledQuota> quota_);
diff --git a/src/Storages/System/StorageSystemQueryCache.cpp b/src/Storages/System/StorageSystemQueryCache.cpp
index 0e589b1b183..245f4b7fd26 100644
--- a/src/Storages/System/StorageSystemQueryCache.cpp
+++ b/src/Storages/System/StorageSystemQueryCache.cpp
@@ -45,7 +45,7 @@ void StorageSystemQueryCache::fillData(MutableColumns & res_columns, ContextPtr
             continue;
 
         res_columns[0]->insert(key.queryStringFromAst()); /// approximates the original query string
-        res_columns[1]->insert(QueryCache::QueryResultWeight()(*query_result));
+        res_columns[1]->insert(QueryCache::QueryCacheEntryWeight()(*query_result));
         res_columns[2]->insert(key.expires_at < std::chrono::system_clock::now());
         res_columns[3]->insert(!key.is_shared);
         res_columns[4]->insert(key.is_compressed);
diff --git a/tests/queries/0_stateless/02494_query_cache_totals_extremes.reference b/tests/queries/0_stateless/02494_query_cache_totals_extremes.reference
new file mode 100644
index 00000000000..2f6e5e7bd87
--- /dev/null
+++ b/tests/queries/0_stateless/02494_query_cache_totals_extremes.reference
@@ -0,0 +1,43 @@
+1st run:
+1	8
+2	2
+
+0	10
+2nd run:
+1	8
+2	2
+
+0	10
+1
+---
+1st run:
+1	8
+2	2
+
+1	2
+2	8
+2nd run:
+1	8
+2	2
+
+1	2
+2	8
+1
+---
+1st run:
+1	8
+2	2
+
+0	10
+
+1	2
+2	8
+2nd run:
+1	8
+2	2
+
+0	10
+
+1	2
+2	8
+1
diff --git a/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql b/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql
new file mode 100644
index 00000000000..d49609885b1
--- /dev/null
+++ b/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql
@@ -0,0 +1,48 @@
+-- Tags: no-parallel
+-- Tag no-parallel: Messes with internal cache
+
+SET allow_experimental_query_cache = true;
+
+SYSTEM DROP QUERY CACHE;
+DROP TABLE IF EXISTS tbl;
+
+CREATE TABLE tbl (key UInt64, agg UInt64) ENGINE = MergeTree ORDER BY key;
+INSERT INTO tbl VALUES (1, 3), (2, 2), (1, 4), (1, 1);
+
+-- A query with totals calculation. The result should written into / read from the query cache.
+-- Check that both queries produce the same result and that a query cache entry exists.
+SELECT '1st run:';
+SELECT key, sum(agg) FROM tbl GROUP BY key WITH totals ORDER BY key SETTINGS use_query_cache = 1;
+SELECT '2nd run:';
+SELECT key, sum(agg) FROM tbl GROUP BY key WITH totals ORDER BY key SETTINGS use_query_cache = 1;
+
+SELECT count(*) FROM system.query_cache;
+
+SELECT '---';
+
+SYSTEM DROP QUERY CACHE;
+
+-- A query with extremes calculation. The result should written into / read from the query cache.
+-- Check that both queries produce the same result.
+SELECT '1st run:';
+SELECT key, sum(agg) FROM tbl GROUP BY key ORDER BY key SETTINGS use_query_cache = 1, extremes = 1;
+SELECT '2nd run:';
+SELECT key, sum(agg) FROM tbl GROUP BY key ORDER BY key SETTINGS use_query_cache = 1, extremes = 1;
+
+SELECT count(*) FROM system.query_cache;
+
+SELECT '---';
+
+SYSTEM DROP QUERY CACHE;
+
+-- A query with totals and extremes calculation. The result should written into / read from the query cache.
+-- Check that both queries produce the same result.
+SELECT '1st run:';
+SELECT key, sum(agg) FROM tbl GROUP BY key WITH totals ORDER BY key SETTINGS use_query_cache = 1, extremes = 1;
+SELECT '2nd run:';
+SELECT key, sum(agg) FROM tbl GROUP BY key WITH totals ORDER BY key SETTINGS use_query_cache = 1, extremes = 1;
+
+SELECT count(*) FROM system.query_cache;
+DROP TABLE IF EXISTS tbl;
+
+SYSTEM DROP QUERY CACHE;

From 1da6ea7988785db0b800b68d0b6dcc86adc4ea7a Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 25 Apr 2023 11:41:33 +0000
Subject: [PATCH 079/689] Move SHOW PROCESSLIST to a better position

---
 docs/en/sql-reference/statements/show.md | 32 ++++++++++++------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md
index 428a04ae030..6179d053b0f 100644
--- a/docs/en/sql-reference/statements/show.md
+++ b/docs/en/sql-reference/statements/show.md
@@ -98,22 +98,6 @@ Result:
 
 - [CREATE DATABASE](https://clickhouse.com/docs/en/sql-reference/statements/create/database/#query-language-create-database)
 
-## SHOW PROCESSLIST
-
-``` sql
-SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format]
-```
-
-Outputs the content of the [system.processes](../../operations/system-tables/processes.md#system_tables-processes) table, that contains a list of queries that is being processed at the moment, excepting `SHOW PROCESSLIST` queries.
-
-The `SELECT * FROM system.processes` query returns data about all the current queries.
-
-Tip (execute in the console):
-
-``` bash
-$ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'"
-```
-
 ## SHOW TABLES
 
 Displays a list of tables.
@@ -277,6 +261,22 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2
 └──────────────┘
 ```
 
+## SHOW PROCESSLIST
+
+``` sql
+SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format]
+```
+
+Outputs the content of the [system.processes](../../operations/system-tables/processes.md#system_tables-processes) table, that contains a list of queries that is being processed at the moment, excepting `SHOW PROCESSLIST` queries.
+
+The `SELECT * FROM system.processes` query returns data about all the current queries.
+
+Tip (execute in the console):
+
+``` bash
+$ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'"
+```
+
 ## SHOW GRANTS
 
 Shows privileges for a user.

From 0574f2b497e2eb2197513de27448e29aff3cdf53 Mon Sep 17 00:00:00 2001
From: Manas Alekar <manas@clickhouse.com>
Date: Tue, 25 Apr 2023 09:01:17 -0700
Subject: [PATCH 080/689] Fix style.

---
 src/Client/ClientBase.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index fc087c9c040..9a145f484bc 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -572,7 +572,8 @@ try
                 CompressionMethod compression_method = chooseCompressionMethod(out_file, compression_method_string);
                 UInt64 compression_level = 3;
 
-                if (query_with_output->is_outfile_append && compression_method != CompressionMethod::None) {
+                if (query_with_output->is_outfile_append && compression_method != CompressionMethod::None)
+				{
                     throw Exception(
                         ErrorCodes::BAD_ARGUMENTS,
                         "Cannot append to compressed file. Please use uncompressed file or remove APPEND keyword.");

From ab1646ab36d56b2ac1415dd2cc975e9fc9b4fa26 Mon Sep 17 00:00:00 2001
From: Manas Alekar <manas@clickhouse.com>
Date: Tue, 25 Apr 2023 10:41:56 -0700
Subject: [PATCH 081/689] Expand tab.

---
 src/Client/ClientBase.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 9a145f484bc..eb89f496d23 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -573,7 +573,7 @@ try
                 UInt64 compression_level = 3;
 
                 if (query_with_output->is_outfile_append && compression_method != CompressionMethod::None)
-				{
+                {
                     throw Exception(
                         ErrorCodes::BAD_ARGUMENTS,
                         "Cannot append to compressed file. Please use uncompressed file or remove APPEND keyword.");

From 935f5f580752959b7722ef5e485430c966f5e50d Mon Sep 17 00:00:00 2001
From: Manas Alekar <manas@clickhouse.com>
Date: Tue, 25 Apr 2023 11:05:56 -0700
Subject: [PATCH 082/689] Add documentation for append.

---
 docs/en/sql-reference/statements/select/into-outfile.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/statements/select/into-outfile.md b/docs/en/sql-reference/statements/select/into-outfile.md
index bd6db9e7d55..352af16042a 100644
--- a/docs/en/sql-reference/statements/select/into-outfile.md
+++ b/docs/en/sql-reference/statements/select/into-outfile.md
@@ -12,7 +12,7 @@ Compressed files are supported. Compression type is detected by the extension of
 **Syntax**
 
 ```sql
-SELECT <expr_list> INTO OUTFILE file_name [AND STDOUT] [COMPRESSION type [LEVEL level]]
+SELECT <expr_list> INTO OUTFILE file_name [AND STDOUT] [APPEND] [COMPRESSION type [LEVEL level]]
 ```
 
 `file_name` and `type` are string literals. Supported compression types are: `'none'`, `'gzip'`, `'deflate'`, `'br'`, `'xz'`, `'zstd'`, `'lz4'`, `'bz2'`.
@@ -25,6 +25,7 @@ SELECT <expr_list> INTO OUTFILE file_name [AND STDOUT] [COMPRESSION type [LEVEL
 - The query will fail if a file with the same file name already exists.
 - The default [output format](../../../interfaces/formats.md) is `TabSeparated` (like in the command-line client batch mode). Use [FORMAT](format.md) clause to change it.
 - If `AND STDOUT` is mentioned in the query then the output that is written to the file is also displayed on standard output. If used with compression, the plaintext is displayed on standard output.
+- If `APPEND` is mentioned in the query then the output is appended to an existing file. If compression is used, append cannot be used.
 
 **Example**
 

From c4ab1e12f1deb1044230c278cbc8d55df425da09 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 25 Apr 2023 18:04:19 +0000
Subject: [PATCH 083/689] Initial version of SHOW INDEXES

---
 docs/en/sql-reference/statements/show.md      |  51 +++++++
 src/Interpreters/InterpreterFactory.cpp       |   6 +
 .../InterpreterShowIndexesQuery.cpp           | 124 ++++++++++++++++++
 .../InterpreterShowIndexesQuery.h             |  33 +++++
 src/Parsers/ASTShowIndexesQuery.cpp           |  38 ++++++
 src/Parsers/ASTShowIndexesQuery.h             |  29 ++++
 src/Parsers/ParserQueryWithOutput.cpp         |   3 +
 src/Parsers/ParserShowColumnsQuery.cpp        |   2 +-
 src/Parsers/ParserShowIndexesQuery.cpp        |  58 ++++++++
 src/Parsers/ParserShowIndexesQuery.h          |  20 +++
 .../0_stateless/02724_show_indexes.reference  |  37 ++++++
 .../0_stateless/02724_show_indexes.sql        |  59 +++++++++
 12 files changed, 459 insertions(+), 1 deletion(-)
 create mode 100644 src/Interpreters/InterpreterShowIndexesQuery.cpp
 create mode 100644 src/Interpreters/InterpreterShowIndexesQuery.h
 create mode 100644 src/Parsers/ASTShowIndexesQuery.cpp
 create mode 100644 src/Parsers/ASTShowIndexesQuery.h
 create mode 100644 src/Parsers/ParserShowIndexesQuery.cpp
 create mode 100644 src/Parsers/ParserShowIndexesQuery.h
 create mode 100644 tests/queries/0_stateless/02724_show_indexes.reference
 create mode 100644 tests/queries/0_stateless/02724_show_indexes.sql

diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md
index 6179d053b0f..db4d55305ce 100644
--- a/docs/en/sql-reference/statements/show.md
+++ b/docs/en/sql-reference/statements/show.md
@@ -261,6 +261,57 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2
 └──────────────┘
 ```
 
+## SHOW INDEX
+
+Displays a list of primary and data skipping indexes of a table.
+
+```sql
+SHOW [EXTENDED] {INDEX | INDEXES |KEYS } {FROM | IN} <table> [{FROM | IN} <db>] [WHERE <expr>}] [INTO OUTFILE <filename>] [FORMAT <format>]
+```
+
+The database and table name can be specified in abbreviated form as `<db>.<table>`, i.e. `FROM tab FROM db` and `FROM db.tab` are
+equivalent. If no database is specified, the query returns the list of columns from the current database.
+
+The optional keyword `EXTENDED` currently has no effect, it only exists for MySQL compatibility.
+
+`SHOW INDEX` produces a result table with the following structure:
+- table - The name of the table (String)
+- non_unique - 0 if the index can contain duplicates, 1 otherwise (UInt8)
+- key_name - The name of the index. 'PRIMARY' if the index is a primary key index. (String)
+- seq_in_index - currently unused
+- column_name - currently unused
+- collation - currently unused
+- cardinality - currently unused
+- sub_part - currently unused
+- packed - currently unused
+- null - currently unused
+- index_type - the index type, e.g. `primary`, `minmax`, `bloom_filter` etc. (String)
+- comment - currently unused
+- index_comment - currently unused
+- visible - if the index is visible to the optimizer, always `YES` (String)
+- expression - the index expression (String)
+
+**Examples**
+
+Getting information about all indexes in table 'tbl'
+
+```sql
+SHOW INDEX FROM 'orders'
+```
+
+Result:
+
+``` text
+┌─table─┬─non_unique─┬─key_name─┬─seq_in_index─┬─column_name─┬─collation─┬─cardinality─┬─sub_part─┬─packed─┬─null─┬─index_type─┬─comment─┬─index_comment─┬─visible─┬─expression─┐
+│ tbl   │          0 │ mm1_idx  │ ᴺᵁᴸᴸ         │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ      │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ     │ ᴺᵁᴸᴸ   │ ᴺᵁᴸᴸ │ minmax     │ ᴺᵁᴸᴸ    │ ᴺᵁᴸᴸ          │ YES     │ a, c, d    │
+│ tbl   │          0 │ mm2_idx  │ ᴺᵁᴸᴸ         │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ      │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ     │ ᴺᵁᴸᴸ   │ ᴺᵁᴸᴸ │ minmax     │ ᴺᵁᴸᴸ    │ ᴺᵁᴸᴸ          │ YES     │ c, d, e    │
+└───────┴────────────┴──────────┴──────────────┴─────────────┴───────────┴─────────────┴──────────┴────────┴──────┴────────────┴─────────┴───────────────┴─────────┴────────────┘
+```
+
+**See also**
+
+- [system.columns](https://clickhouse.com/docs/en/operations/system-tables/columns)
+
 ## SHOW PROCESSLIST
 
 ``` sql
diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp
index 68b22b550f6..eec73fa56e2 100644
--- a/src/Interpreters/InterpreterFactory.cpp
+++ b/src/Interpreters/InterpreterFactory.cpp
@@ -22,6 +22,7 @@
 #include <Parsers/ASTShowProcesslistQuery.h>
 #include <Parsers/ASTShowTablesQuery.h>
 #include <Parsers/ASTShowColumnsQuery.h>
+#include <Parsers/ASTShowIndexesQuery.h>
 #include <Parsers/ASTUseQuery.h>
 #include <Parsers/ASTWatchQuery.h>
 #include <Parsers/ASTCreateNamedCollectionQuery.h>
@@ -81,6 +82,7 @@
 #include <Interpreters/InterpreterShowProcesslistQuery.h>
 #include <Interpreters/InterpreterShowTablesQuery.h>
 #include <Interpreters/InterpreterShowColumnsQuery.h>
+#include <Interpreters/InterpreterShowIndexesQuery.h>
 #include <Interpreters/InterpreterSystemQuery.h>
 #include <Interpreters/InterpreterUseQuery.h>
 #include <Interpreters/InterpreterWatchQuery.h>
@@ -181,6 +183,10 @@ std::unique_ptr<IInterpreter> InterpreterFactory::get(ASTPtr & query, ContextMut
     {
         return std::make_unique<InterpreterShowColumnsQuery>(query, context);
     }
+    else if (query->as<ASTShowIndexesQuery>())
+    {
+        return std::make_unique<InterpreterShowIndexesQuery>(query, context);
+    }
     else if (query->as<ASTShowEnginesQuery>())
     {
         return std::make_unique<InterpreterShowEnginesQuery>(query, context);
diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp
new file mode 100644
index 00000000000..cb282b95451
--- /dev/null
+++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp
@@ -0,0 +1,124 @@
+#include <Interpreters/InterpreterShowIndexesQuery.h>
+
+#include <IO/WriteBufferFromString.h>
+#include <Parsers/ASTShowIndexesQuery.h>
+#include <Parsers/formatAST.h>
+#include <Interpreters/Context.h>
+#include <Interpreters/executeQuery.h>
+#include <IO/Operators.h>
+#include <boost/algorithm/string.hpp>
+
+
+namespace DB
+{
+
+
+InterpreterShowIndexesQuery::InterpreterShowIndexesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_)
+    : WithMutableContext(context_)
+    , query_ptr(query_ptr_)
+{
+}
+
+
+String InterpreterShowIndexesQuery::getRewrittenQuery()
+{
+    const auto & query = query_ptr->as<ASTShowIndexesQuery &>();
+
+    String database;
+    String table;
+    if (query.from_table.contains("."))
+    {
+        /// FROM <db>.<table> (abbreviated form)
+        chassert(query.from_database.empty());
+        std::vector<String> split;
+        boost::split(split, query.from_table, boost::is_any_of("."));
+        chassert(split.size() == 2);
+        database = split[0];
+        table = split[1];
+    }
+    else if (query.from_database.empty())
+    {
+        /// FROM <table>
+        chassert(!query.from_table.empty());
+        database = getContext()->getCurrentDatabase();
+        table = query.from_table;
+    }
+    else
+    {
+        /// FROM <database> FROM <table>
+        chassert(!query.from_database.empty());
+        chassert(!query.from_table.empty());
+        database = query.from_database;
+        table = query.from_table;
+    }
+
+    WriteBufferFromOwnString where_expression_buf;
+    if (query.where_expression)
+        where_expression_buf << "WHERE (" << query.where_expression << ")";
+    String where_expression = where_expression_buf.str();
+
+    WriteBufferFromOwnString rewritten_query;
+    rewritten_query     << "SELECT * FROM ("
+                        << "(SELECT "
+                            << "name AS table, "
+                            << "0 AS non_unique, "
+                            << "'PRIMARY' AS key_name, "
+                            << "NULL AS seq_in_index, "
+                            << "NULL AS column_name, "
+                            << "'A' AS collation, "
+                            << "NULL AS cardinality, "
+                            << "NULL AS sub_part, "
+                            << "NULL AS packed, "
+                            << "NULL AS null, "
+                            << "'primary' AS index_type, "
+                            << "NULL AS comment, "
+                            << "NULL AS index_comment, "
+                            << "'YES' AS visible, "
+                            << "primary_key AS expression "
+                        << "FROM system.tables "
+                        << "WHERE "
+                            << "database = '" << database << "' "
+                            << "AND name = '" << table << "'"
+                    << ") UNION ALL ("
+                        << "SELECT "
+                            << "table AS table, "
+                            << "0 AS non_unique, "
+                            << "name AS key_name, "
+                            << "NULL AS seq_in_index, "
+                            << "NULL AS column_name, "
+                            << "NULL AS collation, "
+                            << "NULL AS cardinality, "
+                            << "NULL AS sub_part, "
+                            << "NULL AS packed, "
+                            << "NULL AS null, "
+                            << "type AS index_type, "
+                            << "NULL AS comment, "
+                            << "NULL AS index_comment, "
+                            << "'YES' AS visible, "
+                            << "expr AS expression "
+                        << "FROM system.data_skipping_indices "
+                        << "WHERE "
+                            << "database = '" << database << "' "
+                            << "AND table = '" << table << "'"
+                            << ")) "
+                        << where_expression;
+
+    /// Sorting is strictly speaking not necessary but 1. it is convenient for users, 2. SQL currently does not allow to
+    /// sort the output of SHOW COLUMNS otherwise (SELECT * FROM (SHOW COLUMNS ...) ORDER BY ...) is rejected) and 3. some
+    /// SQL tests can take advantage of this.
+    rewritten_query << " ORDER BY index_type, expression";
+
+    std::cout << rewritten_query.str() << std::endl;
+
+    return rewritten_query.str();
+}
+
+
+BlockIO InterpreterShowIndexesQuery::execute()
+{
+    return executeQuery(getRewrittenQuery(), getContext(), true);
+}
+
+
+}
+
diff --git a/src/Interpreters/InterpreterShowIndexesQuery.h b/src/Interpreters/InterpreterShowIndexesQuery.h
new file mode 100644
index 00000000000..0b4fa591e35
--- /dev/null
+++ b/src/Interpreters/InterpreterShowIndexesQuery.h
@@ -0,0 +1,33 @@
+#pragma once
+
+#include <Interpreters/IInterpreter.h>
+#include <Parsers/IAST_fwd.h>
+
+
+namespace DB
+{
+
+class Context;
+
+
+/// Returns a list of indexes which meet some conditions.
+class InterpreterShowIndexesQuery : public IInterpreter, WithMutableContext
+{
+public:
+    InterpreterShowIndexesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_);
+
+    BlockIO execute() override;
+
+    /// Ignore quota and limits here because execute() produces a SELECT query which checks quotas/limits by itself.
+    bool ignoreQuota() const override { return true; }
+    bool ignoreLimits() const override { return true; }
+
+private:
+    ASTPtr query_ptr;
+
+    String getRewrittenQuery();
+};
+
+
+}
+
diff --git a/src/Parsers/ASTShowIndexesQuery.cpp b/src/Parsers/ASTShowIndexesQuery.cpp
new file mode 100644
index 00000000000..6930043e5ed
--- /dev/null
+++ b/src/Parsers/ASTShowIndexesQuery.cpp
@@ -0,0 +1,38 @@
+#include <Parsers/ASTShowIndexesQuery.h>
+
+#include <iomanip>
+#include <Common/quoteString.h>
+#include <IO/Operators.h>
+
+namespace DB
+{
+
+ASTPtr ASTShowIndexesQuery::clone() const
+{
+    auto res = std::make_shared<ASTShowIndexesQuery>(*this);
+    res->children.clear();
+    cloneOutputOptions(*res);
+    return res;
+}
+
+void ASTShowIndexesQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
+{
+    settings.ostr << (settings.hilite ? hilite_keyword : "")
+                  << "SHOW "
+                  << (extended ? "EXTENDED " : "")
+                  << "INDEXES"
+                  << (settings.hilite ? hilite_none : "");
+
+    settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(from_table);
+    if (!from_database.empty())
+        settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(from_database);
+
+    if (where_expression)
+    {
+        settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : "");
+        where_expression->formatImpl(settings, state, frame);
+    }
+}
+
+}
+
diff --git a/src/Parsers/ASTShowIndexesQuery.h b/src/Parsers/ASTShowIndexesQuery.h
new file mode 100644
index 00000000000..a0d5e0c6b6f
--- /dev/null
+++ b/src/Parsers/ASTShowIndexesQuery.h
@@ -0,0 +1,29 @@
+#pragma once
+
+#include <Parsers/IAST_fwd.h>
+#include <Parsers/ASTQueryWithOutput.h>
+
+namespace DB
+{
+
+/// Query SHOW INDEXES
+class ASTShowIndexesQuery : public ASTQueryWithOutput
+{
+public:
+    bool extended = false;
+
+    ASTPtr where_expression;
+
+    String from_database;
+    String from_table;
+
+    String getID(char) const override { return "ShowColumns"; }
+    ASTPtr clone() const override;
+    QueryKind getQueryKind() const override { return QueryKind::Show; }
+
+protected:
+    void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
+};
+
+}
+
diff --git a/src/Parsers/ParserQueryWithOutput.cpp b/src/Parsers/ParserQueryWithOutput.cpp
index 2fb7c406d74..22e32fa5c45 100644
--- a/src/Parsers/ParserQueryWithOutput.cpp
+++ b/src/Parsers/ParserQueryWithOutput.cpp
@@ -18,6 +18,7 @@
 #include <Parsers/ParserShowTablesQuery.h>
 #include <Parsers/ParserShowColumnsQuery.h>
 #include <Parsers/ParserShowEngineQuery.h>
+#include <Parsers/ParserShowIndexesQuery.h>
 #include <Parsers/ParserTablePropertiesQuery.h>
 #include <Parsers/ParserWatchQuery.h>
 #include <Parsers/ParserDescribeCacheQuery.h>
@@ -39,6 +40,7 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
     ParserShowTablesQuery show_tables_p;
     ParserShowColumnsQuery show_columns_p;
     ParserShowEnginesQuery show_engine_p;
+    ParserShowIndexesQuery show_indexes_p;
     ParserSelectWithUnionQuery select_p;
     ParserTablePropertiesQuery table_p;
     ParserDescribeTableQuery describe_table_p;
@@ -69,6 +71,7 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
         || show_tables_p.parse(pos, query, expected)
         || show_columns_p.parse(pos, query, expected)
         || show_engine_p.parse(pos, query, expected)
+        || show_indexes_p.parse(pos, query, expected)
         || table_p.parse(pos, query, expected)
         || describe_cache_p.parse(pos, query, expected)
         || describe_table_p.parse(pos, query, expected)
diff --git a/src/Parsers/ParserShowColumnsQuery.cpp b/src/Parsers/ParserShowColumnsQuery.cpp
index 03e66e4ae0f..57328bb0d8b 100644
--- a/src/Parsers/ParserShowColumnsQuery.cpp
+++ b/src/Parsers/ParserShowColumnsQuery.cpp
@@ -27,7 +27,7 @@ bool ParserShowColumnsQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
     if (ParserKeyword("FULL").ignore(pos, expected))
         query->full = true;
 
-    if (!ParserKeyword("COLUMNS").ignore(pos, expected) || ParserKeyword("FIELDS").ignore(pos, expected))
+    if (!(ParserKeyword("COLUMNS").ignore(pos, expected) || ParserKeyword("FIELDS").ignore(pos, expected)))
         return false;
 
     if (ParserKeyword("FROM").ignore(pos, expected) || ParserKeyword("IN").ignore(pos, expected))
diff --git a/src/Parsers/ParserShowIndexesQuery.cpp b/src/Parsers/ParserShowIndexesQuery.cpp
new file mode 100644
index 00000000000..4b7eba8636e
--- /dev/null
+++ b/src/Parsers/ParserShowIndexesQuery.cpp
@@ -0,0 +1,58 @@
+#include <Parsers/ParserShowIndexesQuery.h>
+
+// TODO
+#include <Parsers/ASTIdentifier_fwd.h>
+#include <Parsers/ASTLiteral.h>
+#include <Parsers/ASTShowIndexesQuery.h>
+#include <Parsers/CommonParsers.h>
+#include <Parsers/ExpressionElementParsers.h>
+#include <Parsers/ExpressionListParsers.h>
+
+namespace DB
+{
+
+bool ParserShowIndexesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    ASTPtr from_database;
+    ASTPtr from_table;
+
+    auto query = std::make_shared<ASTShowIndexesQuery>();
+
+    if (!ParserKeyword("SHOW").ignore(pos, expected))
+        return false;
+
+    if (ParserKeyword("EXTENDED").ignore(pos, expected))
+        query->extended = true;
+
+    if (!(ParserKeyword("INDEX").ignore(pos, expected) || ParserKeyword("INDEXES").ignore(pos, expected) || ParserKeyword("KEYS").ignore(pos, expected)))
+        return false;
+
+    if (ParserKeyword("FROM").ignore(pos, expected) || ParserKeyword("IN").ignore(pos, expected))
+    {
+        if (!ParserCompoundIdentifier().parse(pos, from_table, expected))
+            return false;
+    }
+    else
+        return false;
+
+    tryGetIdentifierNameInto(from_table, query->from_table);
+    bool abbreviated_form = query->from_table.contains("."); /// FROM <db>.<table>
+
+    if (!abbreviated_form)
+        if (ParserKeyword("FROM").ignore(pos, expected) || ParserKeyword("IN").ignore(pos, expected))
+            if (!ParserIdentifier().parse(pos, from_database, expected))
+                return false;
+
+    tryGetIdentifierNameInto(from_database, query->from_database);
+
+    if (ParserKeyword("WHERE").ignore(pos, expected))
+        if (!ParserExpressionWithOptionalAlias(false).parse(pos, query->where_expression, expected))
+            return false;
+
+    node = query;
+
+    return true;
+}
+
+}
+
diff --git a/src/Parsers/ParserShowIndexesQuery.h b/src/Parsers/ParserShowIndexesQuery.h
new file mode 100644
index 00000000000..aaa9e8789ff
--- /dev/null
+++ b/src/Parsers/ParserShowIndexesQuery.h
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <Parsers/IParserBase.h>
+
+namespace DB
+{
+
+/** Parses queries of the form
+ * SHOW [EXTENDED] INDEX|INDEXES|KEYS (FROM|IN) tbl [(FROM|IN) db] (WHERE expr)
+ */
+class ParserShowIndexesQuery : public IParserBase
+{
+protected:
+    const char * getName() const override { return "SHOW INDEXES query"; }
+
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+
+}
+
diff --git a/tests/queries/0_stateless/02724_show_indexes.reference b/tests/queries/0_stateless/02724_show_indexes.reference
new file mode 100644
index 00000000000..2ca55c77015
--- /dev/null
+++ b/tests/queries/0_stateless/02724_show_indexes.reference
@@ -0,0 +1,37 @@
+--- Aliases of SHOW INDEX
+tbl	0	blf_idx	\N	\N	\N	\N	\N	\N	\N	bloom_filter	\N	\N	YES	d, b
+tbl	0	mm1_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	a, c, d
+tbl	0	mm2_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	c, d, e
+tbl	0	PRIMARY	\N	\N	A	\N	\N	\N	\N	primary	\N	\N	YES	c, a
+tbl	0	set_idx	\N	\N	\N	\N	\N	\N	\N	set	\N	\N	YES	e
+tbl	0	blf_idx	\N	\N	\N	\N	\N	\N	\N	bloom_filter	\N	\N	YES	d, b
+tbl	0	mm1_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	a, c, d
+tbl	0	mm2_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	c, d, e
+tbl	0	PRIMARY	\N	\N	A	\N	\N	\N	\N	primary	\N	\N	YES	c, a
+tbl	0	set_idx	\N	\N	\N	\N	\N	\N	\N	set	\N	\N	YES	e
+tbl	0	blf_idx	\N	\N	\N	\N	\N	\N	\N	bloom_filter	\N	\N	YES	d, b
+tbl	0	mm1_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	a, c, d
+tbl	0	mm2_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	c, d, e
+tbl	0	PRIMARY	\N	\N	A	\N	\N	\N	\N	primary	\N	\N	YES	c, a
+tbl	0	set_idx	\N	\N	\N	\N	\N	\N	\N	set	\N	\N	YES	e
+--- EXTENDED
+tbl	0	blf_idx	\N	\N	\N	\N	\N	\N	\N	bloom_filter	\N	\N	YES	d, b
+tbl	0	mm1_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	a, c, d
+tbl	0	mm2_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	c, d, e
+tbl	0	PRIMARY	\N	\N	A	\N	\N	\N	\N	primary	\N	\N	YES	c, a
+tbl	0	set_idx	\N	\N	\N	\N	\N	\N	\N	set	\N	\N	YES	e
+--- WHERE
+tbl	0	mm1_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	a, c, d
+tbl	0	mm2_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	c, d, e
+--- Original table
+tbl	0	blf_idx	\N	\N	\N	\N	\N	\N	\N	bloom_filter	\N	\N	YES	d, b
+tbl	0	mm1_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	a, c, d
+tbl	0	mm2_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	c, d, e
+tbl	0	PRIMARY	\N	\N	A	\N	\N	\N	\N	primary	\N	\N	YES	c, a
+tbl	0	set_idx	\N	\N	\N	\N	\N	\N	\N	set	\N	\N	YES	e
+--- Equally named table in other database
+tbl	0	mmi_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	b
+tbl	0	PRIMARY	\N	\N	A	\N	\N	\N	\N	primary	\N	\N	YES	a
+--- Short form
+tbl	0	mmi_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	b
+tbl	0	PRIMARY	\N	\N	A	\N	\N	\N	\N	primary	\N	\N	YES	a
diff --git a/tests/queries/0_stateless/02724_show_indexes.sql b/tests/queries/0_stateless/02724_show_indexes.sql
new file mode 100644
index 00000000000..477e2f049c2
--- /dev/null
+++ b/tests/queries/0_stateless/02724_show_indexes.sql
@@ -0,0 +1,59 @@
+-- Tags: no-parallel
+-- no-parallel: creates a custom database schema and expects to use it exclusively
+
+-- Create a test table and verify that the output of SHOW INDEXES is sane.
+-- The matching of actual/expected results relies on the fact that the output of SHOW INDEX is sorted.
+DROP TABLE IF EXISTS tbl;
+CREATE TABLE tbl
+(
+    a UInt64,
+    b UInt64,
+    c UInt64,
+    d UInt64,
+    e UInt64,
+    INDEX mm1_idx (a, c, d) TYPE minmax,
+    INDEX mm2_idx (c, d, e) TYPE minmax,
+    INDEX set_idx (e)       TYPE set(100),
+    INDEX blf_idx (d, b)    TYPE bloom_filter(0.8)
+)
+ENGINE = MergeTree
+PRIMARY KEY (c, a);
+
+SELECT '--- Aliases of SHOW INDEX';
+SHOW INDEX FROM tbl;
+SHOW INDEXES FROM tbl;
+SHOW KEYS FROM tbl;
+
+SELECT '--- EXTENDED';
+SHOW EXTENDED INDEX FROM tbl;
+--
+SELECT '--- WHERE';
+SHOW INDEX FROM tbl WHERE index_type LIKE '%minmax%';
+
+-- Create a table in a different database. Intentionally using the same table/column names as above so
+-- we notice if something is buggy in the implementation of SHOW INDEX.
+DROP DATABASE IF EXISTS database_123456789abcde;
+CREATE DATABASE database_123456789abcde; -- pseudo-random database name
+
+DROP TABLE IF EXISTS database_123456789abcde.tbl;
+CREATE TABLE database_123456789abcde.tbl
+(
+    a UInt64,
+    b UInt64,
+    INDEX mmi_idx b TYPE minmax
+)
+ENGINE = MergeTree
+PRIMARY KEY a;
+
+SELECT '--- Original table';
+SHOW INDEX FROM tbl;
+
+SELECT '--- Equally named table in other database';
+SHOW INDEX FROM tbl FROM database_123456789abcde;
+
+SELECT '--- Short form';
+SHOW INDEX FROM database_123456789abcde.tbl;
+
+DROP DATABASE database_123456789abcde;
+
+DROP TABLE tbl;

From 403396d0bdec5a75ac736f483eb7b3e779d2a18c Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 25 Apr 2023 21:04:55 +0000
Subject: [PATCH 084/689] Fix style

---
 src/Interpreters/InterpreterShowIndexesQuery.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp
index cb282b95451..d47a20e1ba6 100644
--- a/src/Interpreters/InterpreterShowIndexesQuery.cpp
+++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp
@@ -108,8 +108,6 @@ String InterpreterShowIndexesQuery::getRewrittenQuery()
     /// SQL tests can take advantage of this.
     rewritten_query << " ORDER BY index_type, expression";
 
-    std::cout << rewritten_query.str() << std::endl;
-
     return rewritten_query.str();
 }
 

From c58e0e347cc987dc67573b0b980b56f3d72727fd Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Wed, 26 Apr 2023 09:46:35 +0000
Subject: [PATCH 085/689] Various cleanups

---
 docs/en/sql-reference/statements/show.md      | 46 ++++++++++---------
 .../InterpreterShowColumnsQuery.cpp           | 31 ++-----------
 .../InterpreterShowIndexesQuery.cpp           | 30 +-----------
 src/Interpreters/parseDatabaseAndTableName.h  | 44 ++++++++++++++++++
 src/Parsers/ParserShowColumnsQuery.h          |  2 +-
 src/Parsers/ParserShowIndexesQuery.cpp        |  1 -
 src/Parsers/ParserShowIndexesQuery.h          |  2 +-
 .../0_stateless/02706_show_columns.reference  |  4 ++
 .../0_stateless/02706_show_columns.sql        |  2 +
 9 files changed, 82 insertions(+), 80 deletions(-)
 create mode 100644 src/Interpreters/parseDatabaseAndTableName.h

diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md
index db4d55305ce..45d09c2dec7 100644
--- a/docs/en/sql-reference/statements/show.md
+++ b/docs/en/sql-reference/statements/show.md
@@ -266,51 +266,55 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2
 Displays a list of primary and data skipping indexes of a table.
 
 ```sql
-SHOW [EXTENDED] {INDEX | INDEXES |KEYS } {FROM | IN} <table> [{FROM | IN} <db>] [WHERE <expr>}] [INTO OUTFILE <filename>] [FORMAT <format>]
+SHOW [EXTENDED] {INDEX | INDEXES | KEYS } {FROM | IN} <table> [{FROM | IN} <db>] [WHERE <expr>] [INTO OUTFILE <filename>] [FORMAT <format>]
 ```
 
 The database and table name can be specified in abbreviated form as `<db>.<table>`, i.e. `FROM tab FROM db` and `FROM db.tab` are
-equivalent. If no database is specified, the query returns the list of columns from the current database.
+equivalent. If no database is specified, the query assumes the current database as database.
 
 The optional keyword `EXTENDED` currently has no effect, it only exists for MySQL compatibility.
 
 `SHOW INDEX` produces a result table with the following structure:
 - table - The name of the table (String)
 - non_unique - 0 if the index can contain duplicates, 1 otherwise (UInt8)
-- key_name - The name of the index. 'PRIMARY' if the index is a primary key index. (String)
-- seq_in_index - currently unused
-- column_name - currently unused
-- collation - currently unused
-- cardinality - currently unused
-- sub_part - currently unused
-- packed - currently unused
-- null - currently unused
-- index_type - the index type, e.g. `primary`, `minmax`, `bloom_filter` etc. (String)
-- comment - currently unused
-- index_comment - currently unused
-- visible - if the index is visible to the optimizer, always `YES` (String)
-- expression - the index expression (String)
+- key_name - The name of the index, `PRIMARY` if the index is a primary key index (String)
+- seq_in_index - Currently unused
+- column_name - Currently unused
+- collation - The sorting of the column in the index, `A` if ascending, `D` if descending, `NULL` if unsorted (Nullable(String))
+- cardinality - Currently unused
+- sub_part - Currently unused
+- packed - Currently unused
+- null - Currently unused
+- index_type - The index type, e.g. `primary`, `minmax`, `bloom_filter` etc. (String)
+- comment - Currently unused
+- index_comment - Currently unused
+- visible - If the index is visible to the optimizer, always `YES` (String)
+- expression - The index expression (String)
 
 **Examples**
 
 Getting information about all indexes in table 'tbl'
 
 ```sql
-SHOW INDEX FROM 'orders'
+SHOW INDEX FROM 'tbl'
 ```
 
 Result:
 
 ``` text
-┌─table─┬─non_unique─┬─key_name─┬─seq_in_index─┬─column_name─┬─collation─┬─cardinality─┬─sub_part─┬─packed─┬─null─┬─index_type─┬─comment─┬─index_comment─┬─visible─┬─expression─┐
-│ tbl   │          0 │ mm1_idx  │ ᴺᵁᴸᴸ         │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ      │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ     │ ᴺᵁᴸᴸ   │ ᴺᵁᴸᴸ │ minmax     │ ᴺᵁᴸᴸ    │ ᴺᵁᴸᴸ          │ YES     │ a, c, d    │
-│ tbl   │          0 │ mm2_idx  │ ᴺᵁᴸᴸ         │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ      │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ     │ ᴺᵁᴸᴸ   │ ᴺᵁᴸᴸ │ minmax     │ ᴺᵁᴸᴸ    │ ᴺᵁᴸᴸ          │ YES     │ c, d, e    │
-└───────┴────────────┴──────────┴──────────────┴─────────────┴───────────┴─────────────┴──────────┴────────┴──────┴────────────┴─────────┴───────────────┴─────────┴────────────┘
+┌─table─┬─non_unique─┬─key_name─┬─seq_in_index─┬─column_name─┬─collation─┬─cardinality─┬─sub_part─┬─packed─┬─null─┬─index_type───┬─comment─┬─index_comment─┬─visible─┬─expression─┐
+│ tbl   │          0 │ blf_idx  │ ᴺᵁᴸᴸ         │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ      │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ     │ ᴺᵁᴸᴸ   │ ᴺᵁᴸᴸ │ bloom_filter │ ᴺᵁᴸᴸ    │ ᴺᵁᴸᴸ          │ YES     │ d, b       │
+│ tbl   │          0 │ mm1_idx  │ ᴺᵁᴸᴸ         │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ      │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ     │ ᴺᵁᴸᴸ   │ ᴺᵁᴸᴸ │ minmax       │ ᴺᵁᴸᴸ    │ ᴺᵁᴸᴸ          │ YES     │ a, c, d    │
+│ tbl   │          0 │ mm2_idx  │ ᴺᵁᴸᴸ         │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ      │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ     │ ᴺᵁᴸᴸ   │ ᴺᵁᴸᴸ │ minmax       │ ᴺᵁᴸᴸ    │ ᴺᵁᴸᴸ          │ YES     │ c, d, e    │
+│ tbl   │          0 │ PRIMARY  │ ᴺᵁᴸᴸ         │ ᴺᵁᴸᴸ        │ A         │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ     │ ᴺᵁᴸᴸ   │ ᴺᵁᴸᴸ │ primary      │ ᴺᵁᴸᴸ    │ ᴺᵁᴸᴸ          │ YES     │ c, a       │
+│ tbl   │          0 │ set_idx  │ ᴺᵁᴸᴸ         │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ      │ ᴺᵁᴸᴸ        │ ᴺᵁᴸᴸ     │ ᴺᵁᴸᴸ   │ ᴺᵁᴸᴸ │ set          │ ᴺᵁᴸᴸ    │ ᴺᵁᴸᴸ          │ YES     │ e          │
+└───────┴────────────┴──────────┴──────────────┴─────────────┴───────────┴─────────────┴──────────┴────────┴──────┴──────────────┴─────────┴───────────────┴─────────┴────────────┘
 ```
 
 **See also**
 
-- [system.columns](https://clickhouse.com/docs/en/operations/system-tables/columns)
+- [system.tables](../../operations/system-tables/tables.md)
+- [system.data_skipping_indices](../../operations/system-tables/data_skipping_indices.md)
 
 ## SHOW PROCESSLIST
 
diff --git a/src/Interpreters/InterpreterShowColumnsQuery.cpp b/src/Interpreters/InterpreterShowColumnsQuery.cpp
index 4474be21d8b..86788007e3f 100644
--- a/src/Interpreters/InterpreterShowColumnsQuery.cpp
+++ b/src/Interpreters/InterpreterShowColumnsQuery.cpp
@@ -5,8 +5,8 @@
 #include <Parsers/formatAST.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/executeQuery.h>
+#include <Interpreters/parseDatabaseAndTableName.h>
 #include <IO/Operators.h>
-#include <boost/algorithm/string.hpp>
 
 
 namespace DB
@@ -42,33 +42,8 @@ String InterpreterShowColumnsQuery::getRewrittenQuery()
 
     rewritten_query << "FROM system.columns WHERE ";
 
-    String database;
-    String table;
-    if (query.from_table.contains("."))
-    {
-        /// FROM <db>.<table> (abbreviated form)
-        chassert(query.from_database.empty());
-        std::vector<String> split;
-        boost::split(split, query.from_table, boost::is_any_of("."));
-        chassert(split.size() == 2);
-        database = split[0];
-        table = split[1];
-    }
-    else if (query.from_database.empty())
-    {
-        /// FROM <table>
-        chassert(!query.from_table.empty());
-        database = getContext()->getCurrentDatabase();
-        table = query.from_table;
-    }
-    else
-    {
-        /// FROM <database> FROM <table>
-        chassert(!query.from_database.empty());
-        chassert(!query.from_table.empty());
-        database = query.from_database;
-        table = query.from_table;
-    }
+    auto [database, table] = parseDatabaseAndTableName(query, getContext()->getCurrentDatabase());
+
     rewritten_query << "database = " << DB::quote << database;
     rewritten_query << " AND table = " << DB::quote << table;
 
diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp
index d47a20e1ba6..c5a63cde51e 100644
--- a/src/Interpreters/InterpreterShowIndexesQuery.cpp
+++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp
@@ -5,8 +5,8 @@
 #include <Parsers/formatAST.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/executeQuery.h>
+#include <Interpreters/parseDatabaseAndTableName.h>
 #include <IO/Operators.h>
-#include <boost/algorithm/string.hpp>
 
 
 namespace DB
@@ -24,33 +24,7 @@ String InterpreterShowIndexesQuery::getRewrittenQuery()
 {
     const auto & query = query_ptr->as<ASTShowIndexesQuery &>();
 
-    String database;
-    String table;
-    if (query.from_table.contains("."))
-    {
-        /// FROM <db>.<table> (abbreviated form)
-        chassert(query.from_database.empty());
-        std::vector<String> split;
-        boost::split(split, query.from_table, boost::is_any_of("."));
-        chassert(split.size() == 2);
-        database = split[0];
-        table = split[1];
-    }
-    else if (query.from_database.empty())
-    {
-        /// FROM <table>
-        chassert(!query.from_table.empty());
-        database = getContext()->getCurrentDatabase();
-        table = query.from_table;
-    }
-    else
-    {
-        /// FROM <database> FROM <table>
-        chassert(!query.from_database.empty());
-        chassert(!query.from_table.empty());
-        database = query.from_database;
-        table = query.from_table;
-    }
+    auto [database, table] = parseDatabaseAndTableName(query, getContext()->getCurrentDatabase());
 
     WriteBufferFromOwnString where_expression_buf;
     if (query.where_expression)
diff --git a/src/Interpreters/parseDatabaseAndTableName.h b/src/Interpreters/parseDatabaseAndTableName.h
new file mode 100644
index 00000000000..b336a82028b
--- /dev/null
+++ b/src/Interpreters/parseDatabaseAndTableName.h
@@ -0,0 +1,44 @@
+#pragma once
+
+#include <Common/Exception.h>
+#include <base/defines.h>
+#include <base/types.h>
+#include <boost/algorithm/string.hpp>
+
+namespace DB
+{
+    template <typename Query>
+    inline std::pair<String, String> parseDatabaseAndTableName(const Query & query, const String & current_database)
+    {
+        String database;
+        String table;
+
+        if (query.from_table.contains("."))
+        {
+            /// FROM <db>.<table> (abbreviated form)
+            chassert(query.from_database.empty());
+            std::vector<String> split;
+            boost::split(split, query.from_table, boost::is_any_of("."));
+            chassert(split.size() == 2);
+            database = split[0];
+            table = split[1];
+        }
+        else if (query.from_database.empty())
+        {
+            /// FROM <table>
+            chassert(!query.from_table.empty());
+            database = current_database;
+            table = query.from_table;
+        }
+        else
+        {
+            /// FROM <database> FROM <table>
+            chassert(!query.from_database.empty());
+            chassert(!query.from_table.empty());
+            database = query.from_database;
+            table = query.from_table;
+        }
+
+        return {database, table};
+    }
+}
diff --git a/src/Parsers/ParserShowColumnsQuery.h b/src/Parsers/ParserShowColumnsQuery.h
index 999acf722af..a2941ca71d1 100644
--- a/src/Parsers/ParserShowColumnsQuery.h
+++ b/src/Parsers/ParserShowColumnsQuery.h
@@ -6,7 +6,7 @@ namespace DB
 {
 
 /** Parses queries of the form
- * SHOW [EXTENDED] [FULL] COLUMNS (FROM|IN) tbl [(FROM|IN) db] [(([NOT] (LIKE|ILIKE) expr) | (WHERE expr))] [LIMIT n]
+ * SHOW [EXTENDED] [FULL] COLUMNS FROM|IN tbl [FROM|IN db] [[NOT] LIKE|ILIKE expr | WHERE expr] [LIMIT n]
  */
 class ParserShowColumnsQuery : public IParserBase
 {
diff --git a/src/Parsers/ParserShowIndexesQuery.cpp b/src/Parsers/ParserShowIndexesQuery.cpp
index 4b7eba8636e..b4651ed21f9 100644
--- a/src/Parsers/ParserShowIndexesQuery.cpp
+++ b/src/Parsers/ParserShowIndexesQuery.cpp
@@ -1,6 +1,5 @@
 #include <Parsers/ParserShowIndexesQuery.h>
 
-// TODO
 #include <Parsers/ASTIdentifier_fwd.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTShowIndexesQuery.h>
diff --git a/src/Parsers/ParserShowIndexesQuery.h b/src/Parsers/ParserShowIndexesQuery.h
index aaa9e8789ff..0ece5e89ef7 100644
--- a/src/Parsers/ParserShowIndexesQuery.h
+++ b/src/Parsers/ParserShowIndexesQuery.h
@@ -6,7 +6,7 @@ namespace DB
 {
 
 /** Parses queries of the form
- * SHOW [EXTENDED] INDEX|INDEXES|KEYS (FROM|IN) tbl [(FROM|IN) db] (WHERE expr)
+ * SHOW [EXTENDED] INDEX|INDEXES|KEYS FROM|IN tbl [FROM|IN db] [WHERE expr]
  */
 class ParserShowIndexesQuery : public IParserBase
 {
diff --git a/tests/queries/0_stateless/02706_show_columns.reference b/tests/queries/0_stateless/02706_show_columns.reference
index da967d59cda..fff8d38ede7 100644
--- a/tests/queries/0_stateless/02706_show_columns.reference
+++ b/tests/queries/0_stateless/02706_show_columns.reference
@@ -1,3 +1,7 @@
+--- Aliases of SHOW COLUMNS
+int32	Nullable(Int32)	1		\N	
+str	String	0	SOR	\N	
+uint64	UInt64	0	PRI SOR	\N	
 int32	Nullable(Int32)	1		\N	
 str	String	0	SOR	\N	
 uint64	UInt64	0	PRI SOR	\N	
diff --git a/tests/queries/0_stateless/02706_show_columns.sql b/tests/queries/0_stateless/02706_show_columns.sql
index ce206fedee4..522f513b685 100644
--- a/tests/queries/0_stateless/02706_show_columns.sql
+++ b/tests/queries/0_stateless/02706_show_columns.sql
@@ -15,7 +15,9 @@ ENGINE = MergeTree
 PRIMARY KEY (uint64)
 ORDER BY (uint64, str);
 
+SELECT '--- Aliases of SHOW COLUMNS';
 SHOW COLUMNS FROM tab;
+SHOW FIELDS FROM tab;
 
 SELECT '--- EXTENDED';
 SHOW EXTENDED COLUMNS FROM tab;

From 6035f79bf6e025219343ae5e4b57cb90288e0130 Mon Sep 17 00:00:00 2001
From: Suzy Wang <suzy.wang@ibm.com>
Date: Wed, 26 Apr 2023 07:30:30 -0700
Subject: [PATCH 086/689] fix reinterpretAs queries on s390x

---
 src/Functions/reinterpretAs.cpp | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/src/Functions/reinterpretAs.cpp b/src/Functions/reinterpretAs.cpp
index 1481f73fbbf..e1d0dcd22da 100644
--- a/src/Functions/reinterpretAs.cpp
+++ b/src/Functions/reinterpretAs.cpp
@@ -176,18 +176,23 @@ public:
                     size_t size = offsets_from.size();
                     auto & vec_res = col_res->getData();
                     vec_res.resize_fill(size);
+                    size_t copy_size = 0;
+                    size_t offset_to = 0;
 
                     size_t offset = 0;
                     for (size_t i = 0; i < size; ++i)
                     {
+                        copy_size = std::min(static_cast<UInt64>(sizeof(ToFieldType)), offsets_from[i] - offset - 1);
+                        offset_to = sizeof(ToFieldType) > copy_size ? sizeof(ToFieldType) - copy_size : 0;
                         if constexpr (std::endian::native == std::endian::little)
                             memcpy(&vec_res[i],
                                 &data_from[offset],
-                                std::min(static_cast<UInt64>(sizeof(ToFieldType)), offsets_from[i] - offset - 1));
+                                copy_size);
                         else
-                            reverseMemcpy(&vec_res[i],
+                            reverseMemcpy(
+                                reinterpret_cast<char*>(&vec_res[i]) + offset_to,
                                 &data_from[offset],
-                                std::min(static_cast<UInt64>(sizeof(ToFieldType)), offsets_from[i] - offset - 1));
+                                copy_size);
                         offset = offsets_from[i];
                     }
 
@@ -242,9 +247,15 @@ public:
                     to.resize_fill(size);
 
                     static constexpr size_t copy_size = std::min(sizeof(From), sizeof(To));
+                    size_t offset_to = sizeof(To) > sizeof(From) ? sizeof(To) - sizeof(From):0;
 
                     for (size_t i = 0; i < size; ++i)
-                        memcpy(static_cast<void*>(&to[i]), static_cast<const void*>(&from[i]), copy_size);
+                    {
+                        if constexpr (std::endian::native == std::endian::little)
+                            memcpy(static_cast<void*>(&to[i]), static_cast<const void*>(&from[i]), copy_size);
+                        else
+                            memcpy(reinterpret_cast<char*>(&to[i]) + offset_to, static_cast<const void*>(&from[i]), copy_size);
+                    }
 
                     result = std::move(column_to);
 
@@ -322,11 +333,21 @@ private:
             StringRef data = src.getDataAt(i);
 
             /// Cut trailing zero bytes.
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+            size_t index = 0;
+            while (index < data.size && data.data[index] == 0)
+                    index++;
+            data.size -= index;
+#else
             while (data.size && data.data[data.size - 1] == 0)
                 --data.size;
-
+#endif
             data_to.resize(offset + data.size + 1);
-            memcpy(&data_to[offset], data.data, data.size);
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+            reverseMemcpy(&data_to[offset], data.data+index, data.size);
+#else
+            memcpy(&data_to[offset],data.data, data.size);
+#endif
             offset += data.size;
             data_to[offset] = 0;
             ++offset;

From 173a8a37957dbc295095e96aadc77d7a44a5b1ec Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Wed, 26 Apr 2023 18:38:28 +0000
Subject: [PATCH 087/689] fix tidy build

---
 src/Common/AsyncLoader.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index abeb456cacb..14190b707c6 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -105,11 +105,11 @@ LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, ssize_t priority
 class LoadTask : private boost::noncopyable
 {
 public:
-    LoadTask(AsyncLoader & loader_, LoadJobSet && jobs_, LoadJobSet && goals_ = {});
+    LoadTask(AsyncLoader & loader_, LoadJobSet && jobs_, LoadJobSet && goals_jobs_ = {});
     ~LoadTask();
 
     // Merge all jobs from other task into this task.
-    void merge(const LoadTaskPtr & o);
+    void merge(const LoadTaskPtr & task);
 
     // Schedule all jobs with AsyncLoader.
     void schedule();

From 85d7816d6501b4eb8132d0926dd5c35e211e02c7 Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Thu, 27 Apr 2023 13:55:26 +0200
Subject: [PATCH 088/689] Update src/Common/AsyncLoader.h

---
 src/Common/AsyncLoader.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 14190b707c6..2958f80d829 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -105,7 +105,7 @@ LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, ssize_t priority
 class LoadTask : private boost::noncopyable
 {
 public:
-    LoadTask(AsyncLoader & loader_, LoadJobSet && jobs_, LoadJobSet && goals_jobs_ = {});
+    LoadTask(AsyncLoader & loader_, LoadJobSet && jobs_, LoadJobSet && goal_jobs_ = {});
     ~LoadTask();
 
     // Merge all jobs from other task into this task.

From 30f1bef6e8d1a2086d8ef240601cf99e0e47887e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 27 Apr 2023 16:57:42 +0300
Subject: [PATCH 089/689] Update TableFunctionS3.h

---
 src/TableFunctions/TableFunctionS3.h | 44 ----------------------------
 1 file changed, 44 deletions(-)

diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h
index ed8cd3bd41a..70c4a020669 100644
--- a/src/TableFunctions/TableFunctionS3.h
+++ b/src/TableFunctions/TableFunctionS3.h
@@ -67,52 +67,8 @@ protected:
     ColumnsDescription structure_hint;
 };
 
-class TableFunctionCOS : public TableFunctionS3
-{
-public:
-    static constexpr auto name = "cosn";
-    std::string getName() const override
-    {
-        return name;
-    }
-private:
-    const char * getStorageTypeName() const override { return "COSN"; }
-};
-
-class TableFunctionOSS : public TableFunctionS3
-{
-public:
-    static constexpr auto name = "oss";
-    std::string getName() const override
-    {
-        return name;
-    }
-private:
-    const char * getStorageTypeName() const override { return "OSS"; }
-};
-
 }
 
-class TableFunctionGCS : public TableFunctionS3
-{
-public:
-    static constexpr auto name = "gcs";
-    static constexpr auto signature = " - url\n"
-                                      " - url, format\n"
-                                      " - url, format, structure\n"
-                                      " - url, hmac_key, hmac_secret\n"
-                                      " - url, format, structure, compression_method\n"
-                                      " - url, hmac_key, hmac_secret, format\n"
-                                      " - url, hmac_key, hmac_secret, format, structure\n"
-                                      " - url, hmac_key, hmac_secret, format, structure, compression_method";
-    std::string getName() const override
-    {
-        return name;
-    }
-private:
-    const char * getStorageTypeName() const override { return "GCS"; }
-};
-
 }
 
 #endif

From aec8f17614371f8dc4ea161098736adafeddc70b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 27 Apr 2023 16:58:18 +0300
Subject: [PATCH 090/689] Update TableFunctionS3.cpp

---
 src/TableFunctions/TableFunctionS3.cpp | 47 +++++++++++++++++++++++---
 1 file changed, 42 insertions(+), 5 deletions(-)

diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp
index 4153c6a81c9..841f5a91bc8 100644
--- a/src/TableFunctions/TableFunctionS3.cpp
+++ b/src/TableFunctions/TableFunctionS3.cpp
@@ -215,6 +215,48 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, Context
 }
 
 
+class TableFunctionGCS : public TableFunctionS3
+{
+public:
+    static constexpr auto name = "gcs";
+    std::string getName() const override
+    {
+        return name;
+    }
+private:
+    const char * getStorageTypeName() const override { return "GCS"; }
+};
+
+class TableFunctionCOS : public TableFunctionS3
+{
+public:
+    static constexpr auto name = "cosn";
+    std::string getName() const override
+    {
+        return name;
+    }
+private:
+    const char * getStorageTypeName() const override { return "COSN"; }
+};
+
+class TableFunctionOSS : public TableFunctionS3
+{
+public:
+    static constexpr auto name = "oss";
+    std::string getName() const override
+    {
+        return name;
+    }
+private:
+    const char * getStorageTypeName() const override { return "OSS"; }
+};
+
+
+void registerTableFunctionGCS(TableFunctionFactory & factory)
+{
+    factory.registerFunction<TableFunctionGCS>();
+}
+
 void registerTableFunctionS3(TableFunctionFactory & factory)
 {
     factory.registerFunction<TableFunctionS3>();
@@ -230,11 +272,6 @@ void registerTableFunctionOSS(TableFunctionFactory & factory)
     factory.registerFunction<TableFunctionOSS>();
 }
 
-void registerTableFunctionGCS(TableFunctionFactory & factory)
-{
-    factory.registerFunction<TableFunctionGCS>();
-}
-
 }
 
 #endif

From 536720605797f37a2870c79855a633cca3cd5faa Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 27 Apr 2023 23:24:36 +0300
Subject: [PATCH 091/689] Update TableFunctionS3.h

---
 src/TableFunctions/TableFunctionS3.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h
index 70c4a020669..4724684712b 100644
--- a/src/TableFunctions/TableFunctionS3.h
+++ b/src/TableFunctions/TableFunctionS3.h
@@ -69,6 +69,4 @@ protected:
 
 }
 
-}
-
 #endif

From 2da33b96eba3698dd347d7a0c5d487cfa031f85d Mon Sep 17 00:00:00 2001
From: xmy <xumovens@gmail.com>
Date: Fri, 28 Apr 2023 10:31:49 +0800
Subject: [PATCH 092/689] Allow Int* type argument for
 groupBitAnd/GroupBitOr/groupBitXor

---
 .../aggregate-functions/reference/groupbitand.md   |  4 ++--
 .../aggregate-functions/reference/groupbitor.md    |  4 ++--
 .../aggregate-functions/reference/groupbitxor.md   |  4 ++--
 .../AggregateFunctionBitwise.cpp                   |  2 +-
 .../AggregateFunctionSequenceMatch.cpp             |  2 +-
 .../AggregateFunctionWindowFunnel.cpp              |  2 +-
 src/AggregateFunctions/Helpers.h                   | 14 ++++++++++++--
 7 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md
index f89e3796aaa..5fd5029751a 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md
@@ -13,11 +13,11 @@ groupBitAnd(expr)
 
 **Arguments**
 
-`expr` – An expression that results in `UInt*` type.
+`expr` – An expression that results in `UInt* or Int*` type.
 
 **Return value**
 
-Value of the `UInt*` type.
+Value of the `UInt* or Int*` type.
 
 **Example**
 
diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md
index 75b34d9c5a3..08a5c15da46 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md
@@ -13,11 +13,11 @@ groupBitOr(expr)
 
 **Arguments**
 
-`expr` – An expression that results in `UInt*` type.
+`expr` – An expression that results in `UInt* or Int*` type.
 
 **Returned value**
 
-Value of the `UInt*` type.
+Value of the `UInt* or Int*` type.
 
 **Example**
 
diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md
index ca6fb9f8352..f33e375953c 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md
@@ -13,11 +13,11 @@ groupBitXor(expr)
 
 **Arguments**
 
-`expr` – An expression that results in `UInt*` type.
+`expr` – An expression that results in `UInt* or Int*` type.
 
 **Return value**
 
-Value of the `UInt*` type.
+Value of the `UInt* or Int*` type.
 
 **Example**
 
diff --git a/src/AggregateFunctions/AggregateFunctionBitwise.cpp b/src/AggregateFunctions/AggregateFunctionBitwise.cpp
index b87e899a685..82cb3b327f0 100644
--- a/src/AggregateFunctions/AggregateFunctionBitwise.cpp
+++ b/src/AggregateFunctions/AggregateFunctionBitwise.cpp
@@ -27,7 +27,7 @@ AggregateFunctionPtr createAggregateFunctionBitwise(const std::string & name, co
                         "is illegal, because it cannot be used in bitwise operations",
                         argument_types[0]->getName(), name);
 
-    AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunctionBitwise, Data>(*argument_types[0], argument_types[0]));
+    AggregateFunctionPtr res(createWithOptionSignedIntegerType<AggregateFunctionBitwise, Data, true>(*argument_types[0], argument_types[0]));
 
     if (!res)
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
diff --git a/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp b/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp
index 3dd9a8b658d..f2fe9014ceb 100644
--- a/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp
+++ b/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp
@@ -53,7 +53,7 @@ AggregateFunctionPtr createAggregateFunctionSequenceBase(
 
     String pattern = params.front().safeGet<std::string>();
 
-    AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunction, Data>(*argument_types[0], argument_types, params, pattern));
+    AggregateFunctionPtr res(createWithOptionSignedIntegerType<AggregateFunction, Data>(*argument_types[0], argument_types, params, pattern));
     if (res)
         return res;
 
diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp
index d80d683fd04..71c675f7a3b 100644
--- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp
+++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp
@@ -48,7 +48,7 @@ createAggregateFunctionWindowFunnel(const std::string & name, const DataTypes &
                             cond_arg->getName(), toString(i + 1), name);
     }
 
-    AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunctionWindowFunnel, Data>(*arguments[0], arguments, params));
+    AggregateFunctionPtr res(createWithOptionSignedIntegerType<AggregateFunctionWindowFunnel, Data>(*arguments[0], arguments, params));
     WhichDataType which(arguments.front().get());
     if (res)
         return res;
diff --git a/src/AggregateFunctions/Helpers.h b/src/AggregateFunctions/Helpers.h
index 19904dd9215..4131279a897 100644
--- a/src/AggregateFunctions/Helpers.h
+++ b/src/AggregateFunctions/Helpers.h
@@ -87,8 +87,8 @@ static IAggregateFunction * createWithNumericType(const IDataType & argument_typ
     return nullptr;
 }
 
-template <template <typename, typename> class AggregateFunctionTemplate, template <typename> class Data, typename... TArgs>
-static IAggregateFunction * createWithUnsignedIntegerType(const IDataType & argument_type, TArgs && ... args)
+template <template <typename, typename> class AggregateFunctionTemplate, template <typename> class Data, bool support_signed = false, typename... TArgs>
+static IAggregateFunction * createWithOptionSignedIntegerType(const IDataType & argument_type, TArgs && ... args)
 {
     WhichDataType which(argument_type);
     if (which.idx == TypeIndex::UInt8) return new AggregateFunctionTemplate<UInt8, Data<UInt8>>(std::forward<TArgs>(args)...);
@@ -97,6 +97,16 @@ static IAggregateFunction * createWithUnsignedIntegerType(const IDataType & argu
     if (which.idx == TypeIndex::UInt64) return new AggregateFunctionTemplate<UInt64, Data<UInt64>>(std::forward<TArgs>(args)...);
     if (which.idx == TypeIndex::UInt128) return new AggregateFunctionTemplate<UInt128, Data<UInt128>>(std::forward<TArgs>(args)...);
     if (which.idx == TypeIndex::UInt256) return new AggregateFunctionTemplate<UInt256, Data<UInt256>>(std::forward<TArgs>(args)...);
+
+    if constexpr (support_signed)
+    {
+        if (which.idx == TypeIndex::Int8) return new AggregateFunctionTemplate<Int8, Data<Int8>>(std::forward<TArgs>(args)...);
+        if (which.idx == TypeIndex::Int16) return new AggregateFunctionTemplate<Int16, Data<Int16>>(std::forward<TArgs>(args)...);
+        if (which.idx == TypeIndex::Int32) return new AggregateFunctionTemplate<Int32, Data<Int32>>(std::forward<TArgs>(args)...);
+        if (which.idx == TypeIndex::Int64) return new AggregateFunctionTemplate<Int64, Data<Int64>>(std::forward<TArgs>(args)...);
+        if (which.idx == TypeIndex::Int128) return new AggregateFunctionTemplate<Int128, Data<Int128>>(std::forward<TArgs>(args)...);
+        if (which.idx == TypeIndex::Int256) return new AggregateFunctionTemplate<Int256, Data<Int256>>(std::forward<TArgs>(args)...);
+    }
     return nullptr;
 }
 

From d479404318a6c29a9c02b498989d6cdf44343b82 Mon Sep 17 00:00:00 2001
From: Eridanus <maodingxuan@foxmail.com>
Date: Thu, 27 Apr 2023 20:27:00 +0800
Subject: [PATCH 093/689] improvement(parser): size suffixes in quota creation
 parameters

This commit adds support for creating quotas with size suffixes in the
parameter list of a quota creation statement. The parser now recognizes
size suffixes such as "KB", "MB", and "GB" and correctly converts them
to their corresponding values in bytes. This feature improves the
usability of the quota creation statement and makes it more flexible.
The commit also includes additional tests to ensure that the new syntax
is parsed correctly.
---
 src/Parsers/Access/ParserCreateQuotaQuery.cpp | 183 ++++++++++++++++--
 .../0_stateless/01297_create_quota.reference  |  35 ++++
 .../0_stateless/01297_create_quota.sql        | 110 ++++++++++-
 3 files changed, 314 insertions(+), 14 deletions(-)

diff --git a/src/Parsers/Access/ParserCreateQuotaQuery.cpp b/src/Parsers/Access/ParserCreateQuotaQuery.cpp
index a67051be398..63bb07a43cd 100644
--- a/src/Parsers/Access/ParserCreateQuotaQuery.cpp
+++ b/src/Parsers/Access/ParserCreateQuotaQuery.cpp
@@ -1,20 +1,20 @@
-#include <Parsers/Access/ParserCreateQuotaQuery.h>
-#include <Parsers/Access/ASTCreateQuotaQuery.h>
-#include <Parsers/Access/ASTRolesOrUsersSet.h>
-#include <Parsers/Access/ParserRolesOrUsersSet.h>
+#include <IO/ReadHelpers.h>
 #include <Parsers/ASTIdentifier_fwd.h>
 #include <Parsers/ASTLiteral.h>
+#include <Parsers/Access/ASTCreateQuotaQuery.h>
+#include <Parsers/Access/ASTRolesOrUsersSet.h>
+#include <Parsers/Access/ParserCreateQuotaQuery.h>
+#include <Parsers/Access/ParserRolesOrUsersSet.h>
 #include <Parsers/CommonParsers.h>
 #include <Parsers/ExpressionElementParsers.h>
 #include <Parsers/ExpressionListParsers.h>
-#include <Parsers/parseIntervalKind.h>
 #include <Parsers/parseIdentifierOrStringLiteral.h>
-#include <Common/FieldVisitorConvertToNumber.h>
+#include <Parsers/parseIntervalKind.h>
 #include <base/range.h>
 #include <boost/algorithm/string/case_conv.hpp>
 #include <boost/algorithm/string/join.hpp>
-#include <boost/algorithm/string/predicate.hpp>
 #include <boost/algorithm/string/replace.hpp>
+#include <Common/FieldVisitorConvertToNumber.h>
 
 
 namespace DB
@@ -107,23 +107,182 @@ namespace
         });
     }
 
+    template <typename T, typename = std::enable_if_t<std::is_same_v<T, QuotaValue> || std::is_same_v<T, double>>>
+    bool multipleBinarySuffix(T & max_value, int power_of_two)
+    {
+        constexpr T max_t = std::numeric_limits<T>::max();
+        if constexpr (std::is_same_v<T, double>)
+        {
+            const T factor = std::exp2(power_of_two);
+            // Multiplication will cause overflow
+            if (max_t / factor < max_value)
+                return false;
+            max_value *= factor;
+        }
+        else
+        {
+            // Bit shift will cause overflow
+            if (max_t >> power_of_two < max_value)
+                return false;
+            max_value <<= power_of_two;
+        }
+        return true;
+    }
+
+    template <typename T, typename = std::enable_if_t<std::is_same_v<T, QuotaValue> || std::is_same_v<T, double>>>
+    bool multipleDeciMalSuffix(T & max_value, UInt64 base)
+    {
+        constexpr T max_t = std::numeric_limits<T>::max();
+        // Multiplication will cause overflow
+        if (max_t / base < max_value)
+            return false;
+        max_value *= base;
+        return true;
+    }
+
+    template <typename T, typename = std::enable_if_t<std::is_same_v<T, QuotaValue> || std::is_same_v<T, double>>>
+    bool multipleSizeSuffix(T & max_value, ReadBufferFromMemory & suffix)
+    {
+        // Check suffix empty before call this function
+        // Updates max_value depending on the suffix
+        auto finish = [&suffix, &max_value](UInt64 base, int power_of_two)
+        {
+            ++suffix.position();
+            if (*suffix.position() == 'i')
+            {
+                ++suffix.position();
+                return multipleBinarySuffix(max_value, power_of_two); // NOLINT // For binary suffixes, such as ki, Mi, Gi, etc.
+            }
+            else
+            {
+                return multipleDeciMalSuffix(max_value, base); // For decimal suffixes, such as k, M, G etc.
+            }
+        };
+
+        switch (*suffix.position())
+        {
+            case 'k':
+                [[fallthrough]];
+            case 'K':
+                return finish(1e3, 10);
+            case 'M':
+                return finish(1e6, 20);
+            case 'G':
+                return finish(1e9, 30);
+            case 'T':
+                return finish(1e12, 40);
+            default:
+                return false;
+        }
+    }
+
+    bool isSizeSuffix(const char c)
+    {
+        return c == 'k' || c == 'K' || c == 'M' || c == 'G' || c == 'T';
+    }
+
+    template <typename T, typename = std::enable_if_t<std::is_same_v<T, QuotaValue> || std::is_same_v<T, double>>>
+    bool parseWithSizeSuffix(std::string_view s, T & max_value)
+    {
+        ReadBufferFromMemory buf(s.data(), s.size());
+
+        // Consume space
+        while (!buf.eof() && (*buf.position()) == ' ')
+        {
+            ++buf.position();
+        }
+
+        if constexpr (is_integer<T>)
+        {
+            if (!tryReadIntText(max_value, buf))
+            {
+                return false;
+            }
+        }
+        else
+        {
+            if (!tryReadFloatText(max_value, buf))
+            {
+                return false;
+            }
+            // Underflow
+            if (max_value < 0)
+            {
+                return false;
+            }
+        }
+
+        // Has suffix
+        if (!buf.eof() && isSizeSuffix(*buf.position()))
+        {
+            if (!multipleSizeSuffix(max_value, buf))
+            {
+                return false;
+            }
+        }
+
+        // Consume space
+        while (!buf.eof() && isspace(*buf.position()))
+        {
+            ++buf.position();
+        }
+
+        // More after size suffix, illegal syntax
+        if (!buf.eof())
+        {
+            return false;
+        }
+        return true;
+    }
+
+
+    template <typename T, typename = std::enable_if_t<std::is_same_v<T, double> || std::is_same_v<T, QuotaValue>>>
+    bool fieldToNumber(const Field & f, T & max_value)
+    {
+        if (f.getType() == Field::Types::String)
+        {
+            if (!parseWithSizeSuffix<T>(f.get<const String &>(), max_value))
+            {
+                return false;
+            }
+            return true;
+        }
+        else
+        {
+            max_value = applyVisitor(FieldVisitorConvertToNumber<T>(), f);
+            return true;
+        }
+    }
 
     bool parseMaxValue(IParserBase::Pos & pos, Expected & expected, QuotaType quota_type, QuotaValue & max_value)
     {
         ASTPtr ast;
-        if (!ParserNumber{}.parse(pos, ast, expected))
+        if (!ParserNumber{}.parse(pos, ast, expected) && !ParserStringLiteral{}.parse(pos, ast, expected))
             return false;
 
         const Field & max_field = ast->as<ASTLiteral &>().value;
         const auto & type_info = QuotaTypeInfo::get(quota_type);
         if (type_info.output_denominator == 1)
-            max_value = applyVisitor(FieldVisitorConvertToNumber<QuotaValue>(), max_field);
+        {
+            return fieldToNumber<QuotaValue>(max_field, max_value);
+        }
         else
-            max_value = static_cast<QuotaValue>(applyVisitor(FieldVisitorConvertToNumber<double>(), max_field) * type_info.output_denominator);
-        return true;
+        {
+            // 2023-04-25 : only for EXECUTION_TIME
+            double tmp_max_value = 0;
+            if (fieldToNumber<double>(max_field, tmp_max_value))
+            {
+                constexpr QuotaValue max_quota = std::numeric_limits<QuotaValue>::max();
+                // Multiplication will cause overflow
+                if (max_quota / type_info.output_denominator < tmp_max_value) // NOLINT
+                    return false;
+                max_value = static_cast<QuotaValue>(tmp_max_value * type_info.output_denominator);
+                return true;
+            }
+            return false;
+        }
     }
 
-
     bool parseLimits(IParserBase::Pos & pos, Expected & expected, std::vector<std::pair<QuotaType, QuotaValue>> & limits)
     {
         std::vector<std::pair<QuotaType, QuotaValue>> res_limits;
diff --git a/tests/queries/0_stateless/01297_create_quota.reference b/tests/queries/0_stateless/01297_create_quota.reference
index f3f833d7619..60ab84ab813 100644
--- a/tests/queries/0_stateless/01297_create_quota.reference
+++ b/tests/queries/0_stateless/01297_create_quota.reference
@@ -64,3 +64,38 @@ q4_01297	604800	0	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N
 -- query_selects query_inserts
 CREATE QUOTA q1_01297 KEYED BY user_name FOR INTERVAL 1 minute MAX query_selects = 1 TO r1_01297
 CREATE QUOTA q2_01297 KEYED BY user_name FOR INTERVAL 1 minute MAX query_inserts = 1 TO r1_01297
+-- size suffix
+-- functional test
+CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 12000
+CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX query_selects = 12288
+CREATE QUOTA q3_01297 FOR INTERVAL 1 minute MAX query_selects = 12000000
+CREATE QUOTA q4_01297 FOR INTERVAL 1 minute MAX query_selects = 12582912
+CREATE QUOTA q5_01297 FOR INTERVAL 1 minute MAX query_selects = 12000000000
+CREATE QUOTA q6_01297 FOR INTERVAL 1 minute MAX query_selects = 12884901888
+CREATE QUOTA q7_01297 FOR INTERVAL 1 minute MAX query_selects = 12000000000000
+CREATE QUOTA q8_01297 FOR INTERVAL 1 minute MAX query_selects = 13194139533312
+CREATE QUOTA q9_01297 FOR INTERVAL 1 minute MAX execution_time = 12000
+CREATE QUOTA q10_01297 FOR INTERVAL 1 minute MAX execution_time = 12288
+CREATE QUOTA q11_01297 FOR INTERVAL 1 minute MAX execution_time = 12000000
+CREATE QUOTA q12_01297 FOR INTERVAL 1 minute MAX execution_time = 12582912
+CREATE QUOTA q13_01297 FOR INTERVAL 1 minute MAX execution_time = 12000000000
+CREATE QUOTA q14_01297 FOR INTERVAL 1 minute MAX execution_time = 12884901888
+-- overflow test
+CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 18446744073709551615
+CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 18446744073
+-- zero test
+CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 0
+CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 0
+-- underflow test
+-- syntax test
+CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 12
+CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 12
+CREATE QUOTA q3_01297 FOR INTERVAL 1 minute MAX query_selects = 12000
+CREATE QUOTA q4_01297 FOR INTERVAL 1 minute MAX execution_time = 12000
+CREATE QUOTA q5_01297 FOR INTERVAL 1 minute MAX execution_time = 0
+CREATE QUOTA q6_01297 FOR INTERVAL 1 minute MAX execution_time = 0
+CREATE QUOTA q7_01297 FOR INTERVAL 1 minute MAX execution_time = 0
+CREATE QUOTA q8_01297 FOR INTERVAL 1 minute MAX execution_time = 0
+CREATE QUOTA q9_01297 FOR INTERVAL 1 minute MAX execution_time = 123000
+CREATE QUOTA q10_01297 FOR INTERVAL 1 minute MAX execution_time = 123000
+-- bad syntax test
diff --git a/tests/queries/0_stateless/01297_create_quota.sql b/tests/queries/0_stateless/01297_create_quota.sql
index b30d453257a..5a7cecd03ec 100644
--- a/tests/queries/0_stateless/01297_create_quota.sql
+++ b/tests/queries/0_stateless/01297_create_quota.sql
@@ -1,7 +1,8 @@
 -- Tags: no-parallel
 
-DROP QUOTA IF EXISTS q1_01297, q2_01297, q3_01297, q4_01297, q5_01297, q6_01297, q7_01297, q8_01297, q9_01297, q10_01297;
-DROP QUOTA IF EXISTS q11_01297, q12_01297;
+DROP QUOTA IF EXISTS q1_01297, q2_01297, q3_01297, q4_01297, q5_01297, q6_01297, q7_01297, q8_01297, q9_01297,
+q10_01297, q11_01297, q12_01297, q13_01297, q14_01297, q15_01297, q16_01297, q17_01297;
+
 DROP QUOTA IF EXISTS q2_01297_renamed;
 DROP USER IF EXISTS u1_01297;
 DROP ROLE IF EXISTS r1_01297;
@@ -137,3 +138,108 @@ DROP QUOTA q1_01297, q2_01297;
 DROP ROLE r1_01297;
 DROP USER u1_01297;
 
+SELECT '-- size suffix';
+SELECT '-- functional test';
+CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '12K';
+CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '12Ki';
+CREATE QUOTA q3_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '12M';
+CREATE QUOTA q4_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '12Mi';
+CREATE QUOTA q5_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '12G';
+CREATE QUOTA q6_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '12Gi';
+CREATE QUOTA q7_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '12T';
+CREATE QUOTA q8_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '12Ti';
+CREATE QUOTA q9_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '12K';
+CREATE QUOTA q10_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '12Ki';
+CREATE QUOTA q11_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '12M';
+CREATE QUOTA q12_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '12Mi';
+CREATE QUOTA q13_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '12G';
+CREATE QUOTA q14_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '12Gi';
+SHOW CREATE QUOTA q1_01297;
+SHOW CREATE QUOTA q2_01297;
+SHOW CREATE QUOTA q3_01297;
+SHOW CREATE QUOTA q4_01297;
+SHOW CREATE QUOTA q5_01297;
+SHOW CREATE QUOTA q6_01297;
+SHOW CREATE QUOTA q7_01297;
+SHOW CREATE QUOTA q8_01297;
+SHOW CREATE QUOTA q9_01297;
+SHOW CREATE QUOTA q10_01297;
+SHOW CREATE QUOTA q11_01297;
+SHOW CREATE QUOTA q12_01297;
+SHOW CREATE QUOTA q13_01297;
+SHOW CREATE QUOTA q14_01297;
+DROP QUOTA IF EXISTS q1_01297;
+DROP QUOTA IF EXISTS q2_01297;
+DROP QUOTA IF EXISTS q3_01297;
+DROP QUOTA IF EXISTS q4_01297;
+DROP QUOTA IF EXISTS q5_01297;
+DROP QUOTA IF EXISTS q6_01297;
+DROP QUOTA IF EXISTS q7_01297;
+DROP QUOTA IF EXISTS q8_01297;
+DROP QUOTA IF EXISTS q9_01297;
+DROP QUOTA IF EXISTS q10_01297;
+DROP QUOTA IF EXISTS q11_01297;
+DROP QUOTA IF EXISTS q12_01297;
+DROP QUOTA IF EXISTS q13_01297;
+DROP QUOTA IF EXISTS q14_01297;
+SELECT '-- overflow test';
+CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '18446744073709551615';
+CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '18446744073';
+SHOW CREATE QUOTA q1_01297;
+SHOW CREATE QUOTA q2_01297;
+DROP QUOTA IF EXISTS q1_01297;
+DROP QUOTA IF EXISTS q2_01297;
+CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '18446744073709551616'; -- { clientError 62 }
+CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1T'; -- { clientError 62 }
+CREATE QUOTA q3_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1Ti'; -- { clientError 62 }
+CREATE QUOTA q4_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '18446744074'; -- { clientError 62 }
+SELECT '-- zero test';
+CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '0';
+CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '0';
+SHOW CREATE QUOTA q1_01297;
+SHOW CREATE QUOTA q2_01297;
+DROP QUOTA IF EXISTS q1_01297;
+DROP QUOTA IF EXISTS q2_01297;
+SELECT '-- underflow test';
+CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '-1'; -- { clientError 62 }
+CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '-1'; -- { clientError 62 }
+SELECT '-- syntax test';
+CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = ' 12 ';
+CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = ' 12 ';
+CREATE QUOTA q3_01297 FOR INTERVAL 1 MINUTE MAX query_selects = ' 12k ';
+CREATE QUOTA q4_01297 FOR INTERVAL 1 MINUTE MAX execution_time = ' 12k ';
+CREATE QUOTA q5_01297 FOR INTERVAL 1 MINUTE MAX execution_time = ' 00 ';
+CREATE QUOTA q6_01297 FOR INTERVAL 1 MINUTE MAX execution_time = ' 00 ';
+CREATE QUOTA q7_01297 FOR INTERVAL 1 MINUTE MAX execution_time = ' 00k ';
+CREATE QUOTA q8_01297 FOR INTERVAL 1 MINUTE MAX execution_time = ' 00k ';
+CREATE QUOTA q9_01297 FOR INTERVAL 1 MINUTE MAX execution_time = ' 00123k ';
+CREATE QUOTA q10_01297 FOR INTERVAL 1 MINUTE MAX execution_time = ' 00123k ';
+SHOW CREATE QUOTA q1_01297;
+SHOW CREATE QUOTA q2_01297;
+SHOW CREATE QUOTA q3_01297;
+SHOW CREATE QUOTA q4_01297;
+SHOW CREATE QUOTA q5_01297;
+SHOW CREATE QUOTA q6_01297;
+SHOW CREATE QUOTA q7_01297;
+SHOW CREATE QUOTA q8_01297;
+SHOW CREATE QUOTA q9_01297;
+SHOW CREATE QUOTA q10_01297;
+DROP QUOTA IF EXISTS q1_01297;
+DROP QUOTA IF EXISTS q2_01297;
+DROP QUOTA IF EXISTS q3_01297;
+DROP QUOTA IF EXISTS q4_01297;
+DROP QUOTA IF EXISTS q5_01297;
+DROP QUOTA IF EXISTS q6_01297;
+DROP QUOTA IF EXISTS q7_01297;
+DROP QUOTA IF EXISTS q8_01297;
+DROP QUOTA IF EXISTS q9_01297;
+DROP QUOTA IF EXISTS q10_01297;
+SELECT '-- bad syntax test';
+CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '1 1'; -- { clientError 62 }
+CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1 1'; -- { clientError 62 }
+CREATE QUOTA q3_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '1K 1'; -- { clientError 62 }
+CREATE QUOTA q4_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1K 1'; -- { clientError 62 }
+CREATE QUOTA q5_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '1K1'; -- { clientError 62 }
+CREATE QUOTA q6_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1K1'; -- { clientError 62 }
+CREATE QUOTA q7_01297 FOR INTERVAL 1 MINUTE MAX query_selects = 'foo'; -- { clientError 62 }
+CREATE QUOTA q8_01297 FOR INTERVAL 1 MINUTE MAX execution_time = 'bar'; -- { clientError 62 }

From 7bab39105fef9ca787a691a92f61f8538698bf0d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 29 Apr 2023 00:14:10 +0200
Subject: [PATCH 094/689] Fix progress bar for LowCardinality fields with
 shared dictionaries

---
 src/Columns/ColumnLowCardinality.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h
index e7f4b92d733..f9bf7290601 100644
--- a/src/Columns/ColumnLowCardinality.h
+++ b/src/Columns/ColumnLowCardinality.h
@@ -160,7 +160,9 @@ public:
 
     void reserve(size_t n) override { idx.reserve(n); }
 
-    size_t byteSize() const override { return idx.getPositions()->byteSize() + getDictionary().byteSize(); }
+    /// Don't count the dictionary size as it can be shared between different blocks.
+    size_t byteSize() const override { return idx.getPositions()->byteSize(); }
+
     size_t byteSizeAt(size_t n) const override { return getDictionary().byteSizeAt(getIndexes().getUInt(n)); }
     size_t allocatedBytes() const override { return idx.getPositions()->allocatedBytes() + getDictionary().allocatedBytes(); }
 

From 23defcbf6a9972733fc9b8a5866da30b039b3d37 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Thu, 27 Apr 2023 14:49:35 +0000
Subject: [PATCH 095/689] log progress in AsyncLoader

---
 src/Common/AsyncLoader.cpp | 48 +++++++++++++++++++++++++++++++++++---
 src/Common/AsyncLoader.h   | 18 ++++++++++++++
 2 files changed, 63 insertions(+), 3 deletions(-)

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
index 804b8ea71dc..1a4774d984e 100644
--- a/src/Common/AsyncLoader.cpp
+++ b/src/Common/AsyncLoader.cpp
@@ -5,6 +5,7 @@
 #include <Common/Exception.h>
 #include <Common/noexcept_scope.h>
 #include <Common/setThreadName.h>
+#include <Common/logger_useful.h>
 
 namespace DB
 {
@@ -16,6 +17,17 @@ namespace ErrorCodes
     extern const int ASYNC_LOAD_CANCELED;
 }
 
+static constexpr size_t PRINT_MESSAGE_EACH_N_OBJECTS = 256;
+static constexpr size_t PRINT_MESSAGE_EACH_N_SECONDS = 5;
+
+void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, AtomicStopwatch & watch)
+{
+    if (processed % PRINT_MESSAGE_EACH_N_OBJECTS == 0 || watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS))
+    {
+        LOG_INFO(log, "{}%", processed * 100.0 / total);
+        watch.restart();
+    }
+}
 
 LoadStatus LoadJob::status() const
 {
@@ -84,10 +96,23 @@ void LoadJob::canceled(const std::exception_ptr & ptr)
 void LoadJob::finish()
 {
     func = {}; // To ensure job function is destructed before `AsyncLoader::wait()` and `LoadJob::wait()` return
+    finished_ns = clock_gettime_ns();
     if (waiters > 0)
         finished.notify_all();
 }
 
+void LoadJob::scheduled()
+{
+    scheduled_ns = clock_gettime_ns();
+}
+
+void LoadJob::execute(const LoadJobPtr & self)
+{
+    started_ns = clock_gettime_ns();
+    func(self);
+}
+
+
 LoadTask::LoadTask(AsyncLoader & loader_, LoadJobSet && jobs_, LoadJobSet && goal_jobs_)
     : loader(loader_)
     , jobs(std::move(jobs_))
@@ -127,9 +152,12 @@ void LoadTask::detach()
 
 AsyncLoader::AsyncLoader(Metric metric_threads, Metric metric_active_threads, size_t max_threads_, bool log_failures_)
     : log_failures(log_failures_)
+    , log(&Poco::Logger::get("AsyncLoader"))
     , max_threads(max_threads_)
     , pool(metric_threads, metric_active_threads, max_threads)
-{}
+{
+
+}
 
 AsyncLoader::~AsyncLoader()
 {
@@ -186,6 +214,14 @@ void AsyncLoader::scheduleImpl(const LoadJobSet & jobs)
 {
     std::unique_lock lock{mutex};
 
+    // Restart watches after idle period
+    if (scheduled_jobs.empty())
+    {
+        busy_period_started_ns = clock_gettime_ns();
+        stopwatch.restart();
+        old_jobs = finished_jobs.size();
+    }
+
     // Sanity checks
     for (const auto & job : jobs)
     {
@@ -207,6 +243,7 @@ void AsyncLoader::scheduleImpl(const LoadJobSet & jobs)
         NOEXCEPT_SCOPE({
             ALLOW_ALLOCATIONS_IN_SCOPE;
             scheduled_jobs.emplace(job, Info{.initial_priority = job->load_priority, .priority = job->load_priority});
+            job->scheduled();
         });
     }
 
@@ -332,7 +369,11 @@ void AsyncLoader::remove(const LoadJobSet & jobs)
     // On the third pass all jobs are finished - remove them all
     // It is better to do it under one lock to avoid exposing intermediate states
     for (const auto & job : jobs)
-        finished_jobs.erase(job);
+    {
+        size_t erased = finished_jobs.erase(job);
+        if (old_jobs >= erased && job->finished_ns && job->finished_ns < busy_period_started_ns)
+            old_jobs -= erased;
+    }
 }
 
 void AsyncLoader::setMaxThreads(size_t value)
@@ -459,6 +500,7 @@ void AsyncLoader::finish(std::unique_lock<std::mutex> & lock, const LoadJobPtr &
     NOEXCEPT_SCOPE({
         ALLOW_ALLOCATIONS_IN_SCOPE;
         finished_jobs.insert(job);
+        logAboutProgress(log, finished_jobs.size() - old_jobs, finished_jobs.size() + scheduled_jobs.size() - old_jobs, stopwatch);
     });
 }
 
@@ -548,7 +590,7 @@ void AsyncLoader::worker()
 
         try
         {
-            job->func(job);
+            job->execute(job);
             exception_from_job = {};
         }
         catch (...)
diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 2958f80d829..c20b504e6b3 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -11,8 +11,12 @@
 #include <boost/noncopyable.hpp>
 #include <base/types.h>
 #include <Common/CurrentMetrics.h>
+#include <Common/Stopwatch.h>
 #include <Common/ThreadPool.h>
 
+
+namespace Poco { class Logger; }
+
 namespace DB
 {
 
@@ -23,6 +27,8 @@ class LoadTask;
 using LoadTaskPtr = std::shared_ptr<LoadTask>;
 class AsyncLoader;
 
+void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, AtomicStopwatch & watch);
+
 // Execution status of a load job.
 enum class LoadStatus
 {
@@ -79,6 +85,9 @@ private:
     void canceled(const std::exception_ptr & ptr);
     void finish();
 
+    void scheduled();
+    void execute(const LoadJobPtr & self);
+
     std::function<void(const LoadJobPtr & self)> func;
     std::atomic<ssize_t> load_priority;
 
@@ -87,6 +96,10 @@ private:
     mutable size_t waiters = 0;
     LoadStatus load_status{LoadStatus::PENDING};
     std::exception_ptr load_exception;
+
+    UInt64 scheduled_ns = 0;
+    UInt64 started_ns = 0;
+    UInt64 finished_ns = 0;
 };
 
 template <class Func>
@@ -277,7 +290,12 @@ private:
     void spawn(std::unique_lock<std::mutex> &);
     void worker();
 
+    // Logging
     const bool log_failures; // Worker should log all exceptions caught from job functions.
+    Poco::Logger * log;
+    UInt64 busy_period_started_ns;
+    AtomicStopwatch stopwatch;
+    size_t old_jobs = 0; // Number of jobs that were finished in previous busy period (for correct progress indication)
 
     mutable std::mutex mutex; // Guards all the fields below.
     bool is_running = false;

From 75021138edfe2c0ab792d41a6529fe2e14d5509c Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Sun, 30 Apr 2023 17:52:05 +0000
Subject: [PATCH 096/689] schedule() jobs recursively with its dependencies

---
 src/Common/AsyncLoader.cpp              | 50 +++++++++++------------
 src/Common/AsyncLoader.h                | 53 ++++++++++++++++++++-----
 src/Common/ErrorCodes.cpp               |  2 +-
 src/Common/tests/gtest_async_loader.cpp |  2 +-
 4 files changed, 68 insertions(+), 39 deletions(-)

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
index 1a4774d984e..e6bd6eb12d4 100644
--- a/src/Common/AsyncLoader.cpp
+++ b/src/Common/AsyncLoader.cpp
@@ -12,7 +12,7 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int ASYNC_LOAD_SCHEDULE_FAILED;
+    extern const int ASYNC_LOAD_CYCLE;
     extern const int ASYNC_LOAD_FAILED;
     extern const int ASYNC_LOAD_CANCELED;
 }
@@ -210,7 +210,7 @@ void AsyncLoader::schedule(const std::vector<LoadTaskPtr> & tasks)
     scheduleImpl(all_jobs);
 }
 
-void AsyncLoader::scheduleImpl(const LoadJobSet & jobs)
+void AsyncLoader::scheduleImpl(const LoadJobSet & input_jobs)
 {
     std::unique_lock lock{mutex};
 
@@ -222,14 +222,12 @@ void AsyncLoader::scheduleImpl(const LoadJobSet & jobs)
         old_jobs = finished_jobs.size();
     }
 
-    // Sanity checks
-    for (const auto & job : jobs)
-    {
-        if (job->status() != LoadStatus::PENDING)
-            throw Exception(ErrorCodes::ASYNC_LOAD_SCHEDULE_FAILED, "Trying to schedule already finished load job '{}'", job->name);
-        if (scheduled_jobs.contains(job))
-            throw Exception(ErrorCodes::ASYNC_LOAD_SCHEDULE_FAILED, "Load job '{}' has been already scheduled", job->name);
-    }
+    // Make set of jobs to schedule:
+    // 1) exclude already scheduled or finished jobs
+    // 2) include pending dependencies, that are not yet scheduled
+    LoadJobSet jobs;
+    for (const auto & job : input_jobs)
+        gatherNotScheduled(job, jobs, lock);
 
     // Ensure scheduled_jobs graph will have no cycles. The only way to get a cycle is to add a cycle, assuming old jobs cannot reference new ones.
     checkCycle(jobs, lock);
@@ -285,22 +283,10 @@ void AsyncLoader::scheduleImpl(const LoadJobSet & jobs)
                 if (dep_status == LoadStatus::OK)
                     continue; // Dependency on already successfully finished job -- it's okay.
 
-                if (dep_status == LoadStatus::PENDING)
-                {
-                    // Dependency on not scheduled pending job -- it's bad.
-                    // Probably, there is an error in `jobs` set: not all jobs were passed to `schedule()` call.
-                    // We are not going to run any dependent job, so cancel them all.
-                    std::exception_ptr e;
-                    NOEXCEPT_SCOPE({
-                        ALLOW_ALLOCATIONS_IN_SCOPE;
-                        e = std::make_exception_ptr(Exception(ErrorCodes::ASYNC_LOAD_CANCELED,
-                            "Load job '{}' -> Load job '{}': not scheduled pending load job (it must be also scheduled), all dependent load jobs are canceled",
-                            job->name,
-                            dep->name));
-                    });
-                    finish(lock, job, LoadStatus::CANCELED, e);
-                    break; // This job is now finished, stop its dependencies processing
-                }
+                // Dependency on not scheduled pending job -- it's bad.
+                // Probably, there is an error in `jobs` set, `gatherNotScheduled()` should have fixed it.
+                chassert(dep_status != LoadStatus::PENDING);
+
                 if (dep_status == LoadStatus::FAILED || dep_status == LoadStatus::CANCELED)
                 {
                     // Dependency on already failed or canceled job -- it's okay. Cancel all dependent jobs.
@@ -324,6 +310,16 @@ void AsyncLoader::scheduleImpl(const LoadJobSet & jobs)
     }
 }
 
+void AsyncLoader::gatherNotScheduled(const LoadJobPtr & job, LoadJobSet & jobs, std::unique_lock<std::mutex> & lock)
+{
+    if (job->status() == LoadStatus::PENDING && !scheduled_jobs.contains(job) && !jobs.contains(job))
+    {
+        jobs.insert(job);
+        for (const auto & dep : job->dependencies)
+            gatherNotScheduled(dep, jobs, lock);
+    }
+}
+
 void AsyncLoader::prioritize(const LoadJobPtr & job, ssize_t new_priority)
 {
     if (!job)
@@ -427,7 +423,7 @@ String AsyncLoader::checkCycleImpl(const LoadJobPtr & job, LoadJobSet & left, Lo
         if (auto chain = checkCycleImpl(dep, left, visited, lock); !chain.empty())
         {
             if (!visited.contains(job)) // Check for cycle end
-                throw Exception(ErrorCodes::ASYNC_LOAD_SCHEDULE_FAILED, "Load job dependency cycle detected: {} -> {}", job->name, chain);
+                throw Exception(ErrorCodes::ASYNC_LOAD_CYCLE, "Load job dependency cycle detected: {} -> {}", job->name, chain);
             else
                 return fmt::format("{} -> {}", job->name, chain); // chain is not a cycle yet -- continue building
         }
diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index c20b504e6b3..1c3d9f0b071 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -134,7 +134,8 @@ public:
     // WARNING: Jobs will never be removed() and are going to be stored as finished jobs until ~AsyncLoader().
     void detach();
 
-    // Return the final jobs in this tasks. This jobs subset should be used as `dependencies` for dependent jobs or tasks.
+    // Return the final jobs in this tasks. This job subset should be used as `dependencies` for dependent jobs or tasks:
+    //   auto load_task = loadSomethingAsync(async_loader, load_after_task.goals(), something);
     const LoadJobSet & goals() const { return goal_jobs.empty() ? jobs : goal_jobs; }
 
 private:
@@ -150,6 +151,37 @@ inline LoadTaskPtr makeLoadTask(AsyncLoader & loader, LoadJobSet && jobs, LoadJo
     return std::make_shared<LoadTask>(loader, std::move(jobs), std::move(goals));
 }
 
+inline void waitLoad(const LoadJobSet & jobs)
+{
+    for (const auto & job : jobs)
+        job->wait();
+}
+
+inline void waitLoad(const LoadTaskPtrs & tasks)
+{
+    for (const auto & task : tasks)
+        waitLoad(task->goals());
+}
+
+inline LoadJobSet getGoals(const LoadTaskPtrs & tasks)
+{
+    LoadJobSet result;
+    for (const auto & task : tasks)
+        result.insert(task->goals().begin(), task->goals().end());
+    return result;
+}
+
+inline LoadJobSet joinJobs(const LoadJobSet & jobs1, const LoadJobSet & jobs2)
+{
+    if (jobs1.empty())
+        return jobs2;
+    if (jobs2.empty())
+        return jobs1;
+    LoadJobSet result;
+    result.insert(jobs1.begin(), jobs1.end());
+    result.insert(jobs2.begin(), jobs2.end());
+    return result;
+}
 
 // `AsyncLoader` is a scheduler for DAG of `LoadJob`s. It tracks dependencies and priorities of jobs.
 // Basic usage example:
@@ -168,7 +200,7 @@ inline LoadTaskPtr makeLoadTask(AsyncLoader & loader, LoadJobSet && jobs, LoadJo
 //
 // AsyncLoader tracks state of all scheduled jobs. Job lifecycle is the following:
 // 1)  Job is constructed with PENDING status and initial priority. The job is placed into a task.
-// 2)  The task is scheduled with all its jobs. A scheduled job may be ready (i.e. have all its dependencies finished) or blocked.
+// 2)  The task is scheduled with all its jobs and their dependencies. A scheduled job may be ready (i.e. have all its dependencies finished) or blocked.
 // 3a) When all dependencies are successfully executed, the job became ready. A ready job is enqueued into the ready queue.
 // 3b) If at least one of the job dependencies is failed or canceled, then this job is canceled (with all it's dependent jobs as well).
 //     On cancellation an ASYNC_LOAD_CANCELED exception is generated and saved inside LoadJob object. The job status is changed to CANCELED.
@@ -189,8 +221,8 @@ inline LoadTaskPtr makeLoadTask(AsyncLoader & loader, LoadJobSet && jobs, LoadJo
 // Note that to avoid priority inversion `job_func` should use `self->priority()` to schedule new jobs in AsyncLoader or any other pool.
 // Value stored in load job priority field is atomic and can be increased even during job execution.
 //
-// When task is scheduled it can contain dependencies on previously scheduled jobs. These jobs can have any status.
-// The only forbidden thing is a dependency on job, that was not scheduled in AsyncLoader yet: all the dependent jobs are immediately canceled.
+// When a task is scheduled it can contain dependencies on previously scheduled jobs. These jobs can have any status. If job A being scheduled depends on
+// another job B that is not yet scheduled, then job B will also be scheduled (even if the task does not contain it).
 class AsyncLoader : private boost::noncopyable
 {
 private:
@@ -241,7 +273,7 @@ public:
 
     AsyncLoader(Metric metric_threads, Metric metric_active_threads, size_t max_threads_, bool log_failures_);
 
-    // WARNING: all LoadTask instances returned by `schedule()` should be destructed before AsyncLoader.
+    // WARNING: all tasks instances should be destructed before associated AsyncLoader.
     ~AsyncLoader();
 
     // Start workers to execute scheduled load jobs.
@@ -256,11 +288,11 @@ public:
     //  - or canceled using ~Task() or remove() later.
     void stop();
 
-    // Schedule all jobs of given task.
-    // Higher priority jobs (with greater `priority` value) are executed earlier.
+    // Schedule all jobs of given `task` and their dependencies (if any, not scheduled yet).
+    // Higher priority jobs (with greater `job->priority()` value) are executed earlier.
     // All dependencies of a scheduled job inherit its priority if it is higher. This way higher priority job
-    // never wait for (blocked by) lower priority jobs (no priority inversion).
-    // Task destructor ensures that all the `jobs` are finished (OK, FAILED or CANCELED)
+    // never wait for (blocked by) lower priority jobs. No priority inversion is possible.
+    // Note that `task` destructor ensures that all its jobs are finished (OK, FAILED or CANCELED)
     // and are removed from AsyncLoader, so it is thread-safe to destroy them.
     void schedule(LoadTask & task);
     void schedule(const LoadTaskPtr & task);
@@ -284,7 +316,8 @@ private:
     void checkCycle(const LoadJobSet & jobs, std::unique_lock<std::mutex> & lock);
     String checkCycleImpl(const LoadJobPtr & job, LoadJobSet & left, LoadJobSet & visited, std::unique_lock<std::mutex> & lock);
     void finish(std::unique_lock<std::mutex> & lock, const LoadJobPtr & job, LoadStatus status, std::exception_ptr exception_from_job = {});
-    void scheduleImpl(const LoadJobSet & jobs);
+    void scheduleImpl(const LoadJobSet & input_jobs);
+    void gatherNotScheduled(const LoadJobPtr & job, LoadJobSet & jobs, std::unique_lock<std::mutex> & lock);
     void prioritize(const LoadJobPtr & job, ssize_t new_priority, std::unique_lock<std::mutex> & lock);
     void enqueue(Info & info, const LoadJobPtr & job, std::unique_lock<std::mutex> & lock);
     void spawn(std::unique_lock<std::mutex> &);
diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp
index 64d07ccc9cf..d185f0e87e9 100644
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@@ -650,7 +650,7 @@
     M(679, IO_URING_SUBMIT_ERROR) \
     M(690, MIXED_ACCESS_PARAMETER_TYPES) \
     M(691, UNKNOWN_ELEMENT_OF_ENUM) \
-    M(692, ASYNC_LOAD_SCHEDULE_FAILED) \
+    M(692, ASYNC_LOAD_CYCLE) \
     M(693, ASYNC_LOAD_FAILED) \
     M(694, ASYNC_LOAD_CANCELED) \
     \
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 0a1eddf54ab..48fdfbfecc4 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -25,7 +25,7 @@ namespace CurrentMetrics
 
 namespace DB::ErrorCodes
 {
-    extern const int ASYNC_LOAD_SCHEDULE_FAILED;
+    extern const int ASYNC_LOAD_CYCLE;
     extern const int ASYNC_LOAD_FAILED;
     extern const int ASYNC_LOAD_CANCELED;
 }

From 05477046c3e9a536f04247558a56feb98d260ab0 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Mon, 1 May 2023 11:51:58 +0000
Subject: [PATCH 097/689] more helpers for AsyncLoader

---
 src/Common/AsyncLoader.h | 48 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 1c3d9f0b071..0ddd0fc6e78 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -151,18 +151,53 @@ inline LoadTaskPtr makeLoadTask(AsyncLoader & loader, LoadJobSet && jobs, LoadJo
     return std::make_shared<LoadTask>(loader, std::move(jobs), std::move(goals));
 }
 
+inline void scheduleLoad(const LoadTaskPtr & task)
+{
+    task->schedule();
+}
+
+inline void scheduleLoad(const LoadTaskPtrs & tasks)
+{
+    for (const auto & task : tasks)
+        task->schedule();
+}
+
+template <class... Args>
+inline void scheduleLoad(Args && ... args)
+{
+    (scheduleLoad(std::forward<Args>(args)), ...);
+}
+
 inline void waitLoad(const LoadJobSet & jobs)
 {
     for (const auto & job : jobs)
         job->wait();
 }
 
+inline void waitLoad(const LoadTaskPtr & task)
+{
+    waitLoad(task->goals());
+}
+
 inline void waitLoad(const LoadTaskPtrs & tasks)
 {
     for (const auto & task : tasks)
         waitLoad(task->goals());
 }
 
+template <class... Args>
+inline void waitLoad(Args && ... args)
+{
+    (waitLoad(std::forward<Args>(args)), ...);
+}
+
+template <class... Args>
+inline void scheduleAndWaitLoad(Args && ... args)
+{
+    scheduleLoad(std::forward<Args>(args)...);
+    waitLoad(std::forward<Args>(args)...);
+}
+
 inline LoadJobSet getGoals(const LoadTaskPtrs & tasks)
 {
     LoadJobSet result;
@@ -183,6 +218,19 @@ inline LoadJobSet joinJobs(const LoadJobSet & jobs1, const LoadJobSet & jobs2)
     return result;
 }
 
+inline LoadTaskPtrs joinTasks(const LoadTaskPtrs & tasks1, const LoadTaskPtrs & tasks2)
+{
+    if (tasks1.empty())
+        return tasks2;
+    if (tasks2.empty())
+        return tasks1;
+    LoadTaskPtrs result;
+    result.reserve(tasks1.size() + tasks2.size());
+    result.insert(result.end(), tasks1.begin(), tasks1.end());
+    result.insert(result.end(), tasks2.begin(), tasks2.end());
+    return result;
+}
+
 // `AsyncLoader` is a scheduler for DAG of `LoadJob`s. It tracks dependencies and priorities of jobs.
 // Basic usage example:
 //     auto job_func = [&] (const LoadJobPtr & self) {

From 18d7aa345a3535a239486f77c02818a300833baa Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Mon, 1 May 2023 12:26:37 +0000
Subject: [PATCH 098/689] fix

---
 src/Common/AsyncLoader.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 0ddd0fc6e78..a4639403d13 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -25,6 +25,7 @@ using LoadJobPtr = std::shared_ptr<LoadJob>;
 using LoadJobSet = std::unordered_set<LoadJobPtr>;
 class LoadTask;
 using LoadTaskPtr = std::shared_ptr<LoadTask>;
+using LoadTaskPtrs = std::vector<LoadTaskPtr>;
 class AsyncLoader;
 
 void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, AtomicStopwatch & watch);

From 0f6ff8d381fc4ab6b51a39b89df5d71a167c0332 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Mon, 1 May 2023 12:39:17 +0000
Subject: [PATCH 099/689] fix link error

---
 src/Databases/TablesLoader.cpp | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/src/Databases/TablesLoader.cpp b/src/Databases/TablesLoader.cpp
index ec0fdd85eec..ea0f2072430 100644
--- a/src/Databases/TablesLoader.cpp
+++ b/src/Databases/TablesLoader.cpp
@@ -25,18 +25,6 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-static constexpr size_t PRINT_MESSAGE_EACH_N_OBJECTS = 256;
-static constexpr size_t PRINT_MESSAGE_EACH_N_SECONDS = 5;
-
-void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, AtomicStopwatch & watch)
-{
-    if (processed % PRINT_MESSAGE_EACH_N_OBJECTS == 0 || watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS))
-    {
-        LOG_INFO(log, "{}%", processed * 100.0 / total);
-        watch.restart();
-    }
-}
-
 TablesLoader::TablesLoader(ContextMutablePtr global_context_, Databases databases_, LoadingStrictnessLevel strictness_mode_)
     : global_context(global_context_)
     , databases(std::move(databases_))

From b701293d46ccb7ba8e05c694bedca4d992eca167 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 25 Apr 2023 22:07:04 +0200
Subject: [PATCH 100/689] support named fault injection and add a simple test

---
 .gitmodules                                   |   3 +
 contrib/CMakeLists.txt                        |   1 +
 contrib/libfiu                                |   1 +
 contrib/libfiu-cmake/CMakeLists.txt           |  16 ++
 src/Access/Common/AccessType.h                |   1 +
 src/CMakeLists.txt                            |   4 +
 src/Common/FailPoint.cpp                      | 195 ++++++++++++++++++
 src/Common/FailPoint.h                        |  47 +++++
 .../ZooKeeper/ZooKeeperWithFaultInjection.h   |  17 +-
 src/Common/config.h.in                        |   1 +
 src/Interpreters/InterpreterSystemQuery.cpp   |  15 ++
 src/Parsers/ASTSystemQuery.h                  |   4 +
 src/Parsers/ParserSystemQuery.cpp             |  11 +
 .../MergeTree/ReplicatedMergeTreeSink.cpp     |  26 +++
 src/configure_config.cmake                    |   3 +
 ...02718_insert_meet_hardware_error.reference |   1 +
 .../02718_insert_meet_hardware_error.sql      |  23 +++
 17 files changed, 367 insertions(+), 2 deletions(-)
 create mode 160000 contrib/libfiu
 create mode 100644 contrib/libfiu-cmake/CMakeLists.txt
 create mode 100644 src/Common/FailPoint.cpp
 create mode 100644 src/Common/FailPoint.h
 create mode 100644 tests/queries/0_stateless/02718_insert_meet_hardware_error.reference
 create mode 100644 tests/queries/0_stateless/02718_insert_meet_hardware_error.sql

diff --git a/.gitmodules b/.gitmodules
index ca55281e643..2c81909b1ab 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -335,3 +335,6 @@
 [submodule "contrib/liburing"]
 	path = contrib/liburing
 	url = https://github.com/axboe/liburing
+[submodule "contrib/libfiu"]
+	path = contrib/libfiu
+	url = https://github.com/albertito/libfiu.git
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index be3563d2c61..bdb16665580 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -105,6 +105,7 @@ add_contrib (libfarmhash)
 add_contrib (icu-cmake icu)
 add_contrib (h3-cmake h3)
 add_contrib (mariadb-connector-c-cmake mariadb-connector-c)
+add_contrib (libfiu-cmake libfiu)
 
 if (ENABLE_TESTS)
     add_contrib (googletest-cmake googletest)
diff --git a/contrib/libfiu b/contrib/libfiu
new file mode 160000
index 00000000000..7835d077aec
--- /dev/null
+++ b/contrib/libfiu
@@ -0,0 +1 @@
+Subproject commit 7835d077aecf71523ffc8967d2908becba982703
diff --git a/contrib/libfiu-cmake/CMakeLists.txt b/contrib/libfiu-cmake/CMakeLists.txt
new file mode 100644
index 00000000000..eb77816915d
--- /dev/null
+++ b/contrib/libfiu-cmake/CMakeLists.txt
@@ -0,0 +1,16 @@
+set(FIU_DIR "${ClickHouse_SOURCE_DIR}/contrib/libfiu/")
+
+set(FIU_SOURCES
+  ${FIU_DIR}/libfiu/fiu.c
+  ${FIU_DIR}/libfiu/fiu-rc.c
+  ${FIU_DIR}/libfiu/backtrace.c
+  ${FIU_DIR}/libfiu/wtable.c
+  ${FIU_DIR}/libfiu/hash.c
+)
+
+set(FIU_HEADERS "${FIU_DIR}/libfiu")
+
+add_library(fiu ${FIU_SOURCES})
+target_compile_definitions(fiu PUBLIC DUMMY_BACKTRACE)
+target_include_directories(fiu PUBLIC ${FIU_HEADERS})
+add_library(ch_contrib::fiu ALIAS fiu)
diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h
index 57fa75dc67b..1024d904c03 100644
--- a/src/Access/Common/AccessType.h
+++ b/src/Access/Common/AccessType.h
@@ -185,6 +185,7 @@ enum class AccessType
     M(SYSTEM_FLUSH, "", GROUP, SYSTEM) \
     M(SYSTEM_THREAD_FUZZER, "SYSTEM START THREAD FUZZER, SYSTEM STOP THREAD FUZZER, START THREAD FUZZER, STOP THREAD FUZZER", GLOBAL, SYSTEM) \
     M(SYSTEM_UNFREEZE, "SYSTEM UNFREEZE", GLOBAL, SYSTEM) \
+    M(SYSTEM_FAILPOINT, "SYSTEM ENABLE/DISABLE FAILPOINT", GLOBAL, SYSTEM) \
     M(SYSTEM, "", GROUP, ALL) /* allows to execute SYSTEM {SHUTDOWN|RELOAD CONFIG|...} */ \
     \
     M(dictGet, "dictHas, dictGetHierarchy, dictIsIn", DICTIONARY, ALL) /* allows to execute functions dictGet(), dictHas(), dictGetHierarchy(), dictIsIn() */\
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 76e5ef83e41..6e975c1e677 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -352,6 +352,10 @@ target_link_libraries(clickhouse_common_io
             Poco::Foundation
 )
 
+if (TARGET ch_contrib::fiu)
+    target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::fiu)
+endif()
+
 if (TARGET ch_contrib::cpuid)
     target_link_libraries(clickhouse_common_io PRIVATE ch_contrib::cpuid)
 endif()
diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp
new file mode 100644
index 00000000000..824ad33f018
--- /dev/null
+++ b/src/Common/FailPoint.cpp
@@ -0,0 +1,195 @@
+#include <Common/Exception.h>
+#include <Common/FailPoint.h>
+#include <Poco/StringTokenizer.h>
+#include <Poco/Util/LayeredConfiguration.h>
+#include <Common/logger_useful.h>
+
+#include <boost/core/noncopyable.hpp>
+#include <condition_variable>
+#include <mutex>
+#include <optional>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+extern const int LOGICAL_ERROR;
+};
+
+#if FIU_ENABLE
+static struct InitFiu
+{
+    InitFiu()
+    {
+        fiu_init(0);
+    }
+} init_fiu;
+#endif
+
+#define APPLY_FOR_FAILPOINTS_ONCE(M) \
+    M(rmt_commit_zk_fail_before_op) \
+    M(rmt_commit_zk_fail_after_op)
+
+#define APPLY_FOR_FAILPOINTS(M) \
+    M(dummy_failpoint)
+
+#define APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M) \
+    M(dummy_pausable_failpoint_once)
+
+#define APPLY_FOR_PAUSEABLE_FAILPOINTS(M) \
+    M(dummy_pausable_failpoint)
+
+namespace FailPoints
+{
+#define M(NAME) extern const char(NAME)[] = #NAME "";
+APPLY_FOR_FAILPOINTS_ONCE(M)
+APPLY_FOR_FAILPOINTS(M)
+APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M)
+APPLY_FOR_PAUSEABLE_FAILPOINTS(M)
+#undef M
+}
+
+std::unordered_map<String, std::any> FailPointHelper::fail_point_val;
+std::unordered_map<String, std::shared_ptr<FailPointChannel>> FailPointHelper::fail_point_wait_channels;
+class FailPointChannel : private boost::noncopyable
+{
+public:
+    // wake up all waiting threads when destroy
+    ~FailPointChannel() { notifyAll(); }
+
+    explicit FailPointChannel(UInt64 timeout_)
+        : timeout(timeout_)
+    {}
+    FailPointChannel()
+        : timeout(0)
+    {}
+
+    void wait()
+    {
+        std::unique_lock lock(m);
+        if (timeout == 0)
+            cv.wait(lock);
+        else
+            cv.wait_for(lock, std::chrono::seconds(timeout));
+    }
+
+    void notifyAll()
+    {
+        std::unique_lock lock(m);
+        cv.notify_all();
+    }
+
+private:
+    UInt64 timeout;
+    std::mutex m;
+    std::condition_variable cv;
+};
+
+void FailPointHelper::enablePauseFailPoint(const String & fail_point_name, UInt64 time)
+{
+#define SUB_M(NAME, flags)                                                                                  \
+    if (fail_point_name == FailPoints::NAME)                                                                \
+    {                                                                                                       \
+        /* FIU_ONETIME -- Only fail once; the point of failure will be automatically disabled afterwards.*/ \
+        fiu_enable(FailPoints::NAME, 1, nullptr, flags);                                                    \
+        fail_point_wait_channels.try_emplace(FailPoints::NAME, std::make_shared<FailPointChannel>(time));   \
+        return;                                                                                             \
+    }
+
+#define M(NAME) SUB_M(NAME, FIU_ONETIME)
+    APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M)
+#undef M
+
+#define M(NAME) SUB_M(NAME, 0)
+    APPLY_FOR_PAUSEABLE_FAILPOINTS(M)
+#undef M
+#undef SUB_M
+
+    throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find fail point {}", fail_point_name);
+}
+
+void FailPointHelper::enableFailPoint(const String & fail_point_name, std::optional<std::any> v)
+{
+#if FIU_ENABLE
+#define SUB_M(NAME, flags)                                                                                  \
+    if (fail_point_name == FailPoints::NAME)                                                                \
+    {                                                                                                       \
+        /* FIU_ONETIME -- Only fail once; the point of failure will be automatically disabled afterwards.*/ \
+        fiu_enable(FailPoints::NAME, 1, nullptr, flags);                                                    \
+        if (v.has_value())                                                                                  \
+        {                                                                                                   \
+            fail_point_val.try_emplace(FailPoints::NAME, v.value());                                        \
+        }                                                                                                   \
+        return;                                                                                             \
+    }
+
+#define M(NAME) SUB_M(NAME, FIU_ONETIME)
+    APPLY_FOR_FAILPOINTS_ONCE(M)
+#undef M
+#define M(NAME) SUB_M(NAME, 0)
+    APPLY_FOR_FAILPOINTS(M)
+#undef M
+#undef SUB_M
+
+#define SUB_M(NAME, flags)                                                                                  \
+    if (fail_point_name == FailPoints::NAME)                                                                \
+    {                                                                                                       \
+        /* FIU_ONETIME -- Only fail once; the point of failure will be automatically disabled afterwards.*/ \
+        fiu_enable(FailPoints::NAME, 1, nullptr, flags);                                                    \
+        fail_point_wait_channels.try_emplace(FailPoints::NAME, std::make_shared<FailPointChannel>());       \
+        if (v.has_value())                                                                                  \
+        {                                                                                                   \
+            fail_point_val.try_emplace(FailPoints::NAME, v.value());                                        \
+        }                                                                                                   \
+        return;                                                                                             \
+    }
+
+#define M(NAME) SUB_M(NAME, FIU_ONETIME)
+    APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M)
+#undef M
+
+#define M(NAME) SUB_M(NAME, 0)
+    APPLY_FOR_PAUSEABLE_FAILPOINTS(M)
+#undef M
+#undef SUB_M
+
+#endif
+    throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find fail point {}", fail_point_name);
+}
+
+std::optional<std::any>
+FailPointHelper::getFailPointVal(const String & fail_point_name)
+{
+    if (auto iter = fail_point_val.find(fail_point_name); iter != fail_point_val.end())
+    {
+        return iter->second;
+    }
+    return std::nullopt;
+}
+
+void FailPointHelper::disableFailPoint(const String & fail_point_name)
+{
+    if (auto iter = fail_point_wait_channels.find(fail_point_name); iter != fail_point_wait_channels.end())
+    {
+        /// can not rely on deconstruction to do the notify_all things, because
+        /// if someone wait on this, the deconstruct will never be called.
+        iter->second->notifyAll();
+        fail_point_wait_channels.erase(iter);
+    }
+    fail_point_val.erase(fail_point_name);
+    fiu_disable(fail_point_name.c_str());
+}
+
+void FailPointHelper::wait(const String & fail_point_name)
+{
+    if (auto iter = fail_point_wait_channels.find(fail_point_name); iter == fail_point_wait_channels.end())
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Can not find channel for fail point {}", fail_point_name);
+    else
+    {
+        auto ptr = iter->second;
+        ptr->wait();
+    }
+}
+
+}
diff --git a/src/Common/FailPoint.h b/src/Common/FailPoint.h
new file mode 100644
index 00000000000..86dbf70d9c2
--- /dev/null
+++ b/src/Common/FailPoint.h
@@ -0,0 +1,47 @@
+#pragma once
+#include "config.h"
+
+#include <Common/Exception.h>
+#include <Core/Types.h>
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdocumentation"
+#pragma clang diagnostic ignored "-Wreserved-macro-identifier"
+#endif
+
+#include <fiu.h>
+#include <fiu-control.h>
+
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
+#include <any>
+#include <unordered_map>
+
+namespace DB
+{
+
+// When `fail_point` is enabled, wait till it is disabled
+#define FAIL_POINT_PAUSE(fail_point) fiu_do_on(fail_point, FailPointHelper::wait(fail_point);)
+
+class FailPointChannel;
+class FailPointHelper
+{
+public:
+    static void enableFailPoint(const String & fail_point_name, std::optional<std::any> v = std::nullopt);
+
+    static std::optional<std::any> getFailPointVal(const String & fail_point_name);
+
+    static void enablePauseFailPoint(const String & fail_point_name, UInt64 time);
+
+    static void disableFailPoint(const String & fail_point_name);
+
+    static void wait(const String & fail_point_name);
+
+private:
+    static std::unordered_map<String, std::shared_ptr<FailPointChannel>> fail_point_wait_channels;
+    static std::unordered_map<String, std::any> fail_point_val;
+};
+}
diff --git a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h
index 130590ceb40..3f45c300afa 100644
--- a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h
+++ b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h
@@ -18,17 +18,26 @@ namespace ErrorCodes
 class RandomFaultInjection
 {
 public:
+    bool must_fail_after_op = false;
+    bool must_fail_before_op = false;
+
     RandomFaultInjection(double probability, UInt64 seed_) : rndgen(seed_), distribution(probability) { }
 
     void beforeOperation()
     {
-        if (distribution(rndgen))
+        if (distribution(rndgen) || must_fail_before_op)
+        {
+            must_fail_before_op = false;
             throw zkutil::KeeperException("Fault injection before operation", Coordination::Error::ZSESSIONEXPIRED);
+        }
     }
     void afterOperation()
     {
-        if (distribution(rndgen))
+        if (distribution(rndgen) || must_fail_after_op)
+        {
+            must_fail_after_op = false;
             throw zkutil::KeeperException("Fault injection after operation", Coordination::Error::ZOPERATIONTIMEOUT);
+        }
     }
 
 private:
@@ -41,6 +50,9 @@ private:
 ///
 class ZooKeeperWithFaultInjection
 {
+    template<bool async_insert>
+    friend class ReplicatedMergeTreeSinkImpl;
+
     using zk = zkutil::ZooKeeper;
 
     zk::Ptr keeper;
@@ -74,6 +86,7 @@ class ZooKeeperWithFaultInjection
     }
 
 public:
+
     using Ptr = std::shared_ptr<ZooKeeperWithFaultInjection>;
 
     static ZooKeeperWithFaultInjection::Ptr createInstance(
diff --git a/src/Common/config.h.in b/src/Common/config.h.in
index d5080ba447e..d6b85f9e28c 100644
--- a/src/Common/config.h.in
+++ b/src/Common/config.h.in
@@ -57,3 +57,4 @@
 #cmakedefine01 USE_SKIM
 #cmakedefine01 USE_OPENSSL_INTREE
 #cmakedefine01 USE_ULID
+#cmakedefine01 FIU_ENABLE
diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp
index 36cb57c3678..41a3662ebba 100644
--- a/src/Interpreters/InterpreterSystemQuery.cpp
+++ b/src/Interpreters/InterpreterSystemQuery.cpp
@@ -8,6 +8,7 @@
 #include <Common/escapeForFileName.h>
 #include <Common/ShellCommand.h>
 #include <Common/CurrentMetrics.h>
+#include <Common/FailPoint.h>
 #include <Interpreters/Cache/FileCacheFactory.h>
 #include <Interpreters/Cache/FileCache.h>
 #include <Interpreters/Context.h>
@@ -582,6 +583,18 @@ BlockIO InterpreterSystemQuery::execute()
             result = Unfreezer(getContext()).systemUnfreeze(query.backup_name);
             break;
         }
+        case Type::ENABLE_FAILPOINT:
+        {
+            getContext()->checkAccess(AccessType::SYSTEM_FAILPOINT);
+            FailPointHelper::enableFailPoint(query.fail_point_name);
+            break;
+        }
+        case Type::DISABLE_FAILPOINT:
+        {
+            getContext()->checkAccess(AccessType::SYSTEM_FAILPOINT);
+            FailPointHelper::disableFailPoint(query.fail_point_name);
+            break;
+        }
         default:
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown type of SYSTEM query");
     }
@@ -1175,6 +1188,8 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster()
         case Type::START_LISTEN_QUERIES:
         case Type::STOP_THREAD_FUZZER:
         case Type::START_THREAD_FUZZER:
+        case Type::ENABLE_FAILPOINT:
+        case Type::DISABLE_FAILPOINT:
         case Type::UNKNOWN:
         case Type::END: break;
     }
diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h
index dfe2389edb7..c9315f68394 100644
--- a/src/Parsers/ASTSystemQuery.h
+++ b/src/Parsers/ASTSystemQuery.h
@@ -77,6 +77,8 @@ public:
         START_THREAD_FUZZER,
         STOP_THREAD_FUZZER,
         UNFREEZE,
+        ENABLE_FAILPOINT,
+        DISABLE_FAILPOINT,
         END
     };
 
@@ -109,6 +111,8 @@ public:
 
     String schema_cache_storage;
 
+    String fail_point_name;
+
     SyncReplicaMode sync_replica_mode = SyncReplicaMode::DEFAULT;
 
     String getID(char) const override { return "SYSTEM query"; }
diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp
index 7d41ac0e5d3..a1e9030df9d 100644
--- a/src/Parsers/ParserSystemQuery.cpp
+++ b/src/Parsers/ParserSystemQuery.cpp
@@ -140,6 +140,7 @@ enum class SystemQueryTargetType
                                     Expected & expected)
 {
     String cluster_str;
+
     if (ParserKeyword{"ON"}.ignore(pos, expected))
     {
         if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected))
@@ -250,6 +251,16 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected &
                 return false;
             break;
         }
+        case Type::ENABLE_FAILPOINT:
+        case Type::DISABLE_FAILPOINT:
+        {
+            ASTPtr ast;
+            if (ParserIdentifier{}.parse(pos, ast, expected))
+                res->fail_point_name = ast->as<ASTIdentifier &>().name();
+            else
+                return false;
+            break;
+        }
 
         case Type::RESTART_REPLICA:
         case Type::SYNC_REPLICA:
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index 6941e9fbc5c..eab657c01e6 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -2,6 +2,7 @@
 #include <Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h>
 #include <Storages/MergeTree/ReplicatedMergeTreeSink.h>
 #include <Interpreters/PartLog.h>
+#include <Common/FailPoint.h>
 #include <Common/ProfileEventsScope.h>
 #include <Common/SipHash.h>
 #include <Common/ZooKeeper/KeeperException.h>
@@ -20,6 +21,12 @@ namespace ProfileEvents
 namespace DB
 {
 
+namespace FailPoints
+{
+    extern const char rmt_commit_zk_fail_before_op[];
+    extern const char rmt_commit_zk_fail_after_op[];
+}
+
 namespace ErrorCodes
 {
     extern const int TOO_FEW_LIVE_REPLICAS;
@@ -940,6 +947,25 @@ std::vector<String> ReplicatedMergeTreeSinkImpl<async_insert>::commitPart(
 
         ThreadFuzzer::maybeInjectSleep();
 
+        fiu_do_on(FailPoints::rmt_commit_zk_fail_before_op,
+        {
+            if (!zookeeper->fault_policy)
+            {
+                zookeeper->logger = log;
+                zookeeper->fault_policy = std::make_unique<RandomFaultInjection>(0, 0);
+            }
+            zookeeper->fault_policy->must_fail_before_op = true;
+        });
+        fiu_do_on(FailPoints::rmt_commit_zk_fail_after_op,
+        {
+            if (!zookeeper->fault_policy)
+            {
+                zookeeper->logger = log;
+                zookeeper->fault_policy = std::make_unique<RandomFaultInjection>(0, 0);
+            }
+            zookeeper->fault_policy->must_fail_after_op = true;
+        });
+
         Coordination::Responses responses;
         Coordination::Error multi_code = zookeeper->tryMultiNoThrow(ops, responses); /// 1 RTT
         if (multi_code == Coordination::Error::ZOK)
diff --git a/src/configure_config.cmake b/src/configure_config.cmake
index 9cb0d0efb39..fc0b8dcb07d 100644
--- a/src/configure_config.cmake
+++ b/src/configure_config.cmake
@@ -153,3 +153,6 @@ endif ()
 if (ENABLE_OPENSSL)
     set(USE_OPENSSL_INTREE 1)
 endif ()
+if (TARGET ch_contrib::fiu)
+    set(FIU_ENABLE 1)
+endif()
diff --git a/tests/queries/0_stateless/02718_insert_meet_hardware_error.reference b/tests/queries/0_stateless/02718_insert_meet_hardware_error.reference
new file mode 100644
index 00000000000..f599e28b8ab
--- /dev/null
+++ b/tests/queries/0_stateless/02718_insert_meet_hardware_error.reference
@@ -0,0 +1 @@
+10
diff --git a/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql b/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql
new file mode 100644
index 00000000000..dcacf9fafca
--- /dev/null
+++ b/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql
@@ -0,0 +1,23 @@
+-- Tags: zookeeper
+
+system enable failpoint rmt_commit_zk_fail_before_op;
+
+DROP TABLE IF EXISTS t_hardware_error NO DELAY;
+
+CREATE TABLE t_hardware_error (
+	KeyID UInt32
+) Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/t_async_insert_dedup', '{replica}')
+ORDER BY (KeyID);
+
+-- should not report errors because we retry on hardware_error
+insert into t_hardware_error values (1), (2), (3), (4), (5); 
+
+system disable failpoint rmt_commit_zk_fail_before_op;
+
+system enable failpoint rmt_commit_zk_fail_after_op;
+
+insert into t_hardware_error values (6), (7), (8), (9), (10); 
+
+select count() from t_hardware_error;
+
+system disable failpoint rmt_commit_zk_fail_after_op;

From d608ba41e739835612335cfbfc0c02470649d869 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 2 May 2023 00:05:36 +0200
Subject: [PATCH 101/689] try to fix

---
 CMakeLists.txt                                      |  2 ++
 contrib/libfiu-cmake/CMakeLists.txt                 | 13 +++++++++----
 docker/test/fasttest/run.sh                         |  1 +
 src/Common/FailPoint.cpp                            |  4 ----
 src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h  |  1 -
 src/Parsers/ParserSystemQuery.cpp                   |  1 -
 src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp  | 10 ----------
 .../02718_insert_meet_hardware_error.sql            |  7 ++-----
 8 files changed, 14 insertions(+), 25 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c6ca18773a7..91718f4e867 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -407,6 +407,8 @@ if ((NOT OS_LINUX AND NOT OS_ANDROID) OR (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG"))
     set(ENABLE_GWP_ASAN OFF)
 endif ()
 
+option (ENABLE_FIU "Enable Fiu" ON)
+
 option(WERROR "Enable -Werror compiler option" ON)
 
 if (WERROR)
diff --git a/contrib/libfiu-cmake/CMakeLists.txt b/contrib/libfiu-cmake/CMakeLists.txt
index eb77816915d..6d7f86c7a8f 100644
--- a/contrib/libfiu-cmake/CMakeLists.txt
+++ b/contrib/libfiu-cmake/CMakeLists.txt
@@ -1,3 +1,8 @@
+if (NOT ENABLE_FIU)
+  message (STATUS "Not using gwp-asan")
+  return ()
+endif ()
+
 set(FIU_DIR "${ClickHouse_SOURCE_DIR}/contrib/libfiu/")
 
 set(FIU_SOURCES
@@ -10,7 +15,7 @@ set(FIU_SOURCES
 
 set(FIU_HEADERS "${FIU_DIR}/libfiu")
 
-add_library(fiu ${FIU_SOURCES})
-target_compile_definitions(fiu PUBLIC DUMMY_BACKTRACE)
-target_include_directories(fiu PUBLIC ${FIU_HEADERS})
-add_library(ch_contrib::fiu ALIAS fiu)
+add_library(_fiu ${FIU_SOURCES})
+target_compile_definitions(_fiu PUBLIC DUMMY_BACKTRACE)
+target_include_directories(_fiu PUBLIC ${FIU_HEADERS})
+add_library(ch_contrib::fiu ALIAS _fiu)
diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index 3ed0c4df093..6b320a20665 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -147,6 +147,7 @@ function clone_submodules
             contrib/xxHash
             contrib/simdjson
             contrib/liburing
+	    contrib/libfiu
         )
 
         git submodule sync
diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp
index 824ad33f018..1048b0ecd2c 100644
--- a/src/Common/FailPoint.cpp
+++ b/src/Common/FailPoint.cpp
@@ -1,8 +1,5 @@
 #include <Common/Exception.h>
 #include <Common/FailPoint.h>
-#include <Poco/StringTokenizer.h>
-#include <Poco/Util/LayeredConfiguration.h>
-#include <Common/logger_useful.h>
 
 #include <boost/core/noncopyable.hpp>
 #include <condition_variable>
@@ -28,7 +25,6 @@ static struct InitFiu
 #endif
 
 #define APPLY_FOR_FAILPOINTS_ONCE(M) \
-    M(rmt_commit_zk_fail_before_op) \
     M(rmt_commit_zk_fail_after_op)
 
 #define APPLY_FOR_FAILPOINTS(M) \
diff --git a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h
index 3f45c300afa..ec1918a5d85 100644
--- a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h
+++ b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h
@@ -86,7 +86,6 @@ class ZooKeeperWithFaultInjection
     }
 
 public:
-
     using Ptr = std::shared_ptr<ZooKeeperWithFaultInjection>;
 
     static ZooKeeperWithFaultInjection::Ptr createInstance(
diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp
index a1e9030df9d..247e715d3c3 100644
--- a/src/Parsers/ParserSystemQuery.cpp
+++ b/src/Parsers/ParserSystemQuery.cpp
@@ -140,7 +140,6 @@ enum class SystemQueryTargetType
                                     Expected & expected)
 {
     String cluster_str;
-
     if (ParserKeyword{"ON"}.ignore(pos, expected))
     {
         if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected))
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index eab657c01e6..4109194398d 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -23,7 +23,6 @@ namespace DB
 
 namespace FailPoints
 {
-    extern const char rmt_commit_zk_fail_before_op[];
     extern const char rmt_commit_zk_fail_after_op[];
 }
 
@@ -947,15 +946,6 @@ std::vector<String> ReplicatedMergeTreeSinkImpl<async_insert>::commitPart(
 
         ThreadFuzzer::maybeInjectSleep();
 
-        fiu_do_on(FailPoints::rmt_commit_zk_fail_before_op,
-        {
-            if (!zookeeper->fault_policy)
-            {
-                zookeeper->logger = log;
-                zookeeper->fault_policy = std::make_unique<RandomFaultInjection>(0, 0);
-            }
-            zookeeper->fault_policy->must_fail_before_op = true;
-        });
         fiu_do_on(FailPoints::rmt_commit_zk_fail_after_op,
         {
             if (!zookeeper->fault_policy)
diff --git a/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql b/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql
index dcacf9fafca..72d22d9d5b2 100644
--- a/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql
+++ b/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql
@@ -1,7 +1,5 @@
 -- Tags: zookeeper
 
-system enable failpoint rmt_commit_zk_fail_before_op;
-
 DROP TABLE IF EXISTS t_hardware_error NO DELAY;
 
 CREATE TABLE t_hardware_error (
@@ -9,11 +7,8 @@ CREATE TABLE t_hardware_error (
 ) Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/t_async_insert_dedup', '{replica}')
 ORDER BY (KeyID);
 
--- should not report errors because we retry on hardware_error
 insert into t_hardware_error values (1), (2), (3), (4), (5); 
 
-system disable failpoint rmt_commit_zk_fail_before_op;
-
 system enable failpoint rmt_commit_zk_fail_after_op;
 
 insert into t_hardware_error values (6), (7), (8), (9), (10); 
@@ -21,3 +16,5 @@ insert into t_hardware_error values (6), (7), (8), (9), (10);
 select count() from t_hardware_error;
 
 system disable failpoint rmt_commit_zk_fail_after_op;
+
+DROP TABLE t_hardware_error NO DELAY;

From 2964ca5c84ff2f782145d488edca5d270698020b Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Mon, 1 May 2023 20:24:02 -0400
Subject: [PATCH 102/689] Specify grants for users in `users.xml`

---
 src/Access/UsersConfigAccessStorage.cpp       | 125 +++++++++++++-----
 src/Parsers/Access/ParserGrantQuery.cpp       |   2 +-
 src/Parsers/Access/ParserGrantQuery.h         |   3 +
 .../test_user_grants_from_config/__init__.py  |   0
 .../configs/another_user.xml                  |  18 +++
 .../test_user_grants_from_config/test.py      |  53 ++++++++
 6 files changed, 170 insertions(+), 31 deletions(-)
 create mode 100644 tests/integration/test_user_grants_from_config/__init__.py
 create mode 100644 tests/integration/test_user_grants_from_config/configs/another_user.xml
 create mode 100644 tests/integration/test_user_grants_from_config/test.py

diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp
index 562df61e8aa..e25d2422179 100644
--- a/src/Access/UsersConfigAccessStorage.cpp
+++ b/src/Access/UsersConfigAccessStorage.cpp
@@ -11,6 +11,10 @@
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/quoteString.h>
 #include <Core/Settings.h>
+#include <Interpreters/executeQuery.h>
+#include <Parsers/Access/ASTGrantQuery.h>
+#include <Parsers/Access/ParserGrantQuery.h>
+#include <Parsers/parseQuery.h>
 #include <Poco/Util/AbstractConfiguration.h>
 #include <Poco/MD5Engine.h>
 #include <Poco/JSON/JSON.h>
@@ -49,7 +53,12 @@ namespace
     UUID generateID(const IAccessEntity & entity) { return generateID(entity.getType(), entity.getName()); }
 
 
-    UserPtr parseUser(const Poco::Util::AbstractConfiguration & config, const String & user_name, const std::unordered_set<UUID> & allowed_profile_ids, bool allow_no_password, bool allow_plaintext_password)
+    UserPtr parseUser(
+        const Poco::Util::AbstractConfiguration & config,
+        const String & user_name,
+        const std::unordered_set<UUID> & allowed_profile_ids,
+        bool allow_no_password,
+        bool allow_plaintext_password)
     {
         auto user = std::make_shared<User>();
         user->setName(user_name);
@@ -207,42 +216,94 @@ namespace
             }
         }
 
-        /// By default all databases are accessible
-        /// and the user can grant everything he has.
-        user->access.grantWithGrantOption(AccessType::ALL);
-
-        if (databases)
+        const auto grants_config = user_config + ".grants";
+        std::optional<Strings> grant_queries;
+        if (config.has(grants_config))
         {
-            user->access.revoke(AccessFlags::allFlags() - AccessFlags::allGlobalFlags());
-            user->access.grantWithGrantOption(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG);
-            for (const String & database : *databases)
-                user->access.grantWithGrantOption(AccessFlags::allFlags(), database);
+            Poco::Util::AbstractConfiguration::Keys keys;
+            config.keys(grants_config, keys);
+            grant_queries.emplace();
+            grant_queries->reserve(keys.size());
+            for (const auto & key : keys)
+            {
+                const auto query = config.getString(grants_config + "." + key);
+                grant_queries->push_back(query);
+            }
         }
 
-        if (dictionaries)
-        {
-            user->access.revoke(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG);
-            for (const String & dictionary : *dictionaries)
-                user->access.grantWithGrantOption(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG, dictionary);
-        }
+        if (grant_queries && databases)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Both `grants` and `allow_databases` can't be specified");
 
-        bool access_management = config.getBool(user_config + ".access_management", false);
-        if (!access_management)
+        if (grant_queries)
         {
-            user->access.revoke(AccessType::ACCESS_MANAGEMENT);
-            user->access.revokeGrantOption(AccessType::ALL);
-        }
+            ParserGrantQuery parser;
+            parser.parseWithoutGrantees();
 
-        bool named_collection_control = config.getBool(user_config + ".named_collection_control", false);
-        if (!named_collection_control)
-        {
-            user->access.revoke(AccessType::NAMED_COLLECTION_CONTROL);
-        }
+            for (const auto & string_query : *grant_queries)
+            {
+                String error_message;
+                const char * pos = string_query.data();
+                auto ast = tryParseQuery(parser, pos, pos + string_query.size(), error_message, false, "", false, 0, 0);
 
-        bool show_named_collections_secrets = config.getBool(user_config + ".show_named_collections_secrets", false);
-        if (!show_named_collections_secrets)
+                if (!ast)
+                    throw Exception(ErrorCodes::BAD_ARGUMENTS, "Failed to parse grant query. Error: {}", error_message);
+
+                auto & query = ast->as<ASTGrantQuery &>();
+
+                if (query.roles)
+                    throw Exception(ErrorCodes::BAD_ARGUMENTS, "Roles can't be granted in config file");
+
+                if (!query.cluster.empty())
+                    throw Exception(ErrorCodes::BAD_ARGUMENTS, "Can't grant on cluster using config file");
+
+                for (auto & element : query.access_rights_elements)
+                {
+                    if (query.is_revoke)
+                        user->access.revoke(element);
+                    else
+                        user->access.grant(element);
+                }
+            }
+        }
+        else
         {
-            user->access.revoke(AccessType::SHOW_NAMED_COLLECTIONS_SECRETS);
+            /// By default all databases are accessible
+            /// and the user can grant everything he has.
+            user->access.grantWithGrantOption(AccessType::ALL);
+
+            if (databases)
+            {
+                user->access.revoke(AccessFlags::allFlags() - AccessFlags::allGlobalFlags());
+                user->access.grantWithGrantOption(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG);
+                for (const String & database : *databases)
+                    user->access.grantWithGrantOption(AccessFlags::allFlags(), database);
+            }
+
+            if (dictionaries)
+            {
+                user->access.revoke(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG);
+                for (const String & dictionary : *dictionaries)
+                    user->access.grantWithGrantOption(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG, dictionary);
+            }
+
+            bool access_management = config.getBool(user_config + ".access_management", false);
+            if (!access_management)
+            {
+                user->access.revoke(AccessType::ACCESS_MANAGEMENT);
+                user->access.revokeGrantOption(AccessType::ALL);
+            }
+
+            bool named_collection_control = config.getBool(user_config + ".named_collection_control", false);
+            if (!named_collection_control)
+            {
+                user->access.revoke(AccessType::NAMED_COLLECTION_CONTROL);
+            }
+
+            bool show_named_collections_secrets = config.getBool(user_config + ".show_named_collections_secrets", false);
+            if (!show_named_collections_secrets)
+            {
+                user->access.revoke(AccessType::SHOW_NAMED_COLLECTIONS_SECRETS);
+            }
         }
 
         String default_database = config.getString(user_config + ".default_database", "");
@@ -252,7 +313,11 @@ namespace
     }
 
 
-    std::vector<AccessEntityPtr> parseUsers(const Poco::Util::AbstractConfiguration & config, const std::unordered_set<UUID> & allowed_profile_ids, bool allow_no_password, bool allow_plaintext_password)
+    std::vector<AccessEntityPtr> parseUsers(
+        const Poco::Util::AbstractConfiguration & config,
+        const std::unordered_set<UUID> & allowed_profile_ids,
+        bool allow_no_password,
+        bool allow_plaintext_password)
     {
         Poco::Util::AbstractConfiguration::Keys user_names;
         config.keys("users", user_names);
diff --git a/src/Parsers/Access/ParserGrantQuery.cpp b/src/Parsers/Access/ParserGrantQuery.cpp
index 6869d665385..f495cb60786 100644
--- a/src/Parsers/Access/ParserGrantQuery.cpp
+++ b/src/Parsers/Access/ParserGrantQuery.cpp
@@ -332,7 +332,7 @@ bool ParserGrantQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         parseOnCluster(pos, expected, cluster);
 
     std::shared_ptr<ASTRolesOrUsersSet> grantees;
-    if (!parseToGrantees(pos, expected, is_revoke, grantees))
+    if (!parseToGrantees(pos, expected, is_revoke, grantees) && !allow_no_grantees)
         return false;
 
     if (cluster.empty())
diff --git a/src/Parsers/Access/ParserGrantQuery.h b/src/Parsers/Access/ParserGrantQuery.h
index b14f175c12b..97a8510bc0a 100644
--- a/src/Parsers/Access/ParserGrantQuery.h
+++ b/src/Parsers/Access/ParserGrantQuery.h
@@ -14,11 +14,14 @@ class ParserGrantQuery : public IParserBase
 public:
     ParserGrantQuery & useAttachMode(bool attach_mode_ = true) { attach_mode = attach_mode_; return *this; }
 
+    ParserGrantQuery & parseWithoutGrantees(bool allow_no_grantees_ = true) { allow_no_grantees = allow_no_grantees_; return *this; }
+
 protected:
     const char * getName() const override { return "GRANT or REVOKE query"; }
     bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
 
 private:
     bool attach_mode = false;
+    bool allow_no_grantees = false;
 };
 }
diff --git a/tests/integration/test_user_grants_from_config/__init__.py b/tests/integration/test_user_grants_from_config/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_user_grants_from_config/configs/another_user.xml b/tests/integration/test_user_grants_from_config/configs/another_user.xml
new file mode 100644
index 00000000000..16c026e81d0
--- /dev/null
+++ b/tests/integration/test_user_grants_from_config/configs/another_user.xml
@@ -0,0 +1,18 @@
+<clickhouse>
+    <users>
+        <another>
+            <password/>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <profile>default</profile>
+            <quota>default</quota>
+            <grants>
+                <query>GRANT SHOW ON *.*</query>
+                <query>GRANT CREATE ON *.* WITH GRANT OPTION</query>
+                <query>GRANT SELECT ON system.*</query>
+                <query>REVOKE CREATE ON system.*</query>
+            </grants>
+        </another>
+    </users>
+</clickhouse>
diff --git a/tests/integration/test_user_grants_from_config/test.py b/tests/integration/test_user_grants_from_config/test.py
new file mode 100644
index 00000000000..a4d5c0c904b
--- /dev/null
+++ b/tests/integration/test_user_grants_from_config/test.py
@@ -0,0 +1,53 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+from helpers.test_tools import TSV
+
+cluster = ClickHouseCluster(__file__)
+node = cluster.add_instance(
+    "node",
+    user_configs=[
+        "configs/another_user.xml",
+    ],
+)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def started_cluster():
+    try:
+        cluster.start()
+        node.query("CREATE DATABASE mydb")
+        node.query("CREATE TABLE mydb.table1(x UInt32) ENGINE=Log")
+        node.query("CREATE TABLE mydb.table2(x UInt32) ENGINE=Log")
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def test_allow_read_from_system_tables():
+    assert node.query("SELECT name FROM system.tables WHERE database = 'mydb'") == TSV(
+        [
+            "table1",
+            "table2",
+        ]
+    )
+
+    assert node.query(
+        "SELECT name FROM system.tables WHERE database = 'mydb'", user="another"
+    ) == TSV(
+        [
+            "table1",
+            "table2",
+        ]
+    )
+
+
+def test_grants_from_config():
+    assert node.query("SHOW GRANTS FOR another") == TSV(
+        [
+            "GRANT SHOW ON *.* TO another",
+            "GRANT CREATE ON *.* TO another WITH GRANT OPTION",
+            "GRANT SELECT ON system.* TO another",
+            "REVOKE CREATE DATABASE, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY ON system.* FROM another",
+        ]
+    )

From a0624cae8a4a19dbf68947719699c0d9535203c7 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 2 May 2023 10:45:41 +0200
Subject: [PATCH 103/689] fix build

---
 .gitmodules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitmodules b/.gitmodules
index d19931a55d7..e4524236524 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -337,7 +337,7 @@
 	url = https://github.com/axboe/liburing
 [submodule "contrib/libfiu"]
 	path = contrib/libfiu
-	url = https://github.com/albertito/libfiu.git
+	url = https://github.com/hanfei1991/libfiu.git
 [submodule "contrib/isa-l"]
 	path = contrib/isa-l
 	url = https://github.com/ClickHouse/isa-l.git

From f3b97f9d1f99a4b6f76e530b317e90a330fc7083 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Tue, 2 May 2023 08:44:03 +0000
Subject: [PATCH 104/689] improve helper functions

---
 src/Common/AsyncLoader.h                | 38 ++++++++++++++++---------
 src/Common/tests/gtest_async_loader.cpp | 34 +++++++++++-----------
 2 files changed, 42 insertions(+), 30 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index a4639403d13..d35896942d0 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -45,19 +45,14 @@ enum class LoadStatus
 class LoadJob : private boost::noncopyable
 {
 public:
-    template <class Func>
-    LoadJob(LoadJobSet && dependencies_, String name_, ssize_t priority_, Func && func_)
-        : dependencies(std::move(dependencies_))
+    template <class Func, class LoadJobSetType>
+    LoadJob(LoadJobSetType && dependencies_, String name_, Func && func_, ssize_t priority_ = 0)
+        : dependencies(std::forward<LoadJobSetType>(dependencies_))
         , name(std::move(name_))
         , func(std::forward<Func>(func_))
         , load_priority(priority_)
     {}
 
-    template <class Func>
-    LoadJob(LoadJobSet && dependencies_, String name_, Func && func_)
-        : LoadJob(std::move(dependencies_), std::move(name_), 0, std::forward<Func>(func_))
-    {}
-
     // Current job status.
     LoadStatus status() const;
     std::exception_ptr exception() const;
@@ -103,16 +98,33 @@ private:
     UInt64 finished_ns = 0;
 };
 
-template <class Func>
-LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, Func && func)
+struct EmptyJobFunc
+{
+    void operator()(const LoadJobPtr &) {}
+};
+
+template <class Func = EmptyJobFunc>
+LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, Func && func = EmptyJobFunc())
 {
     return std::make_shared<LoadJob>(std::move(dependencies), std::move(name), std::forward<Func>(func));
 }
 
-template <class Func>
-LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, ssize_t priority, Func && func)
+template <class Func = EmptyJobFunc>
+LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, String name, Func && func = EmptyJobFunc())
 {
-    return std::make_shared<LoadJob>(std::move(dependencies), std::move(name), priority, std::forward<Func>(func));
+    return std::make_shared<LoadJob>(dependencies, std::move(name), std::forward<Func>(func));
+}
+
+template <class Func = EmptyJobFunc>
+LoadJobPtr makeLoadJob(LoadJobSet && dependencies, ssize_t priority, String name, Func && func = EmptyJobFunc())
+{
+    return std::make_shared<LoadJob>(std::move(dependencies), std::move(name), std::forward<Func>(func), priority);
+}
+
+template <class Func = EmptyJobFunc>
+LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, ssize_t priority, String name, Func && func = EmptyJobFunc())
+{
+    return std::make_shared<LoadJob>(dependencies, std::move(name), std::forward<Func>(func), priority);
 }
 
 // Represents a logically connected set of LoadJobs required to achieve some goals (final LoadJob in the set).
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 48fdfbfecc4..96b6e981c65 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -135,7 +135,7 @@ TEST(AsyncLoader, Smoke)
         auto job3 = makeLoadJob({ job2 }, "job3", job_func);
         auto job4 = makeLoadJob({ job2 }, "job4", job_func);
         auto task2 = t.schedule({ job3, job4 });
-        auto job5 = makeLoadJob({ job3, job4 }, "job5", low_priority, job_func);
+        auto job5 = makeLoadJob({ job3, job4 }, low_priority, "job5", job_func);
         task2->merge(t.schedule({ job5 }));
 
         std::thread waiter_thread([=] { job5->wait(); });
@@ -572,14 +572,14 @@ TEST(AsyncLoader, StaticPriorities)
     };
 
     std::vector<LoadJobPtr> jobs;
-    jobs.push_back(makeLoadJob({}, "A", 0, job_func)); // 0
-    jobs.push_back(makeLoadJob({ jobs[0] }, "B", 3, job_func)); // 1
-    jobs.push_back(makeLoadJob({ jobs[0] }, "C", 4, job_func)); // 2
-    jobs.push_back(makeLoadJob({ jobs[0] }, "D", 1, job_func)); // 3
-    jobs.push_back(makeLoadJob({ jobs[0] }, "E", 2, job_func)); // 4
-    jobs.push_back(makeLoadJob({ jobs[3], jobs[4] }, "F", 0, job_func)); // 5
-    jobs.push_back(makeLoadJob({ jobs[5] }, "G", 0, job_func)); // 6
-    jobs.push_back(makeLoadJob({ jobs[6] }, "H", 9, job_func)); // 7
+    jobs.push_back(makeLoadJob({}, 0, "A", job_func)); // 0
+    jobs.push_back(makeLoadJob({ jobs[0] }, 3, "B", job_func)); // 1
+    jobs.push_back(makeLoadJob({ jobs[0] }, 4, "C", job_func)); // 2
+    jobs.push_back(makeLoadJob({ jobs[0] }, 1, "D", job_func)); // 3
+    jobs.push_back(makeLoadJob({ jobs[0] }, 2, "E", job_func)); // 4
+    jobs.push_back(makeLoadJob({ jobs[3], jobs[4] }, 0, "F", job_func)); // 5
+    jobs.push_back(makeLoadJob({ jobs[5] }, 0, "G", job_func)); // 6
+    jobs.push_back(makeLoadJob({ jobs[6] }, 9, "H", job_func)); // 7
     auto task = t.schedule({ jobs.begin(), jobs.end() });
 
     t.loader.start();
@@ -614,14 +614,14 @@ TEST(AsyncLoader, DynamicPriorities)
         //     |       +-> F0 --> G0 --> H0
         //     `-> E2 -'
         std::vector<LoadJobPtr> jobs;
-        jobs.push_back(makeLoadJob({}, "A", 0, job_func)); // 0
-        jobs.push_back(makeLoadJob({ jobs[0] }, "B", 3, job_func)); // 1
-        jobs.push_back(makeLoadJob({ jobs[0] }, "C", 4, job_func)); // 2
-        jobs.push_back(makeLoadJob({ jobs[0] }, "D", 1, job_func)); // 3
-        jobs.push_back(makeLoadJob({ jobs[0] }, "E", 2, job_func)); // 4
-        jobs.push_back(makeLoadJob({ jobs[3], jobs[4] }, "F", 0, job_func)); // 5
-        jobs.push_back(makeLoadJob({ jobs[5] }, "G", 0, job_func)); // 6
-        jobs.push_back(makeLoadJob({ jobs[6] }, "H", 0, job_func)); // 7
+        jobs.push_back(makeLoadJob({}, 0, "A", job_func)); // 0
+        jobs.push_back(makeLoadJob({ jobs[0] }, 3, "B", job_func)); // 1
+        jobs.push_back(makeLoadJob({ jobs[0] }, 4, "C", job_func)); // 2
+        jobs.push_back(makeLoadJob({ jobs[0] }, 1, "D", job_func)); // 3
+        jobs.push_back(makeLoadJob({ jobs[0] }, 2, "E", job_func)); // 4
+        jobs.push_back(makeLoadJob({ jobs[3], jobs[4] }, 0, "F", job_func)); // 5
+        jobs.push_back(makeLoadJob({ jobs[5] }, 0, "G", job_func)); // 6
+        jobs.push_back(makeLoadJob({ jobs[6] }, 0, "H", job_func)); // 7
         auto task = t.schedule({ jobs.begin(), jobs.end() });
 
         job_to_prioritize = jobs[6];

From 8c91dbdfc6290ede79039db22a213ac1bdff6f2b Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 2 May 2023 10:47:38 +0000
Subject: [PATCH 105/689] Properly fix GCS with keys

---
 src/IO/S3/Client.cpp         | 48 ++++++++++++++++++++++++++++++++++--
 src/IO/S3/Client.h           |  3 +++
 src/IO/S3/PocoHTTPClient.cpp | 11 ---------
 src/IO/S3/ProviderType.cpp   | 11 ---------
 src/IO/S3/ProviderType.h     |  2 --
 5 files changed, 49 insertions(+), 26 deletions(-)

diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp
index 98c48076bdc..2e95831eced 100644
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@@ -111,6 +111,36 @@ std::unique_ptr<Client> Client::create(const Client & other)
     return std::unique_ptr<Client>(new Client(other));
 }
 
+namespace
+{
+
+ProviderType deduceProviderType(const std::string & url, const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> & credentials_provider)
+{
+    /// GCS can operate in 2 modes for header and query params names:
+    /// - with both x-amz and x-goog prefixes allowed (but cannot mix different prefixes in same request)
+    /// - only with x-goog prefix
+    /// first mode is allowed only with HMAC (or unsigned requests) so when we
+    /// find credential keys we can simply behave as the underlying storage is S3
+    /// otherwise, we need to be aware we are making requests to GCS
+    /// and replace all headers with a valid prefix when needed
+	if (credentials_provider)
+	{
+        auto credentials = credentials_provider->GetAWSCredentials();
+        if (!credentials.IsEmpty())
+            return ProviderType::AWS;
+	}
+
+    if (url.find(".amazonaws.com") != std::string::npos)
+		return ProviderType::AWS;
+
+    if (url.find("storage.googleapis.com") != std::string::npos)
+        return ProviderType::GCS;
+
+    return ProviderType::UNKNOWN;
+}
+
+}
+
 Client::Client(
     size_t max_redirects_,
     const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> & credentials_provider,
@@ -125,10 +155,10 @@ Client::Client(
     endpoint_provider->GetBuiltInParameters().GetParameter("Region").GetString(explicit_region);
     endpoint_provider->GetBuiltInParameters().GetParameter("Endpoint").GetString(initial_endpoint);
 
-    provider_type = getProviderTypeFromURL(initial_endpoint);
+    provider_type = deduceProviderType(initial_endpoint, credentials_provider);
     LOG_TRACE(log, "Provider type: {}", toString(provider_type));
 
-    detect_region = provider_type == ProviderType::AWS && explicit_region == Aws::Region::AWS_GLOBAL;
+    detect_region = initial_endpoint.find(".amazonaws.com") != std::string::npos && explicit_region == Aws::Region::AWS_GLOBAL;
 
     cache = std::make_shared<ClientCache>();
     ClientCacheRegistry::instance().registerClient(cache);
@@ -400,6 +430,20 @@ ProviderType Client::getProviderType() const
     return provider_type;
 }
 
+void Client::BuildHttpRequest(const Aws::AmazonWebServiceRequest& request,
+                      const std::shared_ptr<Aws::Http::HttpRequest>& httpRequest) const
+{
+    Aws::S3::S3Client::BuildHttpRequest(request, httpRequest);
+
+    if (provider_type == ProviderType::GCS)
+    {
+        /// some GCS requests don't like S3 specific headers that the client sets
+        httpRequest->DeleteHeader("x-amz-api-version");
+        httpRequest->DeleteHeader("amz-sdk-invocation-id");
+        httpRequest->DeleteHeader("amz-sdk-request");
+    }
+}
+
 std::string Client::getRegionForBucket(const std::string & bucket, bool force_detect) const
 {
     std::lock_guard lock(cache->region_cache_mutex);
diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h
index e3218872b4f..3612c448392 100644
--- a/src/IO/S3/Client.h
+++ b/src/IO/S3/Client.h
@@ -163,6 +163,9 @@ public:
     using Aws::S3::S3Client::DisableRequestProcessing;
 
     ProviderType getProviderType() const;
+
+    void BuildHttpRequest(const Aws::AmazonWebServiceRequest& request,
+                          const std::shared_ptr<Aws::Http::HttpRequest>& httpRequest) const override;
 private:
     Client(size_t max_redirects_,
            const std::shared_ptr<Aws::Auth::AWSCredentialsProvider>& credentials_provider,
diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp
index ee14e251cb5..bfda7149343 100644
--- a/src/IO/S3/PocoHTTPClient.cpp
+++ b/src/IO/S3/PocoHTTPClient.cpp
@@ -260,17 +260,6 @@ void PocoHTTPClient::makeRequestInternal(
     Poco::Logger * log = &Poco::Logger::get("AWSClient");
 
     auto uri = request.GetUri().GetURIString();
-#if 0
-    auto provider_type = getProviderTypeFromURL(uri);
-
-    if (provider_type == ProviderType::GCS)
-    {
-        /// some GCS requests don't like S3 specific headers that the client sets
-        request.DeleteHeader("x-amz-api-version");
-        request.DeleteHeader("amz-sdk-invocation-id");
-        request.DeleteHeader("amz-sdk-request");
-    }
-#endif
 
     if (enable_s3_requests_logging)
         LOG_TEST(log, "Make request to: {}", uri);
diff --git a/src/IO/S3/ProviderType.cpp b/src/IO/S3/ProviderType.cpp
index d5f78c0681a..39be90ff63d 100644
--- a/src/IO/S3/ProviderType.cpp
+++ b/src/IO/S3/ProviderType.cpp
@@ -27,17 +27,6 @@ bool supportsMultiPartCopy(ProviderType provider_type)
     return provider_type != ProviderType::GCS;
 }
 
-ProviderType getProviderTypeFromURL(const std::string & url)
-{
-    if (url.find(".amazonaws.com") != std::string::npos)
-        return ProviderType::AWS;
-
-    if (url.find("storage.googleapis.com") != std::string::npos)
-        return ProviderType::GCS;
-
-    return ProviderType::UNKNOWN;
-}
-
 }
 
 #endif
diff --git a/src/IO/S3/ProviderType.h b/src/IO/S3/ProviderType.h
index 4ce19d66f94..ff02357e081 100644
--- a/src/IO/S3/ProviderType.h
+++ b/src/IO/S3/ProviderType.h
@@ -21,8 +21,6 @@ std::string_view toString(ProviderType provider_type);
 
 bool supportsMultiPartCopy(ProviderType provider_type);
 
-ProviderType getProviderTypeFromURL(const std::string & url);
-
 }
 
 #endif

From 468ed32b23c430c01a8655722331806fa1bd0524 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 2 May 2023 11:08:26 +0000
Subject: [PATCH 106/689] Fix whitespace

---
 src/IO/S3/Client.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp
index 2e95831eced..40b16960021 100644
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@@ -123,15 +123,15 @@ ProviderType deduceProviderType(const std::string & url, const std::shared_ptr<A
     /// find credential keys we can simply behave as the underlying storage is S3
     /// otherwise, we need to be aware we are making requests to GCS
     /// and replace all headers with a valid prefix when needed
-	if (credentials_provider)
-	{
+    if (credentials_provider)
+    {
         auto credentials = credentials_provider->GetAWSCredentials();
         if (!credentials.IsEmpty())
             return ProviderType::AWS;
-	}
+    }
 
     if (url.find(".amazonaws.com") != std::string::npos)
-		return ProviderType::AWS;
+        return ProviderType::AWS;
 
     if (url.find("storage.googleapis.com") != std::string::npos)
         return ProviderType::GCS;

From 13739cd26b70993ddef7fb2ee5c83eb1ea0d0c12 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 2 May 2023 12:29:15 +0000
Subject: [PATCH 107/689] Correctly stop CNF convert for too many atomics

---
 src/Analyzer/Passes/CNF.cpp                              | 5 ++++-
 .../queries/0_stateless/02725_cnf_large_check.reference  | 2 ++
 tests/queries/0_stateless/02725_cnf_large_check.sql      | 9 +++++++++
 3 files changed, 15 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/02725_cnf_large_check.reference
 create mode 100644 tests/queries/0_stateless/02725_cnf_large_check.sql

diff --git a/src/Analyzer/Passes/CNF.cpp b/src/Analyzer/Passes/CNF.cpp
index f53d833c107..4448cda8ddd 100644
--- a/src/Analyzer/Passes/CNF.cpp
+++ b/src/Analyzer/Passes/CNF.cpp
@@ -187,7 +187,10 @@ public:
         {
             auto & arguments = function_node->getArguments().getNodes();
             for (auto & argument : arguments)
-                visit(argument);
+            {
+                if (!visit(argument))
+                    return false;
+            }
         }
 
         if (name == "or")
diff --git a/tests/queries/0_stateless/02725_cnf_large_check.reference b/tests/queries/0_stateless/02725_cnf_large_check.reference
new file mode 100644
index 00000000000..226aaf8af79
--- /dev/null
+++ b/tests/queries/0_stateless/02725_cnf_large_check.reference
@@ -0,0 +1,2 @@
+8
+8
diff --git a/tests/queries/0_stateless/02725_cnf_large_check.sql b/tests/queries/0_stateless/02725_cnf_large_check.sql
new file mode 100644
index 00000000000..881d562ab6d
--- /dev/null
+++ b/tests/queries/0_stateless/02725_cnf_large_check.sql
@@ -0,0 +1,9 @@
+DROP TABLE IF EXISTS 02725_cnf;
+
+CREATE TABLE 02725_cnf (c0 UInt8, c1 UInt8, c2 UInt8, c3 UInt8, c4 UInt8, c5 UInt8, c6 UInt8, c7 UInt8, c8 UInt8, c9 UInt8) ENGINE = Memory;
+
+INSERT INTO 02725_cnf VALUES (0, 0, 0, 0, 0, 0, 0, 0, 0, 0), (0, 0, 0, 0, 0, 0, 0, 0, 0, 1), (0, 0, 0, 0, 0, 0, 0, 0, 1, 0), (0, 0, 0, 0, 0, 0, 0, 0, 1, 1), (0, 0, 0, 0, 0, 0, 0, 1, 0, 0), (0, 0, 0, 0, 0, 0, 0, 1, 0, 1), (0, 0, 0, 0, 0, 0, 0, 1, 1, 0), (0, 0, 0, 0, 0, 0, 0, 1, 1, 1);
+SELECT count() FROM 02725_cnf WHERE (c5 AND (NOT c0)) OR ((NOT c3) AND (NOT c6) AND (NOT c1) AND (NOT c6)) OR (c7 AND (NOT c3) AND (NOT c5) AND (NOT c7)) OR ((NOT c8) AND c5) OR ((NOT c0)) OR ((NOT c8) AND (NOT c5) AND c1 AND c6 AND c3) OR (c7 AND (NOT c0) AND c6 AND c1 AND (NOT c2)) OR (c3 AND (NOT c9) AND c1) SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer=1;
+SELECT count() FROM 02725_cnf WHERE (c5 AND (NOT c0)) OR ((NOT c3) AND (NOT c6) AND (NOT c1) AND (NOT c6)) OR (c7 AND (NOT c3) AND (NOT c5) AND (NOT c7)) OR ((NOT c8) AND c5) OR ((NOT c0)) OR ((NOT c8) AND (NOT c5) AND c1 AND c6 AND c3) OR (c7 AND (NOT c0) AND c6 AND c1 AND (NOT c2)) OR (c3 AND (NOT c9) AND c1) SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer=0;
+
+DROP TABLE 02725_cnf;

From 6bdbee33eed54b0c4f7046009fcb988ccff8be85 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 2 May 2023 12:14:17 +0000
Subject: [PATCH 108/689] Incorporate review feedback

---
 .../InterpreterShowColumnsQuery.cpp           |  53 +++++----
 .../InterpreterShowIndexesQuery.cpp           | 106 +++++++++---------
 src/Interpreters/parseDatabaseAndTableName.h  |  44 --------
 src/Parsers/ASTShowColumnsQuery.cpp           |   6 +-
 src/Parsers/ASTShowColumnsQuery.h             |   4 +-
 src/Parsers/ASTShowIndexesQuery.cpp           |   6 +-
 src/Parsers/ASTShowIndexesQuery.h             |   4 +-
 src/Parsers/ParserShowColumnsQuery.cpp        |  34 ++++--
 src/Parsers/ParserShowIndexesQuery.cpp        |  34 ++++--
 9 files changed, 146 insertions(+), 145 deletions(-)
 delete mode 100644 src/Interpreters/parseDatabaseAndTableName.h

diff --git a/src/Interpreters/InterpreterShowColumnsQuery.cpp b/src/Interpreters/InterpreterShowColumnsQuery.cpp
index 86788007e3f..c1a546eae79 100644
--- a/src/Interpreters/InterpreterShowColumnsQuery.cpp
+++ b/src/Interpreters/InterpreterShowColumnsQuery.cpp
@@ -5,7 +5,6 @@
 #include <Parsers/formatAST.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/executeQuery.h>
-#include <Interpreters/parseDatabaseAndTableName.h>
 #include <IO/Operators.h>
 
 
@@ -24,9 +23,17 @@ String InterpreterShowColumnsQuery::getRewrittenQuery()
 {
     const auto & query = query_ptr->as<ASTShowColumnsQuery &>();
 
-    WriteBufferFromOwnString rewritten_query;
+    String database = query.database.empty() ? getContext()->getCurrentDatabase() : query.database;
+    String table = query.table;
 
-    rewritten_query << "SELECT name AS field, type AS type, startsWith(type, 'Nullable') AS null, trim(concatWithSeparator(' ', if(is_in_primary_key, 'PRI', ''), if (is_in_sorting_key, 'SOR', ''))) AS key, if(default_kind IN ('ALIAS', 'DEFAULT', 'MATERIALIZED'), default_expression, NULL) AS default, '' AS extra ";
+    String rewritten_query = R"(
+SELECT
+    name AS field,
+    type AS type,
+    startsWith(type, 'Nullable') AS null,
+    trim(concatWithSeparator(' ', if(is_in_primary_key, 'PRI', ''), if (is_in_sorting_key, 'SOR', ''))) AS key,
+    if(default_kind IN ('ALIAS', 'DEFAULT', 'MATERIALIZED'), default_expression, NULL) AS default,
+    '' AS extra )";
 
     // TODO Interpret query.extended. It is supposed to show internal/virtual columns. Need to fetch virtual column names, see
     // IStorage::getVirtuals(). We can't easily do that via SQL.
@@ -37,35 +44,41 @@ String InterpreterShowColumnsQuery::getRewrittenQuery()
         /// - collation: no such thing in ClickHouse
         /// - comment
         /// - privileges: <not implemented, TODO ask system.grants>
-        rewritten_query << ", NULL AS collation, comment, '' AS privileges ";
+        rewritten_query += R"(,
+    NULL AS collation,
+    comment,
+    '' AS privileges )";
     }
 
-    rewritten_query << "FROM system.columns WHERE ";
-
-    auto [database, table] = parseDatabaseAndTableName(query, getContext()->getCurrentDatabase());
-
-    rewritten_query << "database = " << DB::quote << database;
-    rewritten_query << " AND table = " << DB::quote << table;
+    rewritten_query += fmt::format(R"(
+    FROM system.columns
+    WHERE
+        database = '{}'
+        AND table = '{}' )", database, table);
 
     if (!query.like.empty())
-        rewritten_query
-            << " AND name "
-            << (query.not_like ? "NOT " : "")
-            << (query.case_insensitive_like ? "ILIKE " : "LIKE ")
-            << DB::quote << query.like;
+    {
+        rewritten_query += " AND name ";
+        if (query.not_like)
+            rewritten_query += "NOT ";
+        if (query.case_insensitive_like)
+            rewritten_query += "ILIKE ";
+        else
+            rewritten_query += "LIKE ";
+        rewritten_query += fmt::format("'{}'", query.like);
+    }
     else if (query.where_expression)
-        rewritten_query << " AND (" << query.where_expression << ")";
+        rewritten_query += fmt::format(" AND ({})", query.where_expression);
 
     /// Sorting is strictly speaking not necessary but 1. it is convenient for users, 2. SQL currently does not allow to
     /// sort the output of SHOW COLUMNS otherwise (SELECT * FROM (SHOW COLUMNS ...) ORDER BY ...) is rejected) and 3. some
     /// SQL tests can take advantage of this.
-    rewritten_query << " ORDER BY field, type, null, key, default, extra";
+    rewritten_query += " ORDER BY field, type, null, key, default, extra";
 
     if (query.limit_length)
-        rewritten_query << " LIMIT " << query.limit_length;
-
-    return rewritten_query.str();
+        rewritten_query += fmt::format(" LIMIT {}", query.limit_length);
 
+    return rewritten_query;
 }
 
 
diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp
index c5a63cde51e..ca66b32ca80 100644
--- a/src/Interpreters/InterpreterShowIndexesQuery.cpp
+++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp
@@ -5,7 +5,6 @@
 #include <Parsers/formatAST.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/executeQuery.h>
-#include <Interpreters/parseDatabaseAndTableName.h>
 #include <IO/Operators.h>
 
 
@@ -24,65 +23,62 @@ String InterpreterShowIndexesQuery::getRewrittenQuery()
 {
     const auto & query = query_ptr->as<ASTShowIndexesQuery &>();
 
-    auto [database, table] = parseDatabaseAndTableName(query, getContext()->getCurrentDatabase());
+    String table = query.table;
+    String database = query.database.empty() ? getContext()->getCurrentDatabase() : query.database;
+    String where_expression = query.where_expression ? fmt::format("WHERE ({})", query.where_expression) : "";
 
-    WriteBufferFromOwnString where_expression_buf;
-    if (query.where_expression)
-        where_expression_buf << "WHERE (" << query.where_expression << ")";
-    String where_expression = where_expression_buf.str();
-
-    WriteBufferFromOwnString rewritten_query;
-    rewritten_query     << "SELECT * FROM ("
-                        << "(SELECT "
-                            << "name AS table, "
-                            << "0 AS non_unique, "
-                            << "'PRIMARY' AS key_name, "
-                            << "NULL AS seq_in_index, "
-                            << "NULL AS column_name, "
-                            << "'A' AS collation, "
-                            << "NULL AS cardinality, "
-                            << "NULL AS sub_part, "
-                            << "NULL AS packed, "
-                            << "NULL AS null, "
-                            << "'primary' AS index_type, "
-                            << "NULL AS comment, "
-                            << "NULL AS index_comment, "
-                            << "'YES' AS visible, "
-                            << "primary_key AS expression "
-                        << "FROM system.tables "
-                        << "WHERE "
-                            << "database = '" << database << "' "
-                            << "AND name = '" << table << "'"
-                    << ") UNION ALL ("
-                        << "SELECT "
-                            << "table AS table, "
-                            << "0 AS non_unique, "
-                            << "name AS key_name, "
-                            << "NULL AS seq_in_index, "
-                            << "NULL AS column_name, "
-                            << "NULL AS collation, "
-                            << "NULL AS cardinality, "
-                            << "NULL AS sub_part, "
-                            << "NULL AS packed, "
-                            << "NULL AS null, "
-                            << "type AS index_type, "
-                            << "NULL AS comment, "
-                            << "NULL AS index_comment, "
-                            << "'YES' AS visible, "
-                            << "expr AS expression "
-                        << "FROM system.data_skipping_indices "
-                        << "WHERE "
-                            << "database = '" << database << "' "
-                            << "AND table = '" << table << "'"
-                            << ")) "
-                        << where_expression;
+    String rewritten_query = fmt::format(R"(
+SELECT *
+FROM (
+        (SELECT
+            name AS table,
+            0 AS non_unique,
+            'PRIMARY' AS key_name,
+            NULL AS seq_in_index,
+            NULL AS column_name,
+            'A' AS collation,
+            NULL AS cardinality,
+            NULL AS sub_part,
+            NULL AS packed,
+            NULL AS null,
+            'primary' AS index_type,
+            NULL AS comment,
+            NULL AS index_comment,
+            'YES' AS visible,
+            primary_key AS expression
+        FROM system.tables
+        WHERE
+            database = '{0}'
+            AND name = '{1}')
+    UNION ALL (
+        SELECT
+            table AS table,
+            0 AS non_unique,
+            name AS key_name,
+            NULL AS seq_in_index,
+            NULL AS column_name,
+            NULL AS collation,
+            NULL AS cardinality,
+            NULL AS sub_part,
+            NULL AS packed,
+            NULL AS null,
+            type AS index_type,
+            NULL AS comment,
+            NULL AS index_comment,
+            'YES' AS visible,
+            expr AS expression
+        FROM system.data_skipping_indices
+        WHERE
+            database = '{0}'
+            AND table = '{1}'))
+{2}
+ORDER BY index_type, expression;)", database, table, where_expression);
 
     /// Sorting is strictly speaking not necessary but 1. it is convenient for users, 2. SQL currently does not allow to
-    /// sort the output of SHOW COLUMNS otherwise (SELECT * FROM (SHOW COLUMNS ...) ORDER BY ...) is rejected) and 3. some
+    /// sort the output of SHOW INDEXES otherwise (SELECT * FROM (SHOW INDEXES ...) ORDER BY ...) is rejected) and 3. some
     /// SQL tests can take advantage of this.
-    rewritten_query << " ORDER BY index_type, expression";
 
-    return rewritten_query.str();
+    return rewritten_query;
 }
 
 
diff --git a/src/Interpreters/parseDatabaseAndTableName.h b/src/Interpreters/parseDatabaseAndTableName.h
deleted file mode 100644
index b336a82028b..00000000000
--- a/src/Interpreters/parseDatabaseAndTableName.h
+++ /dev/null
@@ -1,44 +0,0 @@
-#pragma once
-
-#include <Common/Exception.h>
-#include <base/defines.h>
-#include <base/types.h>
-#include <boost/algorithm/string.hpp>
-
-namespace DB
-{
-    template <typename Query>
-    inline std::pair<String, String> parseDatabaseAndTableName(const Query & query, const String & current_database)
-    {
-        String database;
-        String table;
-
-        if (query.from_table.contains("."))
-        {
-            /// FROM <db>.<table> (abbreviated form)
-            chassert(query.from_database.empty());
-            std::vector<String> split;
-            boost::split(split, query.from_table, boost::is_any_of("."));
-            chassert(split.size() == 2);
-            database = split[0];
-            table = split[1];
-        }
-        else if (query.from_database.empty())
-        {
-            /// FROM <table>
-            chassert(!query.from_table.empty());
-            database = current_database;
-            table = query.from_table;
-        }
-        else
-        {
-            /// FROM <database> FROM <table>
-            chassert(!query.from_database.empty());
-            chassert(!query.from_table.empty());
-            database = query.from_database;
-            table = query.from_table;
-        }
-
-        return {database, table};
-    }
-}
diff --git a/src/Parsers/ASTShowColumnsQuery.cpp b/src/Parsers/ASTShowColumnsQuery.cpp
index 1cb4b651f0a..a02dba9c288 100644
--- a/src/Parsers/ASTShowColumnsQuery.cpp
+++ b/src/Parsers/ASTShowColumnsQuery.cpp
@@ -24,9 +24,9 @@ void ASTShowColumnsQuery::formatQueryImpl(const FormatSettings & settings, Forma
                   << "COLUMNS"
                   << (settings.hilite ? hilite_none : "");
 
-    settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(from_table);
-    if (!from_database.empty())
-        settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(from_database);
+    settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(table);
+    if (!database.empty())
+        settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(database);
 
 
     if (!like.empty())
diff --git a/src/Parsers/ASTShowColumnsQuery.h b/src/Parsers/ASTShowColumnsQuery.h
index 79e8a67b558..b49d688dcdb 100644
--- a/src/Parsers/ASTShowColumnsQuery.h
+++ b/src/Parsers/ASTShowColumnsQuery.h
@@ -18,8 +18,8 @@ public:
     ASTPtr where_expression;
     ASTPtr limit_length;
 
-    String from_database;
-    String from_table;
+    String database;
+    String table;
 
     String like;
 
diff --git a/src/Parsers/ASTShowIndexesQuery.cpp b/src/Parsers/ASTShowIndexesQuery.cpp
index 6930043e5ed..df1e242a239 100644
--- a/src/Parsers/ASTShowIndexesQuery.cpp
+++ b/src/Parsers/ASTShowIndexesQuery.cpp
@@ -23,9 +23,9 @@ void ASTShowIndexesQuery::formatQueryImpl(const FormatSettings & settings, Forma
                   << "INDEXES"
                   << (settings.hilite ? hilite_none : "");
 
-    settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(from_table);
-    if (!from_database.empty())
-        settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(from_database);
+    settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(table);
+    if (!database.empty())
+        settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(database);
 
     if (where_expression)
     {
diff --git a/src/Parsers/ASTShowIndexesQuery.h b/src/Parsers/ASTShowIndexesQuery.h
index a0d5e0c6b6f..ab504bf71da 100644
--- a/src/Parsers/ASTShowIndexesQuery.h
+++ b/src/Parsers/ASTShowIndexesQuery.h
@@ -14,8 +14,8 @@ public:
 
     ASTPtr where_expression;
 
-    String from_database;
-    String from_table;
+    String database;
+    String table;
 
     String getID(char) const override { return "ShowColumns"; }
     ASTPtr clone() const override;
diff --git a/src/Parsers/ParserShowColumnsQuery.cpp b/src/Parsers/ParserShowColumnsQuery.cpp
index 57328bb0d8b..8d5fdbf90e3 100644
--- a/src/Parsers/ParserShowColumnsQuery.cpp
+++ b/src/Parsers/ParserShowColumnsQuery.cpp
@@ -7,14 +7,19 @@
 #include <Parsers/ExpressionElementParsers.h>
 #include <Parsers/ExpressionListParsers.h>
 
+#include <boost/algorithm/string.hpp>
+
 namespace DB
 {
 
 bool ParserShowColumnsQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
     ASTPtr like;
-    ASTPtr from_database;
-    ASTPtr from_table;
+    ASTPtr from1;
+    ASTPtr from2;
+
+    String from1_str;
+    String from2_str;
 
     auto query = std::make_shared<ASTShowColumnsQuery>();
 
@@ -32,21 +37,34 @@ bool ParserShowColumnsQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
 
     if (ParserKeyword("FROM").ignore(pos, expected) || ParserKeyword("IN").ignore(pos, expected))
     {
-        if (!ParserCompoundIdentifier().parse(pos, from_table, expected))
+        if (!ParserCompoundIdentifier().parse(pos, from1, expected))
             return false;
     }
     else
         return false;
 
-    tryGetIdentifierNameInto(from_table, query->from_table);
-    bool abbreviated_form = query->from_table.contains("."); /// FROM <db>.<table>
+    tryGetIdentifierNameInto(from1, from1_str);
 
-    if (!abbreviated_form)
+    bool abbreviated_form = from1_str.contains("."); // FROM database.table
+    if (abbreviated_form)
+    {
+        std::vector<String> split;
+        boost::split(split, from1_str, boost::is_any_of("."));
+        query->database = split[0];
+        query->table = split[1];
+    }
+    else
+    {
         if (ParserKeyword("FROM").ignore(pos, expected) || ParserKeyword("IN").ignore(pos, expected))
-            if (!ParserIdentifier().parse(pos, from_database, expected))
+            if (!ParserIdentifier().parse(pos, from2, expected))
                 return false;
 
-    tryGetIdentifierNameInto(from_database, query->from_database);
+        tryGetIdentifierNameInto(from2, from2_str);
+
+        query->table = from1_str;
+        if (!from2_str.empty())
+            query->database = from2_str; /// FROM table FROM database
+    }
 
     if (ParserKeyword("NOT").ignore(pos, expected))
         query->not_like = true;
diff --git a/src/Parsers/ParserShowIndexesQuery.cpp b/src/Parsers/ParserShowIndexesQuery.cpp
index b4651ed21f9..8679f02963c 100644
--- a/src/Parsers/ParserShowIndexesQuery.cpp
+++ b/src/Parsers/ParserShowIndexesQuery.cpp
@@ -7,13 +7,18 @@
 #include <Parsers/ExpressionElementParsers.h>
 #include <Parsers/ExpressionListParsers.h>
 
+#include <boost/algorithm/string.hpp>
+
 namespace DB
 {
 
 bool ParserShowIndexesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
-    ASTPtr from_database;
-    ASTPtr from_table;
+    ASTPtr from1;
+    ASTPtr from2;
+
+    String from1_str;
+    String from2_str;
 
     auto query = std::make_shared<ASTShowIndexesQuery>();
 
@@ -28,21 +33,34 @@ bool ParserShowIndexesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
 
     if (ParserKeyword("FROM").ignore(pos, expected) || ParserKeyword("IN").ignore(pos, expected))
     {
-        if (!ParserCompoundIdentifier().parse(pos, from_table, expected))
+        if (!ParserCompoundIdentifier().parse(pos, from1, expected))
             return false;
     }
     else
         return false;
 
-    tryGetIdentifierNameInto(from_table, query->from_table);
-    bool abbreviated_form = query->from_table.contains("."); /// FROM <db>.<table>
+    tryGetIdentifierNameInto(from1, from1_str);
 
-    if (!abbreviated_form)
+    bool abbreviated_form = from1_str.contains("."); // FROM database.table
+    if (abbreviated_form)
+    {
+        std::vector<String> split;
+        boost::split(split, from1_str, boost::is_any_of("."));
+        query->database = split[0];
+        query->table = split[1];
+    }
+    else
+    {
         if (ParserKeyword("FROM").ignore(pos, expected) || ParserKeyword("IN").ignore(pos, expected))
-            if (!ParserIdentifier().parse(pos, from_database, expected))
+            if (!ParserIdentifier().parse(pos, from2, expected))
                 return false;
 
-    tryGetIdentifierNameInto(from_database, query->from_database);
+        tryGetIdentifierNameInto(from2, from2_str);
+
+        query->table = from1_str;
+        if (!from2_str.empty())
+            query->database = from2_str; /// FROM table FROM database
+    }
 
     if (ParserKeyword("WHERE").ignore(pos, expected))
         if (!ParserExpressionWithOptionalAlias(false).parse(pos, query->where_expression, expected))

From 5a31d2129e6a7b5252a0c8ba75c0f7c47f7d6dc7 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 2 May 2023 14:54:10 +0200
Subject: [PATCH 109/689] fix build in fast test

---
 contrib/libfiu                      | 2 +-
 contrib/libfiu-cmake/CMakeLists.txt | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/contrib/libfiu b/contrib/libfiu
index 7835d077aec..7ae4328050c 160000
--- a/contrib/libfiu
+++ b/contrib/libfiu
@@ -1 +1 @@
-Subproject commit 7835d077aecf71523ffc8967d2908becba982703
+Subproject commit 7ae4328050ccad8867a05a37af941886f717b6fc
diff --git a/contrib/libfiu-cmake/CMakeLists.txt b/contrib/libfiu-cmake/CMakeLists.txt
index 6d7f86c7a8f..eb66d86c704 100644
--- a/contrib/libfiu-cmake/CMakeLists.txt
+++ b/contrib/libfiu-cmake/CMakeLists.txt
@@ -10,7 +10,6 @@ set(FIU_SOURCES
   ${FIU_DIR}/libfiu/fiu-rc.c
   ${FIU_DIR}/libfiu/backtrace.c
   ${FIU_DIR}/libfiu/wtable.c
-  ${FIU_DIR}/libfiu/hash.c
 )
 
 set(FIU_HEADERS "${FIU_DIR}/libfiu")

From 7e5fded3cf32dc2282ad415e8759922975bc5c7d Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 2 May 2023 13:16:15 +0000
Subject: [PATCH 110/689] A little less clutter

---
 src/Parsers/ParserShowColumnsQuery.cpp | 3 +--
 src/Parsers/ParserShowIndexesQuery.cpp | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/Parsers/ParserShowColumnsQuery.cpp b/src/Parsers/ParserShowColumnsQuery.cpp
index 8d5fdbf90e3..e5fc5c39701 100644
--- a/src/Parsers/ParserShowColumnsQuery.cpp
+++ b/src/Parsers/ParserShowColumnsQuery.cpp
@@ -62,8 +62,7 @@ bool ParserShowColumnsQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
         tryGetIdentifierNameInto(from2, from2_str);
 
         query->table = from1_str;
-        if (!from2_str.empty())
-            query->database = from2_str; /// FROM table FROM database
+        query->database = from2_str;
     }
 
     if (ParserKeyword("NOT").ignore(pos, expected))
diff --git a/src/Parsers/ParserShowIndexesQuery.cpp b/src/Parsers/ParserShowIndexesQuery.cpp
index 8679f02963c..fe08b463069 100644
--- a/src/Parsers/ParserShowIndexesQuery.cpp
+++ b/src/Parsers/ParserShowIndexesQuery.cpp
@@ -58,8 +58,7 @@ bool ParserShowIndexesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
         tryGetIdentifierNameInto(from2, from2_str);
 
         query->table = from1_str;
-        if (!from2_str.empty())
-            query->database = from2_str; /// FROM table FROM database
+        query->database = from2_str;
     }
 
     if (ParserKeyword("WHERE").ignore(pos, expected))

From 4f3b7aded4fe33dc2c73b9af9cac5f9e877bc4b1 Mon Sep 17 00:00:00 2001
From: Larry Luo <larry.luo@ca.ibm.com>
Date: Tue, 2 May 2023 11:17:23 -0700
Subject: [PATCH 111/689] Test CREATE table with explicit type definition

---
 .../02724_persist_interval_type.reference        |  2 ++
 .../0_stateless/02724_persist_interval_type.sql  | 16 +++++++++++-----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/tests/queries/0_stateless/02724_persist_interval_type.reference b/tests/queries/0_stateless/02724_persist_interval_type.reference
index 9139216d644..964604605d9 100644
--- a/tests/queries/0_stateless/02724_persist_interval_type.reference
+++ b/tests/queries/0_stateless/02724_persist_interval_type.reference
@@ -2,3 +2,5 @@
 2023-01-01 00:00:02.000000001	2023-01-01 03:00:00.000000001	2023-01-01 00:00:00.000000005	2	3	0
 2023-01-01 00:00:01.000000001	2023-01-01 02:00:00.000000001	2023-01-01 00:00:00.000000004	1	2	0
 2023-01-01 00:00:02.000000001	2023-01-01 03:00:00.000000001	2023-01-01 00:00:00.000000005	2	3	0
+0
+1
diff --git a/tests/queries/0_stateless/02724_persist_interval_type.sql b/tests/queries/0_stateless/02724_persist_interval_type.sql
index 72133aceb00..3acce003c9a 100644
--- a/tests/queries/0_stateless/02724_persist_interval_type.sql
+++ b/tests/queries/0_stateless/02724_persist_interval_type.sql
@@ -1,9 +1,15 @@
-
 DROP TABLE IF EXISTS saved_intervals_tmp;
-DROP TABLE IF EXISTS saved_intervals_mgt;
 create table saved_intervals_tmp Engine=Memory as SELECT number as EventID, toIntervalSecond(number+1) as v1, toIntervalHour(number+2) as v2, toIntervalNanosecond(number+3) as v3 from numbers(2);
-create table saved_intervals_mgt Engine=MergeTree() ORDER BY EventID as SELECT number as EventID, toIntervalSecond(number+1) as v1, toIntervalHour(number+2) as v2, toIntervalNanosecond(number+3) as v3 from numbers(2);
 with toDateTime64('2023-01-01 00:00:00.000000001', 9, 'US/Eastern') as c select c+v1 as c_v1, c+v2 as c_v2, c+v3 as c_v3, date_diff(second, c, c_v1), date_diff(hour, c, c_v2), date_diff(second, c, c_v3) from saved_intervals_tmp;
-with toDateTime64('2023-01-01 00:00:00.000000001', 9, 'US/Eastern') as c select c+v1 as c_v1, c+v2 as c_v2, c+v3 as c_v3, date_diff(second, c, c_v1), date_diff(hour, c, c_v2), date_diff(second, c, c_v3) from saved_intervals_mgt;
 DROP TABLE IF EXISTS saved_intervals_tmp;
-DROP TABLE IF EXISTS saved_intervals_mgt;
\ No newline at end of file
+
+DROP TABLE IF EXISTS saved_intervals_mgt;
+create table saved_intervals_mgt Engine=MergeTree() ORDER BY EventID as SELECT number as EventID, toIntervalSecond(number+1) as v1, toIntervalHour(number+2) as v2, toIntervalNanosecond(number+3) as v3 from numbers(2);
+with toDateTime64('2023-01-01 00:00:00.000000001', 9, 'US/Eastern') as c select c+v1 as c_v1, c+v2 as c_v2, c+v3 as c_v3, date_diff(second, c, c_v1), date_diff(hour, c, c_v2), date_diff(second, c, c_v3) from saved_intervals_mgt;
+DROP TABLE IF EXISTS saved_intervals_mgt;
+
+DROP TABLE IF EXISTS t1;
+CREATE table t1 (v1 IntervalMinute) ENGINE = Memory;
+INSERT INTO t1 with toDateTime64('2023-01-01 00:00:00.000000001', 9, 'US/Eastern') as c SELECT EXTRACT(MINUTE FROM c+toIntervalSecond(number * 60)) from numbers(2);
+select * from t1;
+DROP TABLE IF EXISTS t1;
\ No newline at end of file

From 9bb74e5051cd000c8eef836ad1537795ae5f25cf Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 2 May 2023 21:12:38 +0200
Subject: [PATCH 112/689] fix fast tests

---
 tests/queries/0_stateless/01271_show_privileges.reference   | 1 +
 .../0_stateless/02117_show_create_table_system.reference    | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference
index 553fee1f435..091c446b82a 100644
--- a/tests/queries/0_stateless/01271_show_privileges.reference
+++ b/tests/queries/0_stateless/01271_show_privileges.reference
@@ -136,6 +136,7 @@ SYSTEM FLUSH LOGS	['FLUSH LOGS']	GLOBAL	SYSTEM FLUSH
 SYSTEM FLUSH	[]	\N	SYSTEM
 SYSTEM THREAD FUZZER	['SYSTEM START THREAD FUZZER','SYSTEM STOP THREAD FUZZER','START THREAD FUZZER','STOP THREAD FUZZER']	GLOBAL	SYSTEM
 SYSTEM UNFREEZE	['SYSTEM UNFREEZE']	GLOBAL	SYSTEM
+SYSTEM FAILPOINT	['SYSTEM ENABLE/DISABLE FAILPOINT']	GLOBAL	SYSTEM
 SYSTEM	[]	\N	ALL
 dictGet	['dictHas','dictGetHierarchy','dictIsIn']	DICTIONARY	ALL
 addressToLine	[]	GLOBAL	INTROSPECTION
diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference
index 56330ea8bb9..65c48ac3d7a 100644
--- a/tests/queries/0_stateless/02117_show_create_table_system.reference
+++ b/tests/queries/0_stateless/02117_show_create_table_system.reference
@@ -294,7 +294,7 @@ CREATE TABLE system.grants
 (
     `user_name` Nullable(String),
     `role_name` Nullable(String),
-    `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION CONTROL' = 96, 'SYSTEM SHUTDOWN' = 97, 'SYSTEM DROP DNS CACHE' = 98, 'SYSTEM DROP MARK CACHE' = 99, 'SYSTEM DROP UNCOMPRESSED CACHE' = 100, 'SYSTEM DROP MMAP CACHE' = 101, 'SYSTEM DROP QUERY CACHE' = 102, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 103, 'SYSTEM DROP FILESYSTEM CACHE' = 104, 'SYSTEM DROP SCHEMA CACHE' = 105, 'SYSTEM DROP S3 CLIENT CACHE' = 106, 'SYSTEM DROP CACHE' = 107, 'SYSTEM RELOAD CONFIG' = 108, 'SYSTEM RELOAD USERS' = 109, 'SYSTEM RELOAD SYMBOLS' = 110, 'SYSTEM RELOAD DICTIONARY' = 111, 'SYSTEM RELOAD MODEL' = 112, 'SYSTEM RELOAD FUNCTION' = 113, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 114, 'SYSTEM RELOAD' = 115, 'SYSTEM RESTART DISK' = 116, 'SYSTEM MERGES' = 117, 'SYSTEM TTL MERGES' = 118, 'SYSTEM FETCHES' = 119, 'SYSTEM MOVES' = 120, 'SYSTEM DISTRIBUTED SENDS' = 121, 'SYSTEM REPLICATED SENDS' = 122, 'SYSTEM SENDS' = 123, 'SYSTEM REPLICATION QUEUES' = 124, 'SYSTEM DROP REPLICA' = 125, 'SYSTEM SYNC REPLICA' = 126, 'SYSTEM RESTART REPLICA' = 127, 'SYSTEM RESTORE REPLICA' = 128, 'SYSTEM WAIT LOADING PARTS' = 129, 'SYSTEM SYNC DATABASE REPLICA' = 130, 'SYSTEM SYNC TRANSACTION LOG' = 131, 'SYSTEM SYNC FILE CACHE' = 132, 'SYSTEM FLUSH DISTRIBUTED' = 133, 'SYSTEM FLUSH LOGS' = 134, 'SYSTEM FLUSH' = 135, 'SYSTEM THREAD FUZZER' = 136, 'SYSTEM UNFREEZE' = 137, 'SYSTEM' = 138, 'dictGet' = 139, 'addressToLine' = 140, 'addressToLineWithInlines' = 141, 'addressToSymbol' = 142, 'demangle' = 143, 'INTROSPECTION' = 144, 'FILE' = 145, 'URL' = 146, 'REMOTE' = 147, 'MONGO' = 148, 'MEILISEARCH' = 149, 'MYSQL' = 150, 'POSTGRES' = 151, 'SQLITE' = 152, 'ODBC' = 153, 'JDBC' = 154, 'HDFS' = 155, 'S3' = 156, 'HIVE' = 157, 'SOURCES' = 158, 'CLUSTER' = 159, 'ALL' = 160, 'NONE' = 161),
+    `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION CONTROL' = 96, 'SYSTEM SHUTDOWN' = 97, 'SYSTEM DROP DNS CACHE' = 98, 'SYSTEM DROP MARK CACHE' = 99, 'SYSTEM DROP UNCOMPRESSED CACHE' = 100, 'SYSTEM DROP MMAP CACHE' = 101, 'SYSTEM DROP QUERY CACHE' = 102, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 103, 'SYSTEM DROP FILESYSTEM CACHE' = 104, 'SYSTEM DROP SCHEMA CACHE' = 105, 'SYSTEM DROP S3 CLIENT CACHE' = 106, 'SYSTEM DROP CACHE' = 107, 'SYSTEM RELOAD CONFIG' = 108, 'SYSTEM RELOAD USERS' = 109, 'SYSTEM RELOAD SYMBOLS' = 110, 'SYSTEM RELOAD DICTIONARY' = 111, 'SYSTEM RELOAD MODEL' = 112, 'SYSTEM RELOAD FUNCTION' = 113, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 114, 'SYSTEM RELOAD' = 115, 'SYSTEM RESTART DISK' = 116, 'SYSTEM MERGES' = 117, 'SYSTEM TTL MERGES' = 118, 'SYSTEM FETCHES' = 119, 'SYSTEM MOVES' = 120, 'SYSTEM DISTRIBUTED SENDS' = 121, 'SYSTEM REPLICATED SENDS' = 122, 'SYSTEM SENDS' = 123, 'SYSTEM REPLICATION QUEUES' = 124, 'SYSTEM DROP REPLICA' = 125, 'SYSTEM SYNC REPLICA' = 126, 'SYSTEM RESTART REPLICA' = 127, 'SYSTEM RESTORE REPLICA' = 128, 'SYSTEM WAIT LOADING PARTS' = 129, 'SYSTEM SYNC DATABASE REPLICA' = 130, 'SYSTEM SYNC TRANSACTION LOG' = 131, 'SYSTEM SYNC FILE CACHE' = 132, 'SYSTEM FLUSH DISTRIBUTED' = 133, 'SYSTEM FLUSH LOGS' = 134, 'SYSTEM FLUSH' = 135, 'SYSTEM THREAD FUZZER' = 136, 'SYSTEM UNFREEZE' = 137, 'SYSTEM FAILPOINT' = 138, 'SYSTEM' = 139, 'dictGet' = 140, 'addressToLine' = 141, 'addressToLineWithInlines' = 142, 'addressToSymbol' = 143, 'demangle' = 144, 'INTROSPECTION' = 145, 'FILE' = 146, 'URL' = 147, 'REMOTE' = 148, 'MONGO' = 149, 'MEILISEARCH' = 150, 'MYSQL' = 151, 'POSTGRES' = 152, 'SQLITE' = 153, 'ODBC' = 154, 'JDBC' = 155, 'HDFS' = 156, 'S3' = 157, 'HIVE' = 158, 'SOURCES' = 159, 'CLUSTER' = 160, 'ALL' = 161, 'NONE' = 162),
     `database` Nullable(String),
     `table` Nullable(String),
     `column` Nullable(String),
@@ -577,10 +577,10 @@ ENGINE = SystemPartsColumns
 COMMENT 'SYSTEM TABLE is built on the fly.'
 CREATE TABLE system.privileges
 (
-    `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION CONTROL' = 96, 'SYSTEM SHUTDOWN' = 97, 'SYSTEM DROP DNS CACHE' = 98, 'SYSTEM DROP MARK CACHE' = 99, 'SYSTEM DROP UNCOMPRESSED CACHE' = 100, 'SYSTEM DROP MMAP CACHE' = 101, 'SYSTEM DROP QUERY CACHE' = 102, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 103, 'SYSTEM DROP FILESYSTEM CACHE' = 104, 'SYSTEM DROP SCHEMA CACHE' = 105, 'SYSTEM DROP S3 CLIENT CACHE' = 106, 'SYSTEM DROP CACHE' = 107, 'SYSTEM RELOAD CONFIG' = 108, 'SYSTEM RELOAD USERS' = 109, 'SYSTEM RELOAD SYMBOLS' = 110, 'SYSTEM RELOAD DICTIONARY' = 111, 'SYSTEM RELOAD MODEL' = 112, 'SYSTEM RELOAD FUNCTION' = 113, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 114, 'SYSTEM RELOAD' = 115, 'SYSTEM RESTART DISK' = 116, 'SYSTEM MERGES' = 117, 'SYSTEM TTL MERGES' = 118, 'SYSTEM FETCHES' = 119, 'SYSTEM MOVES' = 120, 'SYSTEM DISTRIBUTED SENDS' = 121, 'SYSTEM REPLICATED SENDS' = 122, 'SYSTEM SENDS' = 123, 'SYSTEM REPLICATION QUEUES' = 124, 'SYSTEM DROP REPLICA' = 125, 'SYSTEM SYNC REPLICA' = 126, 'SYSTEM RESTART REPLICA' = 127, 'SYSTEM RESTORE REPLICA' = 128, 'SYSTEM WAIT LOADING PARTS' = 129, 'SYSTEM SYNC DATABASE REPLICA' = 130, 'SYSTEM SYNC TRANSACTION LOG' = 131, 'SYSTEM SYNC FILE CACHE' = 132, 'SYSTEM FLUSH DISTRIBUTED' = 133, 'SYSTEM FLUSH LOGS' = 134, 'SYSTEM FLUSH' = 135, 'SYSTEM THREAD FUZZER' = 136, 'SYSTEM UNFREEZE' = 137, 'SYSTEM' = 138, 'dictGet' = 139, 'addressToLine' = 140, 'addressToLineWithInlines' = 141, 'addressToSymbol' = 142, 'demangle' = 143, 'INTROSPECTION' = 144, 'FILE' = 145, 'URL' = 146, 'REMOTE' = 147, 'MONGO' = 148, 'MEILISEARCH' = 149, 'MYSQL' = 150, 'POSTGRES' = 151, 'SQLITE' = 152, 'ODBC' = 153, 'JDBC' = 154, 'HDFS' = 155, 'S3' = 156, 'HIVE' = 157, 'SOURCES' = 158, 'CLUSTER' = 159, 'ALL' = 160, 'NONE' = 161),
+    `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION CONTROL' = 96, 'SYSTEM SHUTDOWN' = 97, 'SYSTEM DROP DNS CACHE' = 98, 'SYSTEM DROP MARK CACHE' = 99, 'SYSTEM DROP UNCOMPRESSED CACHE' = 100, 'SYSTEM DROP MMAP CACHE' = 101, 'SYSTEM DROP QUERY CACHE' = 102, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 103, 'SYSTEM DROP FILESYSTEM CACHE' = 104, 'SYSTEM DROP SCHEMA CACHE' = 105, 'SYSTEM DROP S3 CLIENT CACHE' = 106, 'SYSTEM DROP CACHE' = 107, 'SYSTEM RELOAD CONFIG' = 108, 'SYSTEM RELOAD USERS' = 109, 'SYSTEM RELOAD SYMBOLS' = 110, 'SYSTEM RELOAD DICTIONARY' = 111, 'SYSTEM RELOAD MODEL' = 112, 'SYSTEM RELOAD FUNCTION' = 113, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 114, 'SYSTEM RELOAD' = 115, 'SYSTEM RESTART DISK' = 116, 'SYSTEM MERGES' = 117, 'SYSTEM TTL MERGES' = 118, 'SYSTEM FETCHES' = 119, 'SYSTEM MOVES' = 120, 'SYSTEM DISTRIBUTED SENDS' = 121, 'SYSTEM REPLICATED SENDS' = 122, 'SYSTEM SENDS' = 123, 'SYSTEM REPLICATION QUEUES' = 124, 'SYSTEM DROP REPLICA' = 125, 'SYSTEM SYNC REPLICA' = 126, 'SYSTEM RESTART REPLICA' = 127, 'SYSTEM RESTORE REPLICA' = 128, 'SYSTEM WAIT LOADING PARTS' = 129, 'SYSTEM SYNC DATABASE REPLICA' = 130, 'SYSTEM SYNC TRANSACTION LOG' = 131, 'SYSTEM SYNC FILE CACHE' = 132, 'SYSTEM FLUSH DISTRIBUTED' = 133, 'SYSTEM FLUSH LOGS' = 134, 'SYSTEM FLUSH' = 135, 'SYSTEM THREAD FUZZER' = 136, 'SYSTEM UNFREEZE' = 137, 'SYSTEM FAILPOINT' = 138, 'SYSTEM' = 139, 'dictGet' = 140, 'addressToLine' = 141, 'addressToLineWithInlines' = 142, 'addressToSymbol' = 143, 'demangle' = 144, 'INTROSPECTION' = 145, 'FILE' = 146, 'URL' = 147, 'REMOTE' = 148, 'MONGO' = 149, 'MEILISEARCH' = 150, 'MYSQL' = 151, 'POSTGRES' = 152, 'SQLITE' = 153, 'ODBC' = 154, 'JDBC' = 155, 'HDFS' = 156, 'S3' = 157, 'HIVE' = 158, 'SOURCES' = 159, 'CLUSTER' = 160, 'ALL' = 161, 'NONE' = 162),
     `aliases` Array(String),
     `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5, 'NAMED_COLLECTION' = 6)),
-    `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION CONTROL' = 96, 'SYSTEM SHUTDOWN' = 97, 'SYSTEM DROP DNS CACHE' = 98, 'SYSTEM DROP MARK CACHE' = 99, 'SYSTEM DROP UNCOMPRESSED CACHE' = 100, 'SYSTEM DROP MMAP CACHE' = 101, 'SYSTEM DROP QUERY CACHE' = 102, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 103, 'SYSTEM DROP FILESYSTEM CACHE' = 104, 'SYSTEM DROP SCHEMA CACHE' = 105, 'SYSTEM DROP S3 CLIENT CACHE' = 106, 'SYSTEM DROP CACHE' = 107, 'SYSTEM RELOAD CONFIG' = 108, 'SYSTEM RELOAD USERS' = 109, 'SYSTEM RELOAD SYMBOLS' = 110, 'SYSTEM RELOAD DICTIONARY' = 111, 'SYSTEM RELOAD MODEL' = 112, 'SYSTEM RELOAD FUNCTION' = 113, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 114, 'SYSTEM RELOAD' = 115, 'SYSTEM RESTART DISK' = 116, 'SYSTEM MERGES' = 117, 'SYSTEM TTL MERGES' = 118, 'SYSTEM FETCHES' = 119, 'SYSTEM MOVES' = 120, 'SYSTEM DISTRIBUTED SENDS' = 121, 'SYSTEM REPLICATED SENDS' = 122, 'SYSTEM SENDS' = 123, 'SYSTEM REPLICATION QUEUES' = 124, 'SYSTEM DROP REPLICA' = 125, 'SYSTEM SYNC REPLICA' = 126, 'SYSTEM RESTART REPLICA' = 127, 'SYSTEM RESTORE REPLICA' = 128, 'SYSTEM WAIT LOADING PARTS' = 129, 'SYSTEM SYNC DATABASE REPLICA' = 130, 'SYSTEM SYNC TRANSACTION LOG' = 131, 'SYSTEM SYNC FILE CACHE' = 132, 'SYSTEM FLUSH DISTRIBUTED' = 133, 'SYSTEM FLUSH LOGS' = 134, 'SYSTEM FLUSH' = 135, 'SYSTEM THREAD FUZZER' = 136, 'SYSTEM UNFREEZE' = 137, 'SYSTEM' = 138, 'dictGet' = 139, 'addressToLine' = 140, 'addressToLineWithInlines' = 141, 'addressToSymbol' = 142, 'demangle' = 143, 'INTROSPECTION' = 144, 'FILE' = 145, 'URL' = 146, 'REMOTE' = 147, 'MONGO' = 148, 'MEILISEARCH' = 149, 'MYSQL' = 150, 'POSTGRES' = 151, 'SQLITE' = 152, 'ODBC' = 153, 'JDBC' = 154, 'HDFS' = 155, 'S3' = 156, 'HIVE' = 157, 'SOURCES' = 158, 'CLUSTER' = 159, 'ALL' = 160, 'NONE' = 161))
+    `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION CONTROL' = 96, 'SYSTEM SHUTDOWN' = 97, 'SYSTEM DROP DNS CACHE' = 98, 'SYSTEM DROP MARK CACHE' = 99, 'SYSTEM DROP UNCOMPRESSED CACHE' = 100, 'SYSTEM DROP MMAP CACHE' = 101, 'SYSTEM DROP QUERY CACHE' = 102, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 103, 'SYSTEM DROP FILESYSTEM CACHE' = 104, 'SYSTEM DROP SCHEMA CACHE' = 105, 'SYSTEM DROP S3 CLIENT CACHE' = 106, 'SYSTEM DROP CACHE' = 107, 'SYSTEM RELOAD CONFIG' = 108, 'SYSTEM RELOAD USERS' = 109, 'SYSTEM RELOAD SYMBOLS' = 110, 'SYSTEM RELOAD DICTIONARY' = 111, 'SYSTEM RELOAD MODEL' = 112, 'SYSTEM RELOAD FUNCTION' = 113, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 114, 'SYSTEM RELOAD' = 115, 'SYSTEM RESTART DISK' = 116, 'SYSTEM MERGES' = 117, 'SYSTEM TTL MERGES' = 118, 'SYSTEM FETCHES' = 119, 'SYSTEM MOVES' = 120, 'SYSTEM DISTRIBUTED SENDS' = 121, 'SYSTEM REPLICATED SENDS' = 122, 'SYSTEM SENDS' = 123, 'SYSTEM REPLICATION QUEUES' = 124, 'SYSTEM DROP REPLICA' = 125, 'SYSTEM SYNC REPLICA' = 126, 'SYSTEM RESTART REPLICA' = 127, 'SYSTEM RESTORE REPLICA' = 128, 'SYSTEM WAIT LOADING PARTS' = 129, 'SYSTEM SYNC DATABASE REPLICA' = 130, 'SYSTEM SYNC TRANSACTION LOG' = 131, 'SYSTEM SYNC FILE CACHE' = 132, 'SYSTEM FLUSH DISTRIBUTED' = 133, 'SYSTEM FLUSH LOGS' = 134, 'SYSTEM FLUSH' = 135, 'SYSTEM THREAD FUZZER' = 136, 'SYSTEM UNFREEZE' = 137, 'SYSTEM FAILPOINT' = 138, 'SYSTEM' = 139, 'dictGet' = 140, 'addressToLine' = 141, 'addressToLineWithInlines' = 142, 'addressToSymbol' = 143, 'demangle' = 144, 'INTROSPECTION' = 145, 'FILE' = 146, 'URL' = 147, 'REMOTE' = 148, 'MONGO' = 149, 'MEILISEARCH' = 150, 'MYSQL' = 151, 'POSTGRES' = 152, 'SQLITE' = 153, 'ODBC' = 154, 'JDBC' = 155, 'HDFS' = 156, 'S3' = 157, 'HIVE' = 158, 'SOURCES' = 159, 'CLUSTER' = 160, 'ALL' = 161, 'NONE' = 162))
 )
 ENGINE = SystemPrivileges
 COMMENT 'SYSTEM TABLE is built on the fly.'

From 2b21b8397925104e4cdebc8f3290df9d15b54328 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 2 May 2023 13:35:11 +0200
Subject: [PATCH 113/689] Reduce inter-header dependencies

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Functions/FunctionHelpers.h                  | 2 +-
 src/Functions/IFunction.cpp                      | 1 +
 src/Interpreters/AggregationCommon.h             | 1 -
 src/Interpreters/Context.h                       | 3 ---
 src/Planner/PlannerActionsVisitor.h              | 2 --
 src/Planner/PlannerContext.h                     | 2 --
 src/Storages/MarkCache.h                         | 1 +
 src/Storages/MergeTree/MergeTreeReaderStream.cpp | 1 +
 8 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/src/Functions/FunctionHelpers.h b/src/Functions/FunctionHelpers.h
index 18a4e584080..77affe8488d 100644
--- a/src/Functions/FunctionHelpers.h
+++ b/src/Functions/FunctionHelpers.h
@@ -7,8 +7,8 @@
 #include <Columns/IColumn.h>
 #include <Columns/ColumnArray.h>
 #include <Columns/ColumnConst.h>
-#include <Core/Block.h>
 #include <Core/ColumnNumbers.h>
+#include <Core/ColumnsWithTypeAndName.h>
 #include <Core/callOnTypeIndex.h>
 
 
diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp
index e7f6c521238..7563135f21f 100644
--- a/src/Functions/IFunction.cpp
+++ b/src/Functions/IFunction.cpp
@@ -3,6 +3,7 @@
 #include <Common/typeid_cast.h>
 #include <Common/assert_cast.h>
 #include <Common/SipHash.h>
+#include <Core/Block.h>
 #include <Columns/ColumnConst.h>
 #include <Columns/ColumnNullable.h>
 #include <Columns/ColumnTuple.h>
diff --git a/src/Interpreters/AggregationCommon.h b/src/Interpreters/AggregationCommon.h
index 2e6da40ff1f..7ba9011f18b 100644
--- a/src/Interpreters/AggregationCommon.h
+++ b/src/Interpreters/AggregationCommon.h
@@ -3,7 +3,6 @@
 #include <array>
 
 #include <Common/SipHash.h>
-#include <Common/HashTable/Hash.h>
 #include <Common/memcpySmall.h>
 #include <Common/assert_cast.h>
 #include <Core/Defines.h>
diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h
index 36a6a8e6f49..371a9d18171 100644
--- a/src/Interpreters/Context.h
+++ b/src/Interpreters/Context.h
@@ -7,7 +7,6 @@
 #include <Common/RemoteHostFilter.h>
 #include <Common/ThreadPool_fwd.h>
 #include <Common/Throttler_fwd.h>
-#include <Core/Block.h>
 #include <Core/NamesAndTypes.h>
 #include <Core/Settings.h>
 #include <Core/UUID.h>
@@ -25,12 +24,10 @@
 #include "config.h"
 
 #include <boost/container/flat_set.hpp>
-#include <exception>
 #include <functional>
 #include <memory>
 #include <mutex>
 #include <optional>
-#include <thread>
 
 
 namespace Poco::Net { class IPAddress; }
diff --git a/src/Planner/PlannerActionsVisitor.h b/src/Planner/PlannerActionsVisitor.h
index 2a1c166bfc7..8506c309171 100644
--- a/src/Planner/PlannerActionsVisitor.h
+++ b/src/Planner/PlannerActionsVisitor.h
@@ -1,7 +1,5 @@
 #pragma once
 
-#include <Common/HashTable/Hash.h>
-
 #include <Core/Names.h>
 #include <Core/NamesAndTypes.h>
 
diff --git a/src/Planner/PlannerContext.h b/src/Planner/PlannerContext.h
index ccc4ab43638..4199c863033 100644
--- a/src/Planner/PlannerContext.h
+++ b/src/Planner/PlannerContext.h
@@ -1,7 +1,5 @@
 #pragma once
 
-#include <Common/HashTable/Hash.h>
-
 #include <Core/Names.h>
 #include <Core/NamesAndTypes.h>
 
diff --git a/src/Storages/MarkCache.h b/src/Storages/MarkCache.h
index 2b286ebb1e8..0f4af57fc8e 100644
--- a/src/Storages/MarkCache.h
+++ b/src/Storages/MarkCache.h
@@ -5,6 +5,7 @@
 #include <Common/CacheBase.h>
 #include <Common/ProfileEvents.h>
 #include <Common/SipHash.h>
+#include <Common/HashTable/Hash.h>
 #include <Interpreters/AggregationCommon.h>
 #include <Formats/MarkInCompressedFile.h>
 
diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp
index cdca5aa1247..e4aa0ffa4f4 100644
--- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp
+++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp
@@ -2,6 +2,7 @@
 #include <Compression/CachedCompressedReadBuffer.h>
 
 #include <base/getThreadId.h>
+#include <base/range.h>
 #include <utility>
 
 

From ecfbf1e3040013154a860c4fbf52a337bf946fe1 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Tue, 2 May 2023 19:41:42 +0200
Subject: [PATCH 114/689] Remove dependency from DB::Context in readers

---
 programs/keeper/CMakeLists.txt                |  2 +-
 programs/keeper/Keeper.cpp                    |  2 +-
 src/Compression/CompressionFactory.cpp        |  4 +-
 src/Core/SettingsFields.cpp                   |  2 +-
 src/Daemon/SentryWriter.cpp                   |  2 +-
 ...chronousReadIndirectBufferFromRemoteFS.cpp | 14 ++--
 ...ynchronousReadIndirectBufferFromRemoteFS.h |  9 ++-
 .../IO/CachedOnDiskReadBufferFromFile.cpp     | 11 +--
 src/Disks/IO/CachedOnDiskReadBufferFromFile.h |  6 +-
 .../IO/CachedOnDiskWriteBufferFromFile.cpp    | 38 +++++-----
 src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 24 +++---
 src/Disks/IO/ReadBufferFromRemoteFSGather.h   |  5 +-
 src/Disks/IO/ThreadPoolRemoteFSReader.cpp     | 13 +---
 src/Disks/IO/ThreadPoolRemoteFSReader.h       | 11 ++-
 src/Disks/IO/createReadBufferFromFileBase.cpp | 15 +---
 src/Disks/IO/getThreadPoolReader.cpp          | 76 +++++++++++++++++++
 src/Disks/IO/getThreadPoolReader.h            | 23 ++++++
 .../AzureBlobStorage/AzureObjectStorage.cpp   | 12 ++-
 .../Cached/CachedObjectStorage.cpp            |  9 ++-
 .../ObjectStorages/HDFS/HDFSObjectStorage.cpp |  2 +-
 src/Disks/ObjectStorages/IObjectStorage.cpp   |  9 ---
 src/Disks/ObjectStorages/IObjectStorage.h     |  2 -
 .../ObjectStorages/S3/S3ObjectStorage.cpp     | 11 ++-
 .../ObjectStorages/Web/WebObjectStorage.cpp   | 15 +++-
 src/Interpreters/Context.cpp                  | 44 +----------
 src/Interpreters/Context.h                    | 10 +--
 src/Server/ProtocolServerAdapter.cpp          |  4 +-
 src/Server/ProtocolServerAdapter.h            |  2 +-
 .../HDFS/AsynchronousReadBufferFromHDFS.cpp   |  2 +-
 src/Storages/Hive/StorageHive.cpp             |  3 +-
 src/Storages/StorageS3.cpp                    | 10 ++-
 31 files changed, 235 insertions(+), 157 deletions(-)
 create mode 100644 src/Disks/IO/getThreadPoolReader.cpp
 create mode 100644 src/Disks/IO/getThreadPoolReader.h

diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt
index 761335fb707..e5d56023f7b 100644
--- a/programs/keeper/CMakeLists.txt
+++ b/programs/keeper/CMakeLists.txt
@@ -114,7 +114,7 @@ if (BUILD_STANDALONE_KEEPER)
     clickhouse_add_executable(clickhouse-keeper ${CLICKHOUSE_KEEPER_STANDALONE_SOURCES})
 
     # Remove some redundant dependencies
-    target_compile_definitions (clickhouse-keeper PRIVATE -DKEEPER_STANDALONE_BUILD)
+    target_compile_definitions (clickhouse-keeper PRIVATE -DCLICKHOUSE_PROGRAM_STANDALONE_BUILD)
     target_compile_definitions (clickhouse-keeper PUBLIC -DWITHOUT_TEXT_LOG)
 
     target_include_directories(clickhouse-keeper PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../../src") # uses includes from src directory
diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp
index 3853c955171..0eb3df58f30 100644
--- a/programs/keeper/Keeper.cpp
+++ b/programs/keeper/Keeper.cpp
@@ -57,7 +57,7 @@ int mainEntryClickHouseKeeper(int argc, char ** argv)
     }
 }
 
-#ifdef KEEPER_STANDALONE_BUILD
+#ifdef CLICKHOUSE_PROGRAM_STANDALONE_BUILD
 
 // Weak symbols don't work correctly on Darwin
 // so we have a stub implementation to avoid linker errors
diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp
index 514ae4eee3c..61f4e3a988c 100644
--- a/src/Compression/CompressionFactory.cpp
+++ b/src/Compression/CompressionFactory.cpp
@@ -172,7 +172,7 @@ void registerCodecDeflateQpl(CompressionCodecFactory & factory);
 
 /// Keeper use only general-purpose codecs, so we don't need these special codecs
 /// in standalone build
-#ifndef KEEPER_STANDALONE_BUILD
+#ifndef CLICKHOUSE_PROGRAM_STANDALONE_BUILD
 void registerCodecDelta(CompressionCodecFactory & factory);
 void registerCodecT64(CompressionCodecFactory & factory);
 void registerCodecDoubleDelta(CompressionCodecFactory & factory);
@@ -188,7 +188,7 @@ CompressionCodecFactory::CompressionCodecFactory()
     registerCodecZSTD(*this);
     registerCodecLZ4HC(*this);
     registerCodecMultiple(*this);
-#ifndef KEEPER_STANDALONE_BUILD
+#ifndef CLICKHOUSE_PROGRAM_STANDALONE_BUILD
     registerCodecDelta(*this);
     registerCodecT64(*this);
     registerCodecDoubleDelta(*this);
diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp
index 06cd53013ec..8cd4efb68c6 100644
--- a/src/Core/SettingsFields.cpp
+++ b/src/Core/SettingsFields.cpp
@@ -338,7 +338,7 @@ void SettingFieldString::readBinary(ReadBuffer & in)
 /// that. The linker does not complain only because clickhouse-keeper does not call any of below
 /// functions. A cleaner alternative would be more modular libraries, e.g. one for data types, which
 /// could then be linked by the server and the linker.
-#ifndef KEEPER_STANDALONE_BUILD
+#ifndef CLICKHOUSE_PROGRAM_STANDALONE_BUILD
 
 SettingFieldMap::SettingFieldMap(const Field & f) : value(fieldToMap(f)) {}
 
diff --git a/src/Daemon/SentryWriter.cpp b/src/Daemon/SentryWriter.cpp
index 3c62e54b117..041d3292841 100644
--- a/src/Daemon/SentryWriter.cpp
+++ b/src/Daemon/SentryWriter.cpp
@@ -18,7 +18,7 @@
 #include "config.h"
 #include "config_version.h"
 
-#if USE_SENTRY && !defined(KEEPER_STANDALONE_BUILD)
+#if USE_SENTRY && !defined(CLICKHOUSE_PROGRAM_STANDALONE_BUILD)
 
 #    include <sentry.h>
 #    include <cstdio>
diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
index 9448bbaf798..f02016d2d0c 100644
--- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
+++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
@@ -47,14 +47,14 @@ AsynchronousReadIndirectBufferFromRemoteFS::AsynchronousReadIndirectBufferFromRe
         IAsynchronousReader & reader_,
         const ReadSettings & settings_,
         std::shared_ptr<ReadBufferFromRemoteFSGather> impl_,
-        size_t min_bytes_for_seek_)
+        std::shared_ptr<AsyncReadCounters> async_read_counters_,
+        std::shared_ptr<FilesystemReadPrefetchesLog> prefetches_log_)
     : ReadBufferFromFileBase(settings_.remote_fs_buffer_size, nullptr, 0)
     , read_settings(settings_)
     , reader(reader_)
     , base_priority(settings_.priority)
     , impl(impl_)
     , prefetch_buffer(settings_.prefetch_buffer_size)
-    , min_bytes_for_seek(min_bytes_for_seek_)
     , query_id(CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() != nullptr
                ? CurrentThread::getQueryId() : "")
     , current_reader_id(getRandomASCIIString(8))
@@ -63,6 +63,8 @@ AsynchronousReadIndirectBufferFromRemoteFS::AsynchronousReadIndirectBufferFromRe
 #else
     , log(&Poco::Logger::get("AsyncBuffer(" + impl->getFileName() + ")"))
 #endif
+    , async_read_counters(async_read_counters_)
+    , prefetches_log(prefetches_log_)
 {
     ProfileEvents::increment(ProfileEvents::RemoteFSBuffers);
 }
@@ -111,7 +113,7 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::hasPendingDataToRead()
 std::future<IAsynchronousReader::Result> AsynchronousReadIndirectBufferFromRemoteFS::asyncReadInto(char * data, size_t size, int64_t priority)
 {
     IAsynchronousReader::Request request;
-    request.descriptor = std::make_shared<RemoteFSFileDescriptor>(*impl);
+    request.descriptor = std::make_shared<RemoteFSFileDescriptor>(*impl, async_read_counters);
     request.buf = data;
     request.size = size;
     request.offset = file_offset_of_buffer_end;
@@ -186,8 +188,8 @@ void AsynchronousReadIndirectBufferFromRemoteFS::appendToPrefetchLog(FilesystemP
         .reader_id = current_reader_id,
     };
 
-    if (auto prefetch_log = Context::getGlobalContextInstance()->getFilesystemReadPrefetchesLog())
-        prefetch_log->add(elem);
+    if (prefetches_log)
+        prefetches_log->add(elem);
 }
 
 
@@ -335,7 +337,7 @@ off_t AsynchronousReadIndirectBufferFromRemoteFS::seek(off_t offset, int whence)
     if (impl->initialized()
         && read_until_position && new_pos < *read_until_position
         && new_pos > file_offset_of_buffer_end
-        && new_pos < file_offset_of_buffer_end + min_bytes_for_seek)
+        && new_pos < file_offset_of_buffer_end + read_settings.remote_read_min_bytes_for_seek)
     {
         ProfileEvents::increment(ProfileEvents::RemoteFSLazySeeks);
         bytes_to_ignore = new_pos - file_offset_of_buffer_end;
diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h
index 8cb0e2826b4..49b44916a46 100644
--- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h
+++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h
@@ -12,6 +12,7 @@ namespace Poco { class Logger; }
 namespace DB
 {
 
+struct AsyncReadCounters;
 class ReadBufferFromRemoteFSGather;
 
 /**
@@ -34,7 +35,8 @@ public:
     explicit AsynchronousReadIndirectBufferFromRemoteFS(
         IAsynchronousReader & reader_, const ReadSettings & settings_,
         std::shared_ptr<ReadBufferFromRemoteFSGather> impl_,
-        size_t min_bytes_for_seek = DBMS_DEFAULT_BUFFER_SIZE);
+        std::shared_ptr<AsyncReadCounters> async_read_counters_,
+        std::shared_ptr<FilesystemReadPrefetchesLog> prefetches_log_);
 
     ~AsynchronousReadIndirectBufferFromRemoteFS() override;
 
@@ -83,8 +85,6 @@ private:
 
     Memory<> prefetch_buffer;
 
-    size_t min_bytes_for_seek;
-
     std::string query_id;
 
     std::string current_reader_id;
@@ -95,6 +95,9 @@ private:
 
     Poco::Logger * log;
 
+    std::shared_ptr<AsyncReadCounters> async_read_counters;
+    std::shared_ptr<FilesystemReadPrefetchesLog> prefetches_log;
+
     struct LastPrefetchInfo
     {
         UInt64 submit_time = 0;
diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
index 91a04bde0bf..e92034d442b 100644
--- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
+++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
@@ -48,7 +48,8 @@ CachedOnDiskReadBufferFromFile::CachedOnDiskReadBufferFromFile(
     size_t file_size_,
     bool allow_seeks_after_first_read_,
     bool use_external_buffer_,
-    std::optional<size_t> read_until_position_)
+    std::optional<size_t> read_until_position_,
+    std::shared_ptr<FilesystemCacheLog> cache_log_)
     : ReadBufferFromFileBase(settings_.remote_fs_buffer_size, nullptr, 0, file_size_)
 #ifndef NDEBUG
     , log(&Poco::Logger::get("CachedOnDiskReadBufferFromFile(" + source_file_path_ + ")"))
@@ -62,12 +63,12 @@ CachedOnDiskReadBufferFromFile::CachedOnDiskReadBufferFromFile(
     , read_until_position(read_until_position_ ? *read_until_position_ : file_size_)
     , implementation_buffer_creator(implementation_buffer_creator_)
     , query_id(query_id_)
-    , enable_logging(!query_id.empty() && settings_.enable_filesystem_cache_log)
     , current_buffer_id(getRandomASCIIString(8))
     , allow_seeks_after_first_read(allow_seeks_after_first_read_)
     , use_external_buffer(use_external_buffer_)
     , query_context_holder(cache_->getQueryContextHolder(query_id, settings_))
     , is_persistent(settings_.is_file_cache_persistent)
+    , cache_log(cache_log_)
 {
 }
 
@@ -103,7 +104,7 @@ void CachedOnDiskReadBufferFromFile::appendFilesystemCacheLog(
             break;
     }
 
-    if (auto cache_log = Context::getGlobalContextInstance()->getFilesystemCacheLog())
+    if (cache_log)
         cache_log->add(elem);
 }
 
@@ -488,7 +489,7 @@ bool CachedOnDiskReadBufferFromFile::completeFileSegmentAndGetNext()
     auto * current_file_segment = &file_segments->front();
     auto completed_range = current_file_segment->range();
 
-    if (enable_logging)
+    if (cache_log)
         appendFilesystemCacheLog(completed_range, read_type);
 
     chassert(file_offset_of_buffer_end > completed_range.right);
@@ -513,7 +514,7 @@ bool CachedOnDiskReadBufferFromFile::completeFileSegmentAndGetNext()
 
 CachedOnDiskReadBufferFromFile::~CachedOnDiskReadBufferFromFile()
 {
-    if (enable_logging && file_segments && !file_segments->empty())
+    if (cache_log && file_segments && !file_segments->empty())
     {
         appendFilesystemCacheLog(file_segments->front().range(), read_type);
     }
diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h
index bd8d5af7bcd..86443a848e0 100644
--- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h
+++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h
@@ -33,7 +33,8 @@ public:
         size_t file_size_,
         bool allow_seeks_after_first_read_,
         bool use_external_buffer_,
-        std::optional<size_t> read_until_position_ = std::nullopt);
+        std::optional<size_t> read_until_position_,
+        std::shared_ptr<FilesystemCacheLog> cache_log_);
 
     ~CachedOnDiskReadBufferFromFile() override;
 
@@ -136,7 +137,6 @@ private:
     String last_caller_id;
 
     String query_id;
-    bool enable_logging = false;
     String current_buffer_id;
 
     bool allow_seeks_after_first_read;
@@ -147,6 +147,8 @@ private:
     FileCache::QueryContextHolderPtr query_context_holder;
 
     bool is_persistent;
+
+    std::shared_ptr<FilesystemCacheLog> cache_log;
 };
 
 }
diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp
index d72dcecb484..af2226ea6ca 100644
--- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp
+++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp
@@ -153,27 +153,27 @@ FileSegment & FileSegmentRangeWriter::allocateFileSegment(size_t offset, FileSeg
 
 void FileSegmentRangeWriter::appendFilesystemCacheLog(const FileSegment & file_segment)
 {
-    if (cache_log)
+    if (!cache_log)
+        return;
+
+    auto file_segment_range = file_segment.range();
+    size_t file_segment_right_bound = file_segment_range.left + file_segment.getDownloadedSize(false) - 1;
+
+    FilesystemCacheLogElement elem
     {
-        auto file_segment_range = file_segment.range();
-        size_t file_segment_right_bound = file_segment_range.left + file_segment.getDownloadedSize(false) - 1;
+        .event_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()),
+        .query_id = query_id,
+        .source_file_path = source_path,
+        .file_segment_range = { file_segment_range.left, file_segment_right_bound },
+        .requested_range = {},
+        .cache_type = FilesystemCacheLogElement::CacheType::WRITE_THROUGH_CACHE,
+        .file_segment_size = file_segment_range.size(),
+        .read_from_cache_attempted = false,
+        .read_buffer_id = {},
+        .profile_counters = nullptr,
+    };
 
-        FilesystemCacheLogElement elem
-        {
-            .event_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()),
-            .query_id = query_id,
-            .source_file_path = source_path,
-            .file_segment_range = { file_segment_range.left, file_segment_right_bound },
-            .requested_range = {},
-            .cache_type = FilesystemCacheLogElement::CacheType::WRITE_THROUGH_CACHE,
-            .file_segment_size = file_segment_range.size(),
-            .read_from_cache_attempted = false,
-            .read_buffer_id = {},
-            .profile_counters = nullptr,
-        };
-
-        cache_log->add(elem);
-    }
+    cache_log->add(elem);
 }
 
 void FileSegmentRangeWriter::completeFileSegment(FileSegment & file_segment)
diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
index 344abcbc95b..7eda102ebc8 100644
--- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
+++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
@@ -8,7 +8,6 @@
 #include <iostream>
 #include <base/hex.h>
 #include <Interpreters/FilesystemCacheLog.h>
-#include <Interpreters/Context.h>
 
 
 namespace DB
@@ -22,18 +21,21 @@ namespace ErrorCodes
 ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather(
     ReadBufferCreator && read_buffer_creator_,
     const StoredObjects & blobs_to_read_,
-    const ReadSettings & settings_)
+    const ReadSettings & settings_,
+    std::shared_ptr<FilesystemCacheLog> cache_log_)
     : ReadBuffer(nullptr, 0)
     , read_buffer_creator(std::move(read_buffer_creator_))
     , blobs_to_read(blobs_to_read_)
     , settings(settings_)
     , query_id(CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() != nullptr ? CurrentThread::getQueryId() : "")
     , log(&Poco::Logger::get("ReadBufferFromRemoteFSGather"))
-    , enable_cache_log(!query_id.empty() && settings.enable_filesystem_cache_log)
 {
     if (blobs_to_read.empty())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read zero number of objects");
 
+    if (cache_log_ && settings.enable_filesystem_cache_log)
+        cache_log = cache_log_;
+
     current_object = blobs_to_read.front();
 
     with_cache = settings.remote_fs_cache
@@ -43,7 +45,7 @@ ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather(
 
 SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(const StoredObject & object)
 {
-    if (current_buf != nullptr && !with_cache && enable_cache_log)
+    if (current_buf != nullptr && !with_cache)
     {
         appendFilesystemCacheLog();
     }
@@ -68,7 +70,8 @@ SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(c
             object.bytes_size,
             /* allow_seeks */false,
             /* use_external_buffer */true,
-            read_until_position ? std::optional<size_t>(read_until_position) : std::nullopt);
+            read_until_position ? std::optional<size_t>(read_until_position) : std::nullopt,
+            cache_log);
     }
 
     return current_read_buffer_creator();
@@ -76,6 +79,9 @@ SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(c
 
 void ReadBufferFromRemoteFSGather::appendFilesystemCacheLog()
 {
+    if (!cache_log)
+        return;
+
     chassert(!current_object.absolute_path.empty());
     FilesystemCacheLogElement elem
     {
@@ -87,9 +93,7 @@ void ReadBufferFromRemoteFSGather::appendFilesystemCacheLog()
         .file_segment_size = total_bytes_read_from_current_file,
         .read_from_cache_attempted = false,
     };
-
-    if (auto cache_log = Context::getGlobalContextInstance()->getFilesystemCacheLog())
-        cache_log->add(elem);
+    cache_log->add(elem);
 }
 
 IAsynchronousReader::Result ReadBufferFromRemoteFSGather::readInto(char * data, size_t size, size_t offset, size_t ignore)
@@ -275,10 +279,8 @@ size_t ReadBufferFromRemoteFSGather::getImplementationBufferOffset() const
 
 ReadBufferFromRemoteFSGather::~ReadBufferFromRemoteFSGather()
 {
-    if (!with_cache && enable_cache_log)
-    {
+    if (!with_cache)
         appendFilesystemCacheLog();
-    }
 }
 
 }
diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h
index abe57647a48..27ee4ca399c 100644
--- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h
+++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h
@@ -25,7 +25,8 @@ public:
     ReadBufferFromRemoteFSGather(
         ReadBufferCreator && read_buffer_creator_,
         const StoredObjects & blobs_to_read_,
-        const ReadSettings & settings_);
+        const ReadSettings & settings_,
+        std::shared_ptr<FilesystemCacheLog> cache_log_);
 
     ~ReadBufferFromRemoteFSGather() override;
 
@@ -93,7 +94,7 @@ private:
 
     size_t total_bytes_read_from_current_file = 0;
 
-    bool enable_cache_log = false;
+    std::shared_ptr<FilesystemCacheLog> cache_log;
 };
 
 }
diff --git a/src/Disks/IO/ThreadPoolRemoteFSReader.cpp b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp
index 4d0f39357ab..b7e598e2a87 100644
--- a/src/Disks/IO/ThreadPoolRemoteFSReader.cpp
+++ b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp
@@ -11,7 +11,6 @@
 #include <Common/ElapsedTimeProfileEventIncrement.h>
 #include <IO/SeekableReadBuffer.h>
 #include <IO/AsyncReadCounters.h>
-#include <Interpreters/Context.h>
 #include <base/getThreadId.h>
 
 #include <future>
@@ -75,17 +74,11 @@ std::future<IAsynchronousReader::Result> ThreadPoolRemoteFSReader::submit(Reques
     return scheduleFromThreadPool<Result>([request]() -> Result
     {
         CurrentMetrics::Increment metric_increment{CurrentMetrics::RemoteRead};
-
-        std::optional<AsyncReadIncrement> increment;
-        if (CurrentThread::isInitialized())
-        {
-            auto query_context = CurrentThread::get().getQueryContext();
-            if (query_context)
-                increment.emplace(query_context->getAsyncReadCounters());
-        }
-
         auto * remote_fs_fd = assert_cast<RemoteFSFileDescriptor *>(request.descriptor.get());
 
+        auto async_read_counters = remote_fs_fd->getReadCounters();
+        std::optional<AsyncReadIncrement> increment = async_read_counters ? std::optional<AsyncReadIncrement>(async_read_counters) : std::nullopt;
+
         auto watch = std::make_unique<Stopwatch>(CLOCK_MONOTONIC);
         Result result = remote_fs_fd->readInto(request.buf, request.size, request.offset, request.ignore);
         watch->stop();
diff --git a/src/Disks/IO/ThreadPoolRemoteFSReader.h b/src/Disks/IO/ThreadPoolRemoteFSReader.h
index 3a765993292..506d77a64ef 100644
--- a/src/Disks/IO/ThreadPoolRemoteFSReader.h
+++ b/src/Disks/IO/ThreadPoolRemoteFSReader.h
@@ -8,6 +8,8 @@
 namespace DB
 {
 
+struct AsyncReadCounters;
+
 class ThreadPoolRemoteFSReader : public IAsynchronousReader
 {
 public:
@@ -24,12 +26,19 @@ private:
 class RemoteFSFileDescriptor : public IAsynchronousReader::IFileDescriptor
 {
 public:
-    explicit RemoteFSFileDescriptor(ReadBuffer & reader_) : reader(reader_) { }
+    explicit RemoteFSFileDescriptor(
+        ReadBuffer & reader_,
+        std::shared_ptr<AsyncReadCounters> async_read_counters_)
+        : reader(reader_)
+        , async_read_counters(async_read_counters_) {}
 
     IAsynchronousReader::Result readInto(char * data, size_t size, size_t offset, size_t ignore = 0);
 
+    std::shared_ptr<AsyncReadCounters> getReadCounters() const { return async_read_counters; }
+
 private:
     ReadBuffer & reader;
+    std::shared_ptr<AsyncReadCounters> async_read_counters;
 };
 
 }
diff --git a/src/Disks/IO/createReadBufferFromFileBase.cpp b/src/Disks/IO/createReadBufferFromFileBase.cpp
index 8e9a1d86628..7de26b6c333 100644
--- a/src/Disks/IO/createReadBufferFromFileBase.cpp
+++ b/src/Disks/IO/createReadBufferFromFileBase.cpp
@@ -5,7 +5,9 @@
 #include <IO/AsynchronousReadBufferFromFile.h>
 #include <Disks/IO/IOUringReader.h>
 #include <Disks/IO/ThreadPoolReader.h>
+#include <Disks/IO/getThreadPoolReader.h>
 #include <IO/SynchronousReader.h>
+#include <IO/AsynchronousReader.h>
 #include <Common/ProfileEvents.h>
 #include "config.h"
 
@@ -27,7 +29,6 @@ namespace ErrorCodes
     extern const int UNSUPPORTED_METHOD;
 }
 
-
 std::unique_ptr<ReadBufferFromFileBase> createReadBufferFromFileBase(
     const std::string & filename,
     const ReadSettings & settings,
@@ -119,11 +120,7 @@ std::unique_ptr<ReadBufferFromFileBase> createReadBufferFromFileBase(
         }
         else if (settings.local_fs_method == LocalFSReadMethod::pread_fake_async)
         {
-            auto context = Context::getGlobalContextInstance();
-            if (!context)
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context not initialized");
-
-            auto & reader = context->getThreadPoolReader(Context::FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER);
+            auto & reader = getThreadPoolReader(FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER);
             res = std::make_unique<AsynchronousReadBufferFromFileWithDescriptorsCache>(
                 reader,
                 settings.priority,
@@ -137,11 +134,7 @@ std::unique_ptr<ReadBufferFromFileBase> createReadBufferFromFileBase(
         }
         else if (settings.local_fs_method == LocalFSReadMethod::pread_threadpool)
         {
-            auto context = Context::getGlobalContextInstance();
-            if (!context)
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context not initialized");
-
-            auto & reader = context->getThreadPoolReader(Context::FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER);
+            auto & reader = getThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER);
             res = std::make_unique<AsynchronousReadBufferFromFileWithDescriptorsCache>(
                 reader,
                 settings.priority,
diff --git a/src/Disks/IO/getThreadPoolReader.cpp b/src/Disks/IO/getThreadPoolReader.cpp
new file mode 100644
index 00000000000..d604c934e5c
--- /dev/null
+++ b/src/Disks/IO/getThreadPoolReader.cpp
@@ -0,0 +1,76 @@
+#include <Common/ErrorCodes.h>
+#include <Disks/IO/getThreadPoolReader.h>
+#include <IO/AsynchronousReader.h>
+#include <Poco/Util/AbstractConfiguration.h>
+#include <Poco/Util/Application.h>
+#include <IO/SynchronousReader.h>
+#include <Disks/IO/ThreadPoolRemoteFSReader.h>
+#include <Disks/IO/ThreadPoolReader.h>
+
+#ifndef CLICKHOUSE_PROGRAM_STANDALONE_BUILD
+#include <Interpreters/Context.h>
+#endif
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
+IAsynchronousReader & getThreadPoolReader(FilesystemReaderType type)
+{
+#ifndef CLICKHOUSE_PROGRAM_STANDALONE_BUILD
+    auto context = Context::getGlobalContextInstance();
+    if (!context)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context not initialized");
+    return context->getThreadPoolReader(type);
+#else
+    const auto & config = Poco::Util::Application::instance().config();
+    switch (type)
+    {
+        case FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER:
+        {
+            static auto asynchronous_remote_fs_reader = getThreadPoolReaderImpl(type, config);
+            return *asynchronous_remote_fs_reader;
+        }
+        case FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER:
+        {
+            static auto asynchronous_local_fs_reader = getThreadPoolReaderImpl(type, config);
+            return *asynchronous_local_fs_reader;
+        }
+        case FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER:
+        {
+            static auto synchronous_local_fs_reader = getThreadPoolReaderImpl(type, config);
+            return *synchronous_local_fs_reader;
+        }
+    }
+#endif
+}
+
+std::unique_ptr<IAsynchronousReader> getThreadPoolReaderImpl(
+    FilesystemReaderType type, const Poco::Util::AbstractConfiguration & config)
+{
+    switch (type)
+    {
+        case FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER:
+        {
+            auto pool_size = config.getUInt(".threadpool_remote_fs_reader_pool_size", 250);
+            auto queue_size = config.getUInt(".threadpool_remote_fs_reader_queue_size", 1000000);
+            return std::make_unique<ThreadPoolRemoteFSReader>(pool_size, queue_size);
+        }
+        case FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER:
+        {
+            auto pool_size = config.getUInt(".threadpool_local_fs_reader_pool_size", 100);
+            auto queue_size = config.getUInt(".threadpool_local_fs_reader_queue_size", 1000000);
+            return std::make_unique<ThreadPoolReader>(pool_size, queue_size);
+        }
+        case FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER:
+        {
+            return std::make_unique<SynchronousReader>();
+        }
+    }
+}
+
+}
diff --git a/src/Disks/IO/getThreadPoolReader.h b/src/Disks/IO/getThreadPoolReader.h
new file mode 100644
index 00000000000..54dbca7585d
--- /dev/null
+++ b/src/Disks/IO/getThreadPoolReader.h
@@ -0,0 +1,23 @@
+#pragma once
+
+namespace Poco::Util { class AbstractConfiguration; }
+
+namespace DB
+{
+
+class IAsynchronousReader;
+
+enum class FilesystemReaderType
+{
+    SYNCHRONOUS_LOCAL_FS_READER,
+    ASYNCHRONOUS_LOCAL_FS_READER,
+    ASYNCHRONOUS_REMOTE_FS_READER,
+};
+
+IAsynchronousReader & getThreadPoolReader(FilesystemReaderType type);
+
+std::unique_ptr<IAsynchronousReader> getThreadPoolReaderImpl(
+    FilesystemReaderType type,
+    const Poco::Util::AbstractConfiguration & config);
+
+}
diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
index ad938e345bb..0a8211332d0 100644
--- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
@@ -10,6 +10,7 @@
 #include <Disks/IO/ReadBufferFromRemoteFSGather.h>
 
 #include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.h>
+#include <Interpreters/Context.h>
 #include <Common/logger_useful.h>
 
 
@@ -86,6 +87,7 @@ std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObjects( /// NOL
 {
     ReadSettings disk_read_settings = patchSettings(read_settings);
     auto settings_ptr = settings.get();
+    auto global_context = Context::getGlobalContextInstance();
 
     auto read_buffer_creator =
         [this, settings_ptr, disk_read_settings]
@@ -104,12 +106,16 @@ std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObjects( /// NOL
     auto reader_impl = std::make_unique<ReadBufferFromRemoteFSGather>(
         std::move(read_buffer_creator),
         objects,
-        disk_read_settings);
+        disk_read_settings,
+        global_context->getFilesystemCacheLog());
 
     if (disk_read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
     {
-        auto & reader = getThreadPoolReader();
-        return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, disk_read_settings, std::move(reader_impl));
+        auto & reader = global_context->getThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER);
+        return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(
+            reader, disk_read_settings, std::move(reader_impl),
+            global_context->getAsyncReadCounters(),
+            global_context->getFilesystemReadPrefetchesLog());
     }
     else
     {
diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp
index b8022e9ed3a..159c2220a9c 100644
--- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp
@@ -4,6 +4,7 @@
 #include <IO/BoundedReadBuffer.h>
 #include <Disks/IO/CachedOnDiskWriteBufferFromFile.h>
 #include <Disks/IO/CachedOnDiskReadBufferFromFile.h>
+#include <Interpreters/Context.h>
 #include <Interpreters/Cache/FileCache.h>
 #include <Interpreters/Cache/FileCacheFactory.h>
 #include <Common/CurrentThread.h>
@@ -117,7 +118,9 @@ std::unique_ptr<ReadBufferFromFileBase> CachedObjectStorage::readObjects( /// NO
             CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() ? std::string(CurrentThread::getQueryId()) : "",
             file_size.value(),
             /* allow_seeks */true,
-            /* use_external_buffer */false);
+            /* use_external_buffer */false,
+            std::nullopt,
+            Context::getGlobalContextInstance()->getFilesystemCacheLog());
     }
 }
 
@@ -158,7 +161,9 @@ std::unique_ptr<ReadBufferFromFileBase> CachedObjectStorage::readObject( /// NOL
             CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() ? std::string(CurrentThread::getQueryId()) : "",
             file_size.value(),
             /* allow_seeks */true,
-            /* use_external_buffer */false);
+            /* use_external_buffer */false,
+            std::nullopt,
+            Context::getGlobalContextInstance()->getFilesystemCacheLog());
     }
 }
 
diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
index 1d6c6f92280..bf55fb8f668 100644
--- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
@@ -74,7 +74,7 @@ std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObjects( /// NOLI
             hdfs_uri, hdfs_path, config, disk_read_settings, /* read_until_position */0, /* use_external_buffer */true);
     };
 
-    auto hdfs_impl = std::make_unique<ReadBufferFromRemoteFSGather>(std::move(read_buffer_creator), objects, disk_read_settings);
+    auto hdfs_impl = std::make_unique<ReadBufferFromRemoteFSGather>(std::move(read_buffer_creator), objects, disk_read_settings, nullptr);
     auto buf = std::make_unique<ReadIndirectBufferFromRemoteFS>(std::move(hdfs_impl), read_settings);
     return std::make_unique<SeekAvoidingReadBuffer>(std::move(buf), settings->min_bytes_for_seek);
 }
diff --git a/src/Disks/ObjectStorages/IObjectStorage.cpp b/src/Disks/ObjectStorages/IObjectStorage.cpp
index f70e2d96285..1a8e9ae5a13 100644
--- a/src/Disks/ObjectStorages/IObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/IObjectStorage.cpp
@@ -25,15 +25,6 @@ void IObjectStorage::getDirectoryContents(const std::string &,
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getDirectoryContents() is not supported");
 }
 
-IAsynchronousReader & IObjectStorage::getThreadPoolReader()
-{
-    auto context = Context::getGlobalContextInstance();
-    if (!context)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context not initialized");
-
-    return context->getThreadPoolReader(Context::FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER);
-}
-
 ThreadPool & IObjectStorage::getThreadPoolWriter()
 {
     auto context = Context::getGlobalContextInstance();
diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h
index 44fe30499ff..f9e79ad7d55 100644
--- a/src/Disks/ObjectStorages/IObjectStorage.h
+++ b/src/Disks/ObjectStorages/IObjectStorage.h
@@ -157,8 +157,6 @@ public:
 
     virtual const std::string & getCacheName() const;
 
-    static IAsynchronousReader & getThreadPoolReader();
-
     static ThreadPool & getThreadPoolWriter();
 
     virtual void shutdown() = 0;
diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
index 83c0c7446a8..c4f7d35d4d7 100644
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
@@ -118,6 +118,7 @@ std::unique_ptr<ReadBufferFromFileBase> S3ObjectStorage::readObjects( /// NOLINT
     assert(!objects[0].getPathKeyForCache().empty());
 
     ReadSettings disk_read_settings = patchSettings(read_settings);
+    auto global_context = Context::getGlobalContextInstance();
 
     auto settings_ptr = s3_settings.get();
 
@@ -141,12 +142,16 @@ std::unique_ptr<ReadBufferFromFileBase> S3ObjectStorage::readObjects( /// NOLINT
     auto s3_impl = std::make_unique<ReadBufferFromRemoteFSGather>(
         std::move(read_buffer_creator),
         objects,
-        disk_read_settings);
+        disk_read_settings,
+        global_context->getFilesystemCacheLog());
 
     if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
     {
-        auto & reader = getThreadPoolReader();
-        return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, disk_read_settings, std::move(s3_impl));
+        auto & reader = global_context->getThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER);
+        return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(
+            reader, disk_read_settings, std::move(s3_impl),
+            global_context->getAsyncReadCounters(),
+            global_context->getFilesystemReadPrefetchesLog());
     }
     else
     {
diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp
index 4365156d93d..084b99748c6 100644
--- a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp
@@ -13,6 +13,7 @@
 #include <Disks/IO/ReadBufferFromRemoteFSGather.h>
 #include <Disks/IO/ReadBufferFromWebServer.h>
 #include <Disks/IO/ThreadPoolRemoteFSReader.h>
+#include <Disks/IO/getThreadPoolReader.h>
 
 #include <Storages/MergeTree/MergeTreeData.h>
 
@@ -179,12 +180,20 @@ std::unique_ptr<ReadBufferFromFileBase> WebObjectStorage::readObject( /// NOLINT
              read_until_position);
      };
 
-    auto web_impl = std::make_unique<ReadBufferFromRemoteFSGather>(std::move(read_buffer_creator), StoredObjects{object}, read_settings);
+    auto global_context = Context::getGlobalContextInstance();
+    auto web_impl = std::make_unique<ReadBufferFromRemoteFSGather>(
+        std::move(read_buffer_creator),
+        StoredObjects{object},
+        read_settings,
+        global_context->getFilesystemCacheLog());
 
     if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
     {
-        auto & reader = IObjectStorage::getThreadPoolReader();
-        return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, read_settings, std::move(web_impl), min_bytes_for_seek);
+        auto & reader = global_context->getThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER);
+        return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(
+            reader, read_settings, std::move(web_impl),
+            global_context->getAsyncReadCounters(),
+            global_context->getFilesystemReadPrefetchesLog());
     }
     else
     {
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index 944ff6dfa24..b9207987a9f 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -4159,35 +4159,8 @@ OrdinaryBackgroundExecutorPtr Context::getCommonExecutor() const
     return shared->common_executor;
 }
 
-static size_t getThreadPoolReaderSizeFromConfig(Context::FilesystemReaderType type, const Poco::Util::AbstractConfiguration & config)
-{
-    switch (type)
-    {
-        case Context::FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER:
-        {
-            return config.getUInt(".threadpool_remote_fs_reader_pool_size", 250);
-        }
-        case Context::FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER:
-        {
-            return config.getUInt(".threadpool_local_fs_reader_pool_size", 100);
-        }
-        case Context::FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER:
-        {
-            return std::numeric_limits<std::size_t>::max();
-        }
-    }
-}
-
-size_t Context::getThreadPoolReaderSize(FilesystemReaderType type) const
-{
-    const auto & config = getConfigRef();
-    return getThreadPoolReaderSizeFromConfig(type, config);
-}
-
 IAsynchronousReader & Context::getThreadPoolReader(FilesystemReaderType type) const
 {
-    const auto & config = getConfigRef();
-
     auto lock = getLock();
 
     switch (type)
@@ -4195,31 +4168,20 @@ IAsynchronousReader & Context::getThreadPoolReader(FilesystemReaderType type) co
         case FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER:
         {
             if (!shared->asynchronous_remote_fs_reader)
-            {
-                auto pool_size = getThreadPoolReaderSizeFromConfig(type, config);
-                auto queue_size = config.getUInt(".threadpool_remote_fs_reader_queue_size", 1000000);
-                shared->asynchronous_remote_fs_reader = std::make_unique<ThreadPoolRemoteFSReader>(pool_size, queue_size);
-            }
-
+                shared->asynchronous_remote_fs_reader = getThreadPoolReaderImpl(type, getConfigRef());
             return *shared->asynchronous_remote_fs_reader;
         }
         case FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER:
         {
             if (!shared->asynchronous_local_fs_reader)
-            {
-                auto pool_size = getThreadPoolReaderSizeFromConfig(type, config);
-                auto queue_size = config.getUInt(".threadpool_local_fs_reader_queue_size", 1000000);
-                shared->asynchronous_local_fs_reader = std::make_unique<ThreadPoolReader>(pool_size, queue_size);
-            }
+                shared->asynchronous_local_fs_reader = getThreadPoolReaderImpl(type, getConfigRef());
 
             return *shared->asynchronous_local_fs_reader;
         }
         case FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER:
         {
             if (!shared->synchronous_local_fs_reader)
-            {
-                shared->synchronous_local_fs_reader = std::make_unique<SynchronousReader>();
-            }
+                shared->synchronous_local_fs_reader = getThreadPoolReaderImpl(type, getConfigRef());
 
             return *shared->synchronous_local_fs_reader;
         }
diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h
index 36a6a8e6f49..a8a0b7cc073 100644
--- a/src/Interpreters/Context.h
+++ b/src/Interpreters/Context.h
@@ -12,6 +12,7 @@
 #include <Core/Settings.h>
 #include <Core/UUID.h>
 #include <IO/AsyncReadCounters.h>
+#include <Disks/IO/getThreadPoolReader.h>
 #include <Interpreters/ClientInfo.h>
 #include <Interpreters/Context_fwd.h>
 #include <Interpreters/DatabaseCatalog.h>
@@ -1098,17 +1099,8 @@ public:
     OrdinaryBackgroundExecutorPtr getFetchesExecutor() const;
     OrdinaryBackgroundExecutorPtr getCommonExecutor() const;
 
-    enum class FilesystemReaderType
-    {
-        SYNCHRONOUS_LOCAL_FS_READER,
-        ASYNCHRONOUS_LOCAL_FS_READER,
-        ASYNCHRONOUS_REMOTE_FS_READER,
-    };
-
     IAsynchronousReader & getThreadPoolReader(FilesystemReaderType type) const;
 
-    size_t getThreadPoolReaderSize(FilesystemReaderType type) const;
-
     std::shared_ptr<AsyncReadCounters> getAsyncReadCounters() const;
 
     ThreadPool & getThreadPoolWriter() const;
diff --git a/src/Server/ProtocolServerAdapter.cpp b/src/Server/ProtocolServerAdapter.cpp
index dbc676432f5..915b6265993 100644
--- a/src/Server/ProtocolServerAdapter.cpp
+++ b/src/Server/ProtocolServerAdapter.cpp
@@ -1,7 +1,7 @@
 #include <Server/ProtocolServerAdapter.h>
 #include <Server/TCPServer.h>
 
-#if USE_GRPC && !defined(KEEPER_STANDALONE_BUILD)
+#if USE_GRPC && !defined(CLICKHOUSE_PROGRAM_STANDALONE_BUILD)
 #include <Server/GRPCServer.h>
 #endif
 
@@ -37,7 +37,7 @@ ProtocolServerAdapter::ProtocolServerAdapter(
 {
 }
 
-#if USE_GRPC && !defined(KEEPER_STANDALONE_BUILD)
+#if USE_GRPC && !defined(CLICKHOUSE_PROGRAM_STANDALONE_BUILD)
 class ProtocolServerAdapter::GRPCServerAdapterImpl : public Impl
 {
 public:
diff --git a/src/Server/ProtocolServerAdapter.h b/src/Server/ProtocolServerAdapter.h
index 514354f9723..e08b12e67f2 100644
--- a/src/Server/ProtocolServerAdapter.h
+++ b/src/Server/ProtocolServerAdapter.h
@@ -23,7 +23,7 @@ public:
     ProtocolServerAdapter & operator =(ProtocolServerAdapter && src) = default;
     ProtocolServerAdapter(const std::string & listen_host_, const char * port_name_, const std::string & description_, std::unique_ptr<TCPServer> tcp_server_);
 
-#if USE_GRPC && !defined(KEEPER_STANDALONE_BUILD)
+#if USE_GRPC && !defined(CLICKHOUSE_PROGRAM_STANDALONE_BUILD)
     ProtocolServerAdapter(const std::string & listen_host_, const char * port_name_, const std::string & description_, std::unique_ptr<GRPCServer> grpc_server_);
 #endif
 
diff --git a/src/Storages/HDFS/AsynchronousReadBufferFromHDFS.cpp b/src/Storages/HDFS/AsynchronousReadBufferFromHDFS.cpp
index c683d59579b..a8502b1bd65 100644
--- a/src/Storages/HDFS/AsynchronousReadBufferFromHDFS.cpp
+++ b/src/Storages/HDFS/AsynchronousReadBufferFromHDFS.cpp
@@ -67,7 +67,7 @@ bool AsynchronousReadBufferFromHDFS::hasPendingDataToRead()
 std::future<IAsynchronousReader::Result> AsynchronousReadBufferFromHDFS::asyncReadInto(char * data, size_t size, int64_t priority)
 {
     IAsynchronousReader::Request request;
-    request.descriptor = std::make_shared<RemoteFSFileDescriptor>(*impl);
+    request.descriptor = std::make_shared<RemoteFSFileDescriptor>(*impl, nullptr);
     request.buf = data;
     request.size = size;
     request.offset = file_offset_of_buffer_end;
diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp
index 965b27b4c05..f554a14ec75 100644
--- a/src/Storages/Hive/StorageHive.cpp
+++ b/src/Storages/Hive/StorageHive.cpp
@@ -21,6 +21,7 @@
 #include <Interpreters/ExpressionAnalyzer.h>
 #include <Interpreters/TreeRewriter.h>
 #include <IO/ReadBufferFromString.h>
+#include <Disks/IO/getThreadPoolReader.h>
 #include <Storages/Cache/ExternalDataSourceCache.h>
 #include <Parsers/ASTExpressionList.h>
 #include <Parsers/ASTCreateQuery.h>
@@ -232,7 +233,7 @@ public:
                         if (thread_pool_read)
                         {
                             return std::make_unique<AsynchronousReadBufferFromHDFS>(
-                                IObjectStorage::getThreadPoolReader(), read_settings, std::move(buf));
+                                getThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER), read_settings, std::move(buf));
                         }
                         else
                         {
diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index bd3e8fe886d..125135a5d1a 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -646,6 +646,7 @@ StorageS3Source::ReadBufferOrFactory StorageS3Source::createS3ReadBuffer(const S
 std::unique_ptr<ReadBuffer> StorageS3Source::createAsyncS3ReadBuffer(
     const String & key, const ReadSettings & read_settings, size_t object_size)
 {
+    auto context = getContext();
     auto read_buffer_creator =
         [this, read_settings, object_size]
         (const std::string & path, size_t read_until_position) -> std::shared_ptr<ReadBufferFromFileBase>
@@ -667,10 +668,13 @@ std::unique_ptr<ReadBuffer> StorageS3Source::createAsyncS3ReadBuffer(
     auto s3_impl = std::make_unique<ReadBufferFromRemoteFSGather>(
         std::move(read_buffer_creator),
         StoredObjects{StoredObject{key, object_size}},
-        read_settings);
+        read_settings,
+        context->getFilesystemCacheLog());
 
-    auto & pool_reader = getContext()->getThreadPoolReader(Context::FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER);
-    auto async_reader = std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(pool_reader, read_settings, std::move(s3_impl));
+    auto & pool_reader = context->getThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER);
+    auto async_reader = std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(
+        pool_reader, read_settings, std::move(s3_impl),
+        context->getAsyncReadCounters(), context->getFilesystemReadPrefetchesLog());
 
     async_reader->setReadUntilEnd();
     if (read_settings.remote_fs_prefetch)

From d8d0d78db80c5468f27dbeac4ffbfb6f04bd4e70 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Wed, 3 May 2023 00:09:52 +0200
Subject: [PATCH 115/689] Fix build

---
 src/Storages/examples/async_read_buffer_from_hdfs.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Storages/examples/async_read_buffer_from_hdfs.cpp b/src/Storages/examples/async_read_buffer_from_hdfs.cpp
index 17aa5479de5..4f6aed8ef65 100644
--- a/src/Storages/examples/async_read_buffer_from_hdfs.cpp
+++ b/src/Storages/examples/async_read_buffer_from_hdfs.cpp
@@ -6,6 +6,7 @@
 #include <IO/copyData.h>
 #include <IO/WriteBufferFromString.h>
 #include <Disks/ObjectStorages/IObjectStorage.h>
+#include <Disks/IO/getThreadPoolReader.h>
 #include <Interpreters/Context.h>
 #include <Common/Config/ConfigProcessor.h>
 #include <Storages/HDFS/AsynchronousReadBufferFromHDFS.h>
@@ -25,7 +26,7 @@ int main()
     String path = "/path/to/hdfs/file";
     ReadSettings settings = {};
     auto in = std::make_unique<ReadBufferFromHDFS>(hdfs_namenode_url, path, *config, settings);
-    auto & reader = IObjectStorage::getThreadPoolReader();
+    auto & reader = getThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER);
     AsynchronousReadBufferFromHDFS buf(reader, {}, std::move(in));
 
     String output;

From f449df85b62119d7e6f2ab99ef986f9e4f7b1ca8 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 3 May 2023 00:31:09 +0200
Subject: [PATCH 116/689] Deprecate in-memory parts

---
 src/Common/ProfileEvents.cpp                  |  2 -
 src/Storages/MergeTree/MergeTreeData.cpp      | 56 +------------
 src/Storages/MergeTree/MergeTreeData.h        |  5 +-
 .../MergeTree/MergeTreeDataPartType.h         |  2 +-
 src/Storages/MergeTree/MergeTreeSettings.h    | 19 +++--
 .../MergeTree/MergeTreeWriteAheadLog.cpp      | 47 -----------
 src/Storages/StorageMergeTree.cpp             |  8 --
 src/Storages/StorageMergeTree.h               |  1 -
 src/Storages/StorageReplicatedMergeTree.cpp   |  8 --
 src/Storages/StorageReplicatedMergeTree.h     |  1 -
 tests/integration/test_backup_restore/test.py | 36 ---------
 .../test_polymorphic_parts/test.py            | 53 +------------
 tests/performance/polymorphic_parts_s.xml     |  8 --
 .../01130_in_memory_parts.reference           | 39 ----------
 .../0_stateless/01130_in_memory_parts.sql     | 48 ------------
 .../01130_in_memory_parts_check.reference     |  1 -
 .../01130_in_memory_parts_check.sql           | 13 ----
 .../01130_in_memory_parts_default.reference   |  1 -
 .../01130_in_memory_parts_default.sql         | 21 -----
 .../01130_in_memory_parts_nested.reference    | 25 ------
 .../01130_in_memory_parts_nested.sql          | 19 -----
 .../01130_in_memory_parts_partitons.reference | 60 --------------
 .../01130_in_memory_parts_partitons.sql       | 59 --------------
 .../01475_read_subcolumns_storages.sh         |  1 -
 ...e_condition_rename_clear_zookeeper_long.sh |  4 +-
 .../01600_parts_types_metrics_long.sh         |  2 +-
 .../01643_merge_tree_fsync_smoke.reference    |  2 -
 .../01643_merge_tree_fsync_smoke.sql          |  7 --
 ...eplicated_merge_tree_fsync_smoke.reference |  2 -
 ...1643_replicated_merge_tree_fsync_smoke.sql | 11 ---
 ...6_clickhouse_client_autocomplete.reference |  1 -
 .../01676_clickhouse_client_autocomplete.sh   |  2 -
 .../01710_projection_fetch_long.reference     |  8 +-
 .../01710_projection_fetch_long.sql           |  4 +-
 .../01710_projection_part_check.sql           |  2 +-
 .../02148_in_memory_part_flush.reference      |  4 -
 .../02148_in_memory_part_flush.sql            | 26 -------
 .../02410_inmemory_wal_cleanup.reference      | 35 ---------
 .../02410_inmemory_wal_cleanup.sql            | 29 -------
 .../02423_drop_memory_parts.reference         | 14 ----
 .../0_stateless/02423_drop_memory_parts.sql   | 40 ----------
 .../02559_nested_multiple_levels_default.sql  | 18 +----
 ...rofile_events_from_query_log_and_client.sh |  4 -
 ...merge_tree_insert_zookeeper_long.reference |  1 -
 ...cated_merge_tree_insert_zookeeper_long.sql | 15 ----
 .../02703_max_local_read_bandwidth.sh         |  2 +-
 .../02703_max_local_write_bandwidth.sh        |  2 +-
 .../0_stateless/02704_max_backup_bandwidth.sh |  2 +-
 utils/CMakeLists.txt                          |  1 -
 utils/wal-dump/CMakeLists.txt                 |  2 -
 utils/wal-dump/main.cpp                       | 78 -------------------
 51 files changed, 31 insertions(+), 820 deletions(-)
 delete mode 100644 tests/queries/0_stateless/01130_in_memory_parts.reference
 delete mode 100644 tests/queries/0_stateless/01130_in_memory_parts.sql
 delete mode 100644 tests/queries/0_stateless/01130_in_memory_parts_check.reference
 delete mode 100644 tests/queries/0_stateless/01130_in_memory_parts_check.sql
 delete mode 100644 tests/queries/0_stateless/01130_in_memory_parts_default.reference
 delete mode 100644 tests/queries/0_stateless/01130_in_memory_parts_default.sql
 delete mode 100644 tests/queries/0_stateless/01130_in_memory_parts_nested.reference
 delete mode 100644 tests/queries/0_stateless/01130_in_memory_parts_nested.sql
 delete mode 100644 tests/queries/0_stateless/01130_in_memory_parts_partitons.reference
 delete mode 100644 tests/queries/0_stateless/01130_in_memory_parts_partitons.sql
 delete mode 100644 tests/queries/0_stateless/02148_in_memory_part_flush.reference
 delete mode 100644 tests/queries/0_stateless/02148_in_memory_part_flush.sql
 delete mode 100644 tests/queries/0_stateless/02410_inmemory_wal_cleanup.reference
 delete mode 100644 tests/queries/0_stateless/02410_inmemory_wal_cleanup.sql
 delete mode 100644 tests/queries/0_stateless/02423_drop_memory_parts.reference
 delete mode 100644 tests/queries/0_stateless/02423_drop_memory_parts.sql
 delete mode 100644 tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.reference
 delete mode 100644 tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.sql
 delete mode 100644 utils/wal-dump/CMakeLists.txt
 delete mode 100644 utils/wal-dump/main.cpp

diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp
index 166db25e14c..c7ad55c318a 100644
--- a/src/Common/ProfileEvents.cpp
+++ b/src/Common/ProfileEvents.cpp
@@ -188,10 +188,8 @@
     \
     M(InsertedWideParts, "Number of parts inserted in Wide format.") \
     M(InsertedCompactParts, "Number of parts inserted in Compact format.") \
-    M(InsertedInMemoryParts, "Number of parts inserted in InMemory format.") \
     M(MergedIntoWideParts, "Number of parts merged into Wide format.") \
     M(MergedIntoCompactParts, "Number of parts merged into Compact format.") \
-    M(MergedIntoInMemoryParts, "Number of parts in merged into InMemory format.") \
     \
     M(MergeTreeDataProjectionWriterRows, "Number of rows INSERTed to MergeTree tables projection.") \
     M(MergeTreeDataProjectionWriterUncompressedBytes, "Uncompressed bytes (for columns as they stored in memory) INSERTed to MergeTree tables projection.") \
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 83f5c0d359c..4c848b15f3e 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -117,10 +117,8 @@ namespace ProfileEvents
     extern const Event DelayedInsertsMilliseconds;
     extern const Event InsertedWideParts;
     extern const Event InsertedCompactParts;
-    extern const Event InsertedInMemoryParts;
     extern const Event MergedIntoWideParts;
     extern const Event MergedIntoCompactParts;
-    extern const Event MergedIntoInMemoryParts;
 }
 
 namespace CurrentMetrics
@@ -380,8 +378,7 @@ MergeTreeData::MergeTreeData(
 
     String reason;
     if (!canUsePolymorphicParts(*settings, &reason) && !reason.empty())
-        LOG_WARNING(log, "{} Settings 'min_rows_for_wide_part', 'min_bytes_for_wide_part', "
-            "'min_rows_for_compact_part' and 'min_bytes_for_compact_part' will be ignored.", reason);
+        LOG_WARNING(log, "{} Settings 'min_rows_for_wide_part'and 'min_bytes_for_wide_part' will be ignored.", reason);
 
 #if !USE_ROCKSDB
     if (use_metadata_cache)
@@ -2304,22 +2301,6 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa
     }
 }
 
-void MergeTreeData::flushAllInMemoryPartsIfNeeded()
-{
-    if (getSettings()->in_memory_parts_enable_wal)
-        return;
-
-    auto metadata_snapshot = getInMemoryMetadataPtr();
-    DataPartsVector parts = getDataPartsVectorForInternalUsage();
-    for (const auto & part : parts)
-    {
-        if (auto part_in_memory = asInMemoryPart(part))
-        {
-            part_in_memory->flushToDisk(part_in_memory->getDataPartStorage().getPartDirectory(), metadata_snapshot);
-        }
-    }
-}
-
 size_t MergeTreeData::clearOldPartsFromFilesystem(bool force)
 {
     DataPartsVector parts_to_remove = grabOldParts(force);
@@ -3290,7 +3271,7 @@ void MergeTreeData::checkMutationIsPossible(const MutationCommands & /*commands*
     /// Some validation will be added
 }
 
-MergeTreeDataPartFormat MergeTreeData::choosePartFormat(size_t bytes_uncompressed, size_t rows_count, bool only_on_disk) const
+MergeTreeDataPartFormat MergeTreeData::choosePartFormat(size_t bytes_uncompressed, size_t rows_count) const
 {
     using PartType = MergeTreeDataPartType;
     using PartStorageType = MergeTreeDataPartStorageType;
@@ -3304,9 +3285,6 @@ MergeTreeDataPartFormat MergeTreeData::choosePartFormat(size_t bytes_uncompresse
         return bytes_uncompressed < min_bytes_for || rows_count < min_rows_for;
     };
 
-    if (!only_on_disk && satisfies(settings->min_bytes_for_compact_part, settings->min_rows_for_compact_part))
-        return {PartType::InMemory, PartStorageType::Full};
-
     auto part_type = PartType::Wide;
     if (satisfies(settings->min_bytes_for_wide_part, settings->min_rows_for_wide_part))
         part_type = PartType::Compact;
@@ -3316,7 +3294,7 @@ MergeTreeDataPartFormat MergeTreeData::choosePartFormat(size_t bytes_uncompresse
 
 MergeTreeDataPartFormat MergeTreeData::choosePartFormatOnDisk(size_t bytes_uncompressed, size_t rows_count) const
 {
-    return choosePartFormat(bytes_uncompressed, rows_count, true);
+    return choosePartFormat(bytes_uncompressed, rows_count);
 }
 
 MergeTreeDataPartBuilder MergeTreeData::getDataPartBuilder(
@@ -6049,19 +6027,6 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData:
             }
         }
 
-        MergeTreeData::WriteAheadLogPtr wal;
-        auto get_inited_wal = [&] ()
-        {
-            if (!wal)
-                wal = data.getWriteAheadLog();
-            return wal;
-        };
-
-        if (settings->in_memory_parts_enable_wal)
-            for (const auto & part : precommitted_parts)
-                if (auto part_in_memory = asInMemoryPart(part))
-                    get_inited_wal()->addPart(part_in_memory);
-
         NOEXCEPT_SCOPE({
             auto current_time = time(nullptr);
 
@@ -6105,10 +6070,6 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData:
 
                         data.modifyPartState(covered_part, DataPartState::Outdated);
                         data.removePartContributionToColumnAndSecondaryIndexSizes(covered_part);
-
-                        if (settings->in_memory_parts_enable_wal)
-                            if (isInMemoryPart(covered_part))
-                                get_inited_wal()->dropPart(covered_part->name);
                     }
 
                     reduce_parts += covered_parts.size();
@@ -7789,11 +7750,8 @@ bool MergeTreeData::canUsePolymorphicParts(const MergeTreeSettings & settings, S
                     "Table can't create parts with adaptive granularity, but settings"
                     " min_rows_for_wide_part = {}"
                     ", min_bytes_for_wide_part = {}"
-                    ", min_rows_for_compact_part = {}"
-                    ", min_bytes_for_compact_part = {}"
                     ". Parts with non-adaptive granularity can be stored only in Wide (default) format.",
-                    settings.min_rows_for_wide_part,    settings.min_bytes_for_wide_part,
-                    settings.min_rows_for_compact_part, settings.min_bytes_for_compact_part);
+                    settings.min_rows_for_wide_part, settings.min_bytes_for_wide_part);
         }
 
         return false;
@@ -8173,9 +8131,6 @@ void MergeTreeData::incrementInsertedPartsProfileEvent(MergeTreeDataPartType typ
         case MergeTreeDataPartType::Compact:
             ProfileEvents::increment(ProfileEvents::InsertedCompactParts);
             break;
-        case MergeTreeDataPartType::InMemory:
-            ProfileEvents::increment(ProfileEvents::InsertedInMemoryParts);
-            break;
         default:
             break;
     }
@@ -8191,9 +8146,6 @@ void MergeTreeData::incrementMergedPartsProfileEvent(MergeTreeDataPartType type)
         case MergeTreeDataPartType::Compact:
             ProfileEvents::increment(ProfileEvents::MergedIntoCompactParts);
             break;
-        case MergeTreeDataPartType::InMemory:
-            ProfileEvents::increment(ProfileEvents::MergedIntoInMemoryParts);
-            break;
         default:
             break;
     }
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index 42cbd311b86..2c36c6c3637 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -225,7 +225,7 @@ public:
     using OperationDataPartsLock = std::unique_lock<std::mutex>;
     OperationDataPartsLock lockOperationsWithParts() const { return OperationDataPartsLock(operation_with_data_parts_mutex); }
 
-    MergeTreeDataPartFormat choosePartFormat(size_t bytes_uncompressed, size_t rows_count, bool only_on_disk = false) const;
+    MergeTreeDataPartFormat choosePartFormat(size_t bytes_uncompressed, size_t rows_count) const;
     MergeTreeDataPartFormat choosePartFormatOnDisk(size_t bytes_uncompressed, size_t rows_count) const;
     MergeTreeDataPartBuilder getDataPartBuilder(const String & name, const VolumePtr & volume, const String & part_dir) const;
 
@@ -651,9 +651,6 @@ public:
     /// Removes parts from data_parts, they should be in Deleting state
     void removePartsFinally(const DataPartsVector & parts);
 
-    /// When WAL is not enabled, the InMemoryParts need to be persistent.
-    void flushAllInMemoryPartsIfNeeded();
-
     /// Delete irrelevant parts from memory and disk.
     /// If 'force' - don't wait for old_parts_lifetime.
     size_t clearOldPartsFromFilesystem(bool force = false);
diff --git a/src/Storages/MergeTree/MergeTreeDataPartType.h b/src/Storages/MergeTree/MergeTreeDataPartType.h
index bd2acb9ef65..8b06da5167e 100644
--- a/src/Storages/MergeTree/MergeTreeDataPartType.h
+++ b/src/Storages/MergeTree/MergeTreeDataPartType.h
@@ -44,7 +44,7 @@ public:
         /// Data of all columns is stored in one file. Marks are also stored in single file.
         Compact,
 
-        /// Format with buffering data in RAM.
+        /// Format with buffering data in RAM. Obsolete - new parts cannot be created in this format.
         InMemory,
 
         Unknown,
diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h
index 7ea7fab6e5d..8018decca2b 100644
--- a/src/Storages/MergeTree/MergeTreeSettings.h
+++ b/src/Storages/MergeTree/MergeTreeSettings.h
@@ -33,10 +33,6 @@ struct Settings;
     /** Data storing format settings. */ \
     M(UInt64, min_bytes_for_wide_part, 10485760, "Minimal uncompressed size in bytes to create part in wide format instead of compact", 0) \
     M(UInt64, min_rows_for_wide_part, 0, "Minimal number of rows to create part in wide format instead of compact", 0) \
-    M(UInt64, min_bytes_for_compact_part, 0, "Experimental. Minimal uncompressed size in bytes to create part in compact format instead of saving it in RAM", 0) \
-    M(UInt64, min_rows_for_compact_part, 0, "Experimental. Minimal number of rows to create part in compact format instead of saving it in RAM", 0) \
-    M(Bool, in_memory_parts_enable_wal, true, "Whether to write blocks in Native format to write-ahead-log before creation in-memory part", 0) \
-    M(UInt64, write_ahead_log_max_bytes, 1024 * 1024 * 1024, "Rotate WAL, if it exceeds that amount of bytes", 0) \
     M(Float, ratio_of_defaults_for_sparse_serialization, 1.0, "Minimal ratio of number of default values to number of all values in column to store it in sparse serializations. If >= 1, columns will be always written in full serialization.", 0) \
     \
     /** Merge settings. */ \
@@ -59,9 +55,6 @@ struct Settings;
     M(UInt64, min_compressed_bytes_to_fsync_after_fetch, 0, "Minimal number of compressed bytes to do fsync for part after fetch (0 - disabled)", 0) \
     M(Bool, fsync_after_insert, false, "Do fsync for every inserted part. Significantly decreases performance of inserts, not recommended to use with wide parts.", 0) \
     M(Bool, fsync_part_directory, false, "Do fsync for part directory after all part operations (writes, renames, etc.).", 0) \
-    M(UInt64, write_ahead_log_bytes_to_fsync, 100ULL * 1024 * 1024, "Amount of bytes, accumulated in WAL to do fsync.", 0) \
-    M(UInt64, write_ahead_log_interval_ms_to_fsync, 100, "Interval in milliseconds after which fsync for WAL is being done.", 0) \
-    M(Bool, in_memory_parts_insert_sync, false, "If true insert of part with in-memory format will wait for fsync of WAL", 0) \
     M(UInt64, non_replicated_deduplication_window, 0, "How many last blocks of hashes should be kept on disk (0 - disabled).", 0) \
     M(UInt64, max_parts_to_merge_at_once, 100, "Max amount of parts which can be merged at once (0 - disabled). Doesn't affect OPTIMIZE FINAL query.", 0) \
     M(UInt64, merge_selecting_sleep_ms, 5000, "Sleep time for merge selecting when no part selected, a lower setting will trigger selecting tasks in background_schedule_pool frequently which result in large amount of requests to zookeeper in large-scale clusters", 0) \
@@ -183,7 +176,14 @@ struct Settings;
     M(UInt64, replicated_max_parallel_sends_for_table, 0, "Obsolete setting, does nothing.", 0) \
     M(UInt64, replicated_max_parallel_fetches, 0, "Obsolete setting, does nothing.", 0) \
     M(UInt64, replicated_max_parallel_fetches_for_table, 0, "Obsolete setting, does nothing.", 0) \
-    M(Bool, write_final_mark, true, "Obsolete setting, does nothing.", 0) \
+    M(Bool, write_final_mark, true, "Obsolete setting, does nothing.", 0)                                                                                                                                                                                            \
+    M(UInt64, min_bytes_for_compact_part, 0, "Obsolete setting, does nothing.", 0) \
+    M(UInt64, min_rows_for_compact_part, 0, "Obsolete setting, does nothing.", 0) \
+    M(Bool, in_memory_parts_enable_wal, true, "Obsolete setting, does nothing.", 0) \
+    M(UInt64, write_ahead_log_max_bytes, 1024 * 1024 * 1024, "Obsolete setting, does nothing.", 0) \
+    M(UInt64, write_ahead_log_bytes_to_fsync, 100ULL * 1024 * 1024, "Obsolete setting, does nothing.", 0) \
+    M(UInt64, write_ahead_log_interval_ms_to_fsync, 100, "Obsolete setting, does nothing.", 0) \
+    M(Bool, in_memory_parts_insert_sync, false, "Obsolete setting, does nothing.", 0) \
     /// Settings that should not change after the creation of a table.
     /// NOLINTNEXTLINE
 #define APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(M) \
@@ -211,8 +211,7 @@ struct MergeTreeSettings : public BaseSettings<MergeTreeSettingsTraits>
 
     static bool isPartFormatSetting(const String & name)
     {
-        return name == "min_bytes_for_wide_part" || name == "min_rows_for_wide_part"
-            || name == "min_bytes_for_compact_part" || name == "min_rows_for_compact_part";
+        return name == "min_bytes_for_wide_part" || name == "min_rows_for_wide_part";
     }
 
     /// Check that the values are sane taking also query-level settings into account.
diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp
index 23832464758..39c4157a42e 100644
--- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp
+++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp
@@ -84,31 +84,6 @@ void MergeTreeWriteAheadLog::init()
     bytes_at_last_sync = 0;
 }
 
-void MergeTreeWriteAheadLog::addPart(DataPartInMemoryPtr & part)
-{
-    std::unique_lock lock(write_mutex);
-
-    auto part_info = MergeTreePartInfo::fromPartName(part->name, storage.format_version);
-    min_block_number = std::min(min_block_number, part_info.min_block);
-    max_block_number = std::max(max_block_number, part_info.max_block);
-
-    writeIntBinary(WAL_VERSION, *out);
-
-    ActionMetadata metadata{};
-    metadata.part_uuid = part->uuid;
-    metadata.write(*out);
-
-    writeIntBinary(static_cast<UInt8>(ActionType::ADD_PART), *out);
-    writeStringBinary(part->name, *out);
-    block_out->write(part->block);
-    block_out->flush();
-    sync(lock);
-
-    auto max_wal_bytes = storage.getSettings()->write_ahead_log_max_bytes;
-    if (out->count() > max_wal_bytes)
-        rotate(lock);
-}
-
 void MergeTreeWriteAheadLog::dropPart(const String & part_name)
 {
     std::unique_lock lock(write_mutex);
@@ -121,7 +96,6 @@ void MergeTreeWriteAheadLog::dropPart(const String & part_name)
     writeIntBinary(static_cast<UInt8>(ActionType::DROP_PART), *out);
     writeStringBinary(part_name, *out);
     out->next();
-    sync(lock);
 }
 
 void MergeTreeWriteAheadLog::rotate(const std::unique_lock<std::mutex> &)
@@ -269,27 +243,6 @@ MergeTreeData::MutableDataPartsVector MergeTreeWriteAheadLog::restore(
     return result;
 }
 
-void MergeTreeWriteAheadLog::sync(std::unique_lock<std::mutex> & lock)
-{
-    size_t bytes_to_sync = storage.getSettings()->write_ahead_log_bytes_to_fsync;
-    time_t time_to_sync = storage.getSettings()->write_ahead_log_interval_ms_to_fsync;
-    size_t current_bytes = out->count();
-
-    if (bytes_to_sync && current_bytes - bytes_at_last_sync > bytes_to_sync)
-    {
-        sync_task->schedule();
-        bytes_at_last_sync = current_bytes;
-    }
-    else if (time_to_sync && !sync_scheduled)
-    {
-        sync_task->scheduleAfter(time_to_sync);
-        sync_scheduled = true;
-    }
-
-    if (storage.getSettings()->in_memory_parts_insert_sync)
-        sync_cv.wait(lock, [this] { return !sync_scheduled; });
-}
-
 void MergeTreeWriteAheadLog::shutdown()
 {
     {
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 981e856fc83..eaf92356976 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -167,14 +167,6 @@ void StorageMergeTree::startup()
     }
 }
 
-void StorageMergeTree::flush()
-{
-    if (flush_called.exchange(true))
-        return;
-
-    flushAllInMemoryPartsIfNeeded();
-}
-
 void StorageMergeTree::shutdown()
 {
     if (shutdown_called.exchange(true))
diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h
index 3a39ae12856..e0431a5c09a 100644
--- a/src/Storages/StorageMergeTree.h
+++ b/src/Storages/StorageMergeTree.h
@@ -45,7 +45,6 @@ public:
         bool has_force_restore_data_flag);
 
     void startup() override;
-    void flush() override;
     void shutdown() override;
 
     ~StorageMergeTree() override;
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index e2f41f9b454..0d92ad2c4c7 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -4421,14 +4421,6 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread)
     }
 }
 
-void StorageReplicatedMergeTree::flush()
-{
-    if (flush_called.exchange(true))
-        return;
-
-    flushAllInMemoryPartsIfNeeded();
-}
-
 
 void StorageReplicatedMergeTree::partialShutdown()
 {
diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h
index ade4e4f0b4b..02a7ae52adc 100644
--- a/src/Storages/StorageReplicatedMergeTree.h
+++ b/src/Storages/StorageReplicatedMergeTree.h
@@ -113,7 +113,6 @@ public:
     void startup() override;
     void shutdown() override;
     void partialShutdown();
-    void flush() override;
     ~StorageReplicatedMergeTree() override;
 
     static String getDefaultZooKeeperPath(const Poco::Util::AbstractConfiguration & config);
diff --git a/tests/integration/test_backup_restore/test.py b/tests/integration/test_backup_restore/test.py
index 193e638186c..b91a7cfe1c8 100644
--- a/tests/integration/test_backup_restore/test.py
+++ b/tests/integration/test_backup_restore/test.py
@@ -171,39 +171,3 @@ def test_replace_partition(started_cluster):
     assert TSV(res) == expected
 
     instance.query("DROP TABLE IF EXISTS test.tbl3")
-
-
-def test_freeze_in_memory(started_cluster):
-    instance.query(
-        "CREATE TABLE test.t_in_memory(a UInt32, s String) ENGINE = MergeTree ORDER BY a SETTINGS min_rows_for_compact_part = 1000"
-    )
-    instance.query("INSERT INTO test.t_in_memory VALUES (1, 'a')")
-    instance.query("ALTER TABLE test.t_in_memory FREEZE")
-
-    fp_backup = get_last_backup_path(
-        started_cluster.instances["node"], "test", "t_in_memory"
-    )
-    part_path = fp_backup + "/all_1_1_0/"
-
-    assert TSV(
-        instance.query(
-            "SELECT part_type, is_frozen FROM system.parts WHERE database = 'test' AND table = 't_in_memory'"
-        )
-    ) == TSV("InMemory\t1\n")
-    instance.exec_in_container(["test", "-f", part_path + "/data.bin"])
-    assert instance.exec_in_container(["cat", part_path + "/count.txt"]).strip() == "1"
-
-    instance.query(
-        "CREATE TABLE test.t_in_memory_2(a UInt32, s String) ENGINE = MergeTree ORDER BY a"
-    )
-    copy_backup_to_detached(
-        started_cluster.instances["node"], "test", "t_in_memory", "t_in_memory_2"
-    )
-
-    instance.query("ALTER TABLE test.t_in_memory_2 ATTACH PARTITION ID 'all'")
-    assert TSV(
-        instance.query(
-            "SELECT part_type FROM system.parts WHERE database = 'test' AND table = 't_in_memory_2'"
-        )
-    ) == TSV("Compact\n")
-    assert TSV(instance.query("SELECT a, s FROM test.t_in_memory_2")) == TSV("1\ta\n")
diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py
index 361b4855747..006662ff4bf 100644
--- a/tests/integration/test_polymorphic_parts/test.py
+++ b/tests/integration/test_polymorphic_parts/test.py
@@ -49,9 +49,8 @@ def create_tables(name, nodes, node_settings, shard):
             PARTITION BY toYYYYMM(date)
             ORDER BY id
             SETTINGS index_granularity = 64, index_granularity_bytes = {index_granularity_bytes},
-            min_rows_for_wide_part = {min_rows_for_wide_part}, min_rows_for_compact_part = {min_rows_for_compact_part},
-            min_bytes_for_wide_part = 0, min_bytes_for_compact_part = 0,
-            in_memory_parts_enable_wal = 1
+            min_rows_for_wide_part = {min_rows_for_wide_part},
+            min_bytes_for_wide_part = 0
             """.format(
                 name=name, shard=shard, repl=i, **settings
             )
@@ -87,17 +86,14 @@ node2 = cluster.add_instance(
 settings_default = {
     "index_granularity_bytes": 10485760,
     "min_rows_for_wide_part": 512,
-    "min_rows_for_compact_part": 0,
 }
 settings_compact_only = {
     "index_granularity_bytes": 10485760,
     "min_rows_for_wide_part": 1000000,
-    "min_rows_for_compact_part": 0,
 }
 settings_not_adaptive = {
     "index_granularity_bytes": 0,
     "min_rows_for_wide_part": 512,
-    "min_rows_for_compact_part": 0,
 }
 
 node3 = cluster.add_instance(
@@ -116,12 +112,10 @@ node4 = cluster.add_instance(
 settings_compact = {
     "index_granularity_bytes": 10485760,
     "min_rows_for_wide_part": 512,
-    "min_rows_for_compact_part": 0,
 }
 settings_wide = {
     "index_granularity_bytes": 10485760,
     "min_rows_for_wide_part": 0,
-    "min_rows_for_compact_part": 0,
 }
 
 node5 = cluster.add_instance(
@@ -131,12 +125,6 @@ node6 = cluster.add_instance(
     "node6", main_configs=["configs/compact_parts.xml"], with_zookeeper=True
 )
 
-settings_in_memory = {
-    "index_granularity_bytes": 10485760,
-    "min_rows_for_wide_part": 512,
-    "min_rows_for_compact_part": 256,
-}
-
 node9 = cluster.add_instance("node9", with_zookeeper=True, stay_alive=True)
 node10 = cluster.add_instance("node10", with_zookeeper=True)
 
@@ -190,42 +178,6 @@ def start_cluster():
             "shard2",
         )
         create_tables_old_format("polymorphic_table", [node5, node6], "shard3")
-        create_tables(
-            "in_memory_table",
-            [node9, node10],
-            [settings_in_memory, settings_in_memory],
-            "shard4",
-        )
-        create_tables(
-            "wal_table",
-            [node11, node12],
-            [settings_in_memory, settings_in_memory],
-            "shard4",
-        )
-        create_tables(
-            "restore_table",
-            [node11, node12],
-            [settings_in_memory, settings_in_memory],
-            "shard5",
-        )
-        create_tables(
-            "deduplication_table",
-            [node9, node10],
-            [settings_in_memory, settings_in_memory],
-            "shard5",
-        )
-        create_tables(
-            "sync_table",
-            [node9, node10],
-            [settings_in_memory, settings_in_memory],
-            "shard5",
-        )
-        create_tables(
-            "alters_table",
-            [node9, node10],
-            [settings_in_memory, settings_in_memory],
-            "shard5",
-        )
 
         yield cluster
 
@@ -422,7 +374,6 @@ settings7 = {"index_granularity_bytes": 10485760}
 settings8 = {
     "index_granularity_bytes": 10485760,
     "min_rows_for_wide_part": 512,
-    "min_rows_for_compact_part": 0,
 }
 
 
diff --git a/tests/performance/polymorphic_parts_s.xml b/tests/performance/polymorphic_parts_s.xml
index 5021e135bb9..b4dd87a7ae3 100644
--- a/tests/performance/polymorphic_parts_s.xml
+++ b/tests/performance/polymorphic_parts_s.xml
@@ -13,13 +13,6 @@
         SAMPLE BY intHash32(UserID)
         SETTINGS min_bytes_for_wide_part = '10M'
     </create_query>
-    <create_query>
-        CREATE TABLE hits_memory AS hits_10m_single ENGINE = MergeTree()
-        PARTITION BY toYYYYMM(EventDate)
-        ORDER BY (CounterID, EventDate, intHash32(UserID))
-        SAMPLE BY intHash32(UserID)
-        SETTINGS min_bytes_for_compact_part = '1M', min_bytes_for_wide_part = '10M', in_memory_parts_enable_wal = 1
-    </create_query>
     <create_query>
        CREATE TABLE hits_buffer AS hits_10m_single 
        ENGINE = Buffer(default, hits_wide, 1, 0, 0, 10000, 10000, 0, 0)
@@ -38,6 +31,5 @@
 
     <drop_query>DROP TABLE IF EXISTS hits_wide</drop_query>
     <drop_query>DROP TABLE IF EXISTS hits_compact</drop_query>
-    <drop_query>DROP TABLE IF EXISTS hits_memory</drop_query>
     <drop_query>DROP TABLE IF EXISTS hits_buffer</drop_query>
 </test>
diff --git a/tests/queries/0_stateless/01130_in_memory_parts.reference b/tests/queries/0_stateless/01130_in_memory_parts.reference
deleted file mode 100644
index ad5435abb59..00000000000
--- a/tests/queries/0_stateless/01130_in_memory_parts.reference
+++ /dev/null
@@ -1,39 +0,0 @@
-system.parts
-InMemory	2
-1
-1
-Simple selects
-0	0
-1	1
-2	2
-3	0
-4	1
-50	2
-51	0
-52	1
-53	2
-54	0
-34
-0
-20
-10
-Mutations and Alters
-66
-1	1
-2	2
-4	1
-5	2
-7	1
-[1,1]
-[]
-[4,16]
-[]
-[7,49]
-1	1
-2	1
-1	[1,1]
-2	[]
-4	[4,16]
-5	[]
-7	[7,49]
-0
diff --git a/tests/queries/0_stateless/01130_in_memory_parts.sql b/tests/queries/0_stateless/01130_in_memory_parts.sql
deleted file mode 100644
index 2b15ae24763..00000000000
--- a/tests/queries/0_stateless/01130_in_memory_parts.sql
+++ /dev/null
@@ -1,48 +0,0 @@
--- Tags: no-s3-storage
-
-DROP TABLE IF EXISTS in_memory;
-CREATE TABLE in_memory (a UInt32, b UInt32)
-    ENGINE = MergeTree ORDER BY a
-    SETTINGS min_rows_for_compact_part = 1000, min_rows_for_compact_part = 1000;
-
-INSERT INTO in_memory SELECT number, number % 3 FROM numbers(100);
-SELECT 'system.parts';
-SELECT DISTINCT part_type, marks FROM system.parts WHERE database = currentDatabase() AND table = 'in_memory' AND active;
-SELECT DISTINCT data_uncompressed_bytes > 0 FROM system.parts WHERE database = currentDatabase() AND table = 'in_memory' AND active;
-SELECT DISTINCT column_data_uncompressed_bytes > 0 FROM system.parts_columns WHERE database = currentDatabase() AND table = 'in_memory' AND active;
-
-SELECT 'Simple selects';
-
-SELECT * FROM in_memory ORDER BY a LIMIT 5;
-SELECT * FROM in_memory ORDER BY a LIMIT 5 OFFSET 50;
-SELECT count() FROM in_memory WHERE b = 0 SETTINGS max_block_size = 10;
--- Check index
-SELECT count() FROM in_memory WHERE a > 100 SETTINGS max_rows_to_read = 0, force_primary_key = 1;
-SELECT count() FROM in_memory WHERE a >= 10 AND a < 30 SETTINGS force_primary_key = 1;
-SELECT DISTINCT blockSize() FROM in_memory SETTINGS max_block_size = 10;
-
-SELECT 'Mutations and Alters';
-SET mutations_sync = 1;
-
-ALTER TABLE in_memory DELETE WHERE b = 0;
-
-SELECT count() FROM in_memory;
-SELECT * FROM in_memory ORDER BY a LIMIT 5;
-
-ALTER TABLE in_memory ADD COLUMN arr Array(UInt64);
-ALTER TABLE in_memory UPDATE arr = [a, a * a] WHERE b = 1;
-
-SELECT arr FROM in_memory ORDER BY a LIMIT 5;
-
-ALTER TABLE in_memory MODIFY COLUMN b String;
-ALTER TABLE in_memory RENAME COLUMN b to str;
-SELECT DISTINCT str, length(str) FROM in_memory ORDER BY str;
-ALTER TABLE in_memory DROP COLUMN str;
-
-SELECT * FROM in_memory ORDER BY a LIMIT 5;
-
--- in-memory parts works if they're empty.
-ALTER TABLE in_memory DELETE WHERE 1;
-SELECT count() FROM in_memory;
-
-DROP TABLE in_memory;
diff --git a/tests/queries/0_stateless/01130_in_memory_parts_check.reference b/tests/queries/0_stateless/01130_in_memory_parts_check.reference
deleted file mode 100644
index 15f72836ff1..00000000000
--- a/tests/queries/0_stateless/01130_in_memory_parts_check.reference
+++ /dev/null
@@ -1 +0,0 @@
-201901_1_1_0	1	
diff --git a/tests/queries/0_stateless/01130_in_memory_parts_check.sql b/tests/queries/0_stateless/01130_in_memory_parts_check.sql
deleted file mode 100644
index c2f5eba5949..00000000000
--- a/tests/queries/0_stateless/01130_in_memory_parts_check.sql
+++ /dev/null
@@ -1,13 +0,0 @@
--- Tags: no-s3-storage
-
--- Part of 00961_check_table test, but with in-memory parts
-
-SET check_query_single_value_result = 0;
-DROP TABLE IF EXISTS mt_table;
-CREATE TABLE mt_table (d Date, key UInt64, data String) ENGINE = MergeTree() PARTITION BY toYYYYMM(d) ORDER BY key
-    SETTINGS min_rows_for_compact_part = 1000, min_rows_for_compact_part = 1000;
-
-CHECK TABLE mt_table;
-INSERT INTO mt_table VALUES (toDate('2019-01-02'), 1, 'Hello'), (toDate('2019-01-02'), 2, 'World');
-CHECK TABLE mt_table;
-DROP TABLE mt_table;
diff --git a/tests/queries/0_stateless/01130_in_memory_parts_default.reference b/tests/queries/0_stateless/01130_in_memory_parts_default.reference
deleted file mode 100644
index c51afdb6658..00000000000
--- a/tests/queries/0_stateless/01130_in_memory_parts_default.reference
+++ /dev/null
@@ -1 +0,0 @@
-0	bbb_aaa
diff --git a/tests/queries/0_stateless/01130_in_memory_parts_default.sql b/tests/queries/0_stateless/01130_in_memory_parts_default.sql
deleted file mode 100644
index 776d5f89fcf..00000000000
--- a/tests/queries/0_stateless/01130_in_memory_parts_default.sql
+++ /dev/null
@@ -1,21 +0,0 @@
--- Tags: no-s3-storage
--- Test 01266_default_prewhere_reqq, but with in-memory parts
-DROP TABLE IF EXISTS t1;
-
-CREATE TABLE t1
-(
-    date Date, 
-    s1 String,
-    s2 String
-) ENGINE = MergeTree() PARTITION BY toYYYYMMDD(date) ORDER BY (date, s1)
-SETTINGS index_granularity = 8192, min_rows_for_compact_part = 1000, min_rows_for_wide_part = 1000;
-
-
-set max_threads=1;
-
-insert into t1 (date, s1,s2) values(today()-1,'aaa','bbb');
-alter table t1 add column s3 String DEFAULT concat(s2,'_',s1);
--- insert into t1 (date, s1,s2) values(today(),'aaa2','bbb2');
-select ignore(date), s3 from t1 where  s2='bbb';
-
-DROP TABLE t1;
diff --git a/tests/queries/0_stateless/01130_in_memory_parts_nested.reference b/tests/queries/0_stateless/01130_in_memory_parts_nested.reference
deleted file mode 100644
index 3882abb9467..00000000000
--- a/tests/queries/0_stateless/01130_in_memory_parts_nested.reference
+++ /dev/null
@@ -1,25 +0,0 @@
-[0]
-[0,0,0]
-[0,0,0,0,0]
-[0,0,0,0,0,0,0]
-[0,0,0,0,0,0,0,0,0]
-[0]
-[0,0,0]
-[0,0,0,0,0]
-[0,0,0,0,0,0,0]
-[0,0,0,0,0,0,0,0,0]
-[0]
-[0,0,0]
-[0,0,0,0,0]
-[0,0,0,0,0,0,0]
-[0,0,0,0,0,0,0,0,0]
-[0]
-[0,2,4]
-[0,2,4,6,8]
-[0,2,4,6,8,10,12]
-[0,2,4,6,8,10,12,14,16]
-[0]	[0]
-[0,1,2]	[0,2,4]
-[0,1,2,3,4]	[0,2,4,6,8]
-[0,1,2,3,4,5,6]	[0,2,4,6,8,10,12]
-[0,1,2,3,4,5,6,7,8]	[0,2,4,6,8,10,12,14,16]
diff --git a/tests/queries/0_stateless/01130_in_memory_parts_nested.sql b/tests/queries/0_stateless/01130_in_memory_parts_nested.sql
deleted file mode 100644
index 45e778b9f04..00000000000
--- a/tests/queries/0_stateless/01130_in_memory_parts_nested.sql
+++ /dev/null
@@ -1,19 +0,0 @@
--- Tags: no-s3-storage
--- Test 00576_nested_and_prewhere, but with in-memory parts.
-DROP TABLE IF EXISTS nested;
-
-CREATE TABLE nested (x UInt64, filter UInt8, n Nested(a UInt64)) ENGINE = MergeTree ORDER BY x
-    SETTINGS min_rows_for_compact_part = 200000, min_rows_for_wide_part = 300000;
-
-INSERT INTO nested SELECT number, number % 2, range(number % 10) FROM system.numbers LIMIT 100000;
-
-ALTER TABLE nested ADD COLUMN n.b Array(UInt64);
-SELECT DISTINCT n.b FROM nested PREWHERE filter;
-SELECT DISTINCT n.b FROM nested PREWHERE filter SETTINGS max_block_size = 123;
-SELECT DISTINCT n.b FROM nested PREWHERE filter SETTINGS max_block_size = 1234;
-
-ALTER TABLE nested ADD COLUMN n.c Array(UInt64) DEFAULT arrayMap(x -> x * 2, n.a);
-SELECT DISTINCT n.c FROM nested PREWHERE filter;
-SELECT DISTINCT n.a, n.c FROM nested PREWHERE filter;
-
-DROP TABLE nested;
diff --git a/tests/queries/0_stateless/01130_in_memory_parts_partitons.reference b/tests/queries/0_stateless/01130_in_memory_parts_partitons.reference
deleted file mode 100644
index 44cbbed3f57..00000000000
--- a/tests/queries/0_stateless/01130_in_memory_parts_partitons.reference
+++ /dev/null
@@ -1,60 +0,0 @@
-1	2	foo
-1	3	bar
-2	4	aa
-2	5	bb
-2	6	cc
-3	7	qq
-3	8	ww
-3	9	ee
-3	10	rr
-1_1_1_0	InMemory	2
-2_2_2_0	InMemory	3
-3_3_3_0	InMemory	4
-^ init ==================
-2	4	aa
-2	5	bb
-2	6	cc
-3	7	qq
-3	8	ww
-3	9	ee
-3	10	rr
-2_2_2_0	InMemory	3
-3_3_3_0	InMemory	4
-^ drop 1 ==================
-3	7	qq
-3	8	ww
-3	9	ee
-3	10	rr
-3_3_3_0	InMemory	4
-^ detach 2 ==================
-2	4	aa
-2	5	bb
-2	6	cc
-3	7	qq
-3	8	ww
-3	9	ee
-3	10	rr
-2_4_4_0	Compact	3
-3_3_3_0	InMemory	4
-^ attach 2 =================
-2	4	aa
-2	5	bb
-2	6	cc
-3	7	qq
-3	8	ww
-3	9	ee
-3	10	rr
-2_4_4_0	Compact	3
-3_3_3_0	InMemory	4
-^ detach attach ==================
-2	4	aa
-2	5	bb
-2	6	cc
-3	11	tt
-3	12	yy
-t2	2_4_4_0	Compact	3
-t2	3_6_6_0	Compact	2
-t3	3_1_1_0	InMemory	2
-^ replace ==================
-3_1_1_0	InMemory	1	2
-^ freeze ==================
diff --git a/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql b/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql
deleted file mode 100644
index 18da2d2bd30..00000000000
--- a/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql
+++ /dev/null
@@ -1,59 +0,0 @@
--- Tags: no-parallel, no-s3-storage
-
-DROP TABLE IF EXISTS t2;
-
-CREATE TABLE t2(id UInt32, a UInt64, s String)
-    ENGINE = MergeTree ORDER BY a PARTITION BY id
-    SETTINGS min_rows_for_compact_part = 1000, min_rows_for_wide_part = 2000;
-
-SYSTEM STOP MERGES t2;
-
-INSERT INTO t2 VALUES (1, 2, 'foo'), (1, 3, 'bar');
-INSERT INTO t2 VALUES (2, 4, 'aa'), (2, 5, 'bb'), (2, 6, 'cc');
-INSERT INTO t2 VALUES (3, 7, 'qq'), (3, 8, 'ww'), (3, 9, 'ee'), (3, 10, 'rr');
-
-SELECT * FROM t2 ORDER BY a;
-SELECT name, part_type, rows FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name;
-SELECT '^ init ==================';
-
-ALTER TABLE t2 DROP PARTITION 1;
-SELECT * FROM t2 ORDER BY a;
-SELECT name, part_type, rows FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name;
-SELECT '^ drop 1 ==================';
-
-ALTER TABLE t2 DETACH PARTITION 2;
-SELECT * FROM t2 ORDER BY a;
-SELECT name, part_type, rows FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name;
-SELECT '^ detach 2 ==================';
-
-ALTER TABLE t2 ATTACH PARTITION 2;
-SELECT * FROM t2 ORDER BY a;
-SELECT name, part_type, rows FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name;
-SELECT '^ attach 2 =================';
-
-DETACH TABLE t2;
-ATTACH TABLE t2;
-
-SELECT * FROM t2 ORDER BY a;
-SELECT name, part_type, rows FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name;
-SELECT '^ detach attach ==================';
-
-DROP TABLE IF EXISTS t3;
-
-CREATE TABLE t3(id UInt32, a UInt64, s String)
-    ENGINE = MergeTree ORDER BY a PARTITION BY id
-    SETTINGS min_rows_for_compact_part = 1000, min_rows_for_wide_part = 2000;
-
-INSERT INTO t3 VALUES (3, 11, 'tt'), (3, 12, 'yy');
-ALTER TABLE t2 REPLACE PARTITION 3 FROM t3;
-SELECT * FROM t2 ORDER BY a;
-SELECT table, name, part_type, rows FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name;
-SELECT table, name, part_type, rows FROM system.parts WHERE table = 't3' AND active AND database = currentDatabase() ORDER BY name;
-SELECT '^ replace ==================';
-
-ALTER TABLE t3 FREEZE PARTITION 3;
-SELECT name, part_type, is_frozen, rows FROM system.parts WHERE table = 't3' AND active AND database = currentDatabase() ORDER BY name;
-SELECT '^ freeze ==================';
-
-DROP TABLE t2;
-DROP TABLE t3;
diff --git a/tests/queries/0_stateless/01475_read_subcolumns_storages.sh b/tests/queries/0_stateless/01475_read_subcolumns_storages.sh
index d770d5118ac..4a9b9639f78 100755
--- a/tests/queries/0_stateless/01475_read_subcolumns_storages.sh
+++ b/tests/queries/0_stateless/01475_read_subcolumns_storages.sh
@@ -12,7 +12,6 @@ create_query="CREATE TABLE subcolumns(n Nullable(UInt32), a1 Array(UInt32),\
 
 # "StripeLog"
 declare -a ENGINES=("Log" "TinyLog" "Memory" \
-    "MergeTree ORDER BY tuple() SETTINGS min_bytes_for_compact_part='10M'" \
     "MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part='10M'" \
     "MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part=0")
 
diff --git a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.sh b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.sh
index efe24aa3a88..80318ba67fb 100755
--- a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.sh
+++ b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.sh
@@ -8,9 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_renames0"
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_renames50"
 
-
-$CLICKHOUSE_CLIENT --query "CREATE TABLE table_for_renames0 (value UInt64, data String) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_rename', '1') ORDER BY tuple() SETTINGS cleanup_delay_period = 1, cleanup_delay_period_random_add = 0, min_rows_for_compact_part = 100000, min_rows_for_compact_part = 10000000, write_ahead_log_max_bytes = 1"
-
+$CLICKHOUSE_CLIENT --query "CREATE TABLE table_for_renames0 (value UInt64, data String) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_rename', '1') ORDER BY tuple() SETTINGS cleanup_delay_period = 1, cleanup_delay_period_random_add = 0"
 
 $CLICKHOUSE_CLIENT --query "INSERT INTO table_for_renames0 SELECT number, toString(number) FROM numbers(1000)"
 
diff --git a/tests/queries/0_stateless/01600_parts_types_metrics_long.sh b/tests/queries/0_stateless/01600_parts_types_metrics_long.sh
index 0b9afcf633e..5f724e81042 100755
--- a/tests/queries/0_stateless/01600_parts_types_metrics_long.sh
+++ b/tests/queries/0_stateless/01600_parts_types_metrics_long.sh
@@ -35,7 +35,7 @@ $CLICKHOUSE_CLIENT --database_atomic_wait_for_drop_and_detach_synchronously=1 --
 # InMemory - [0..5]
 # Compact  - (5..10]
 # Wide     - >10
-$CLICKHOUSE_CLIENT --query="CREATE TABLE data_01600 (part_type String, key Int) ENGINE = MergeTree PARTITION BY part_type ORDER BY key SETTINGS min_bytes_for_wide_part=0, min_bytes_for_compact_part=0, min_rows_for_wide_part=10, min_rows_for_compact_part=5"
+$CLICKHOUSE_CLIENT --query="CREATE TABLE data_01600 (part_type String, key Int) ENGINE = MergeTree PARTITION BY part_type ORDER BY key SETTINGS min_bytes_for_wide_part=0, min_rows_for_wide_part=10"
 
 # InMemory
 $CLICKHOUSE_CLIENT --query="INSERT INTO data_01600 SELECT 'InMemory', number FROM system.numbers LIMIT 1"
diff --git a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference
index 613c455fc59..98bb953263a 100644
--- a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference
+++ b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference
@@ -8,8 +8,6 @@ wide fsync_after_insert
 1
 wide fsync_after_insert,fsync_part_directory
 1
-memory in_memory_parts_insert_sync
-1
 wide fsync_part_directory,vertical
 1
 2
diff --git a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql
index ad0dfca0db2..dfc761e1764 100644
--- a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql
+++ b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql
@@ -37,13 +37,6 @@ select * from data_01643;
 optimize table data_01643 final;
 drop table data_01643;
 
-select 'memory in_memory_parts_insert_sync';
-create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_compact_part=2, in_memory_parts_insert_sync=1, fsync_after_insert=1, fsync_part_directory=1;
-insert into data_01643 values (1);
-select * from data_01643;
-optimize table data_01643 final;
-drop table data_01643;
-
 select 'wide fsync_part_directory,vertical';
 create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_part_directory=1, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0;
 insert into data_01643 values (1);
diff --git a/tests/queries/0_stateless/01643_replicated_merge_tree_fsync_smoke.reference b/tests/queries/0_stateless/01643_replicated_merge_tree_fsync_smoke.reference
index 613c455fc59..98bb953263a 100644
--- a/tests/queries/0_stateless/01643_replicated_merge_tree_fsync_smoke.reference
+++ b/tests/queries/0_stateless/01643_replicated_merge_tree_fsync_smoke.reference
@@ -8,8 +8,6 @@ wide fsync_after_insert
 1
 wide fsync_after_insert,fsync_part_directory
 1
-memory in_memory_parts_insert_sync
-1
 wide fsync_part_directory,vertical
 1
 2
diff --git a/tests/queries/0_stateless/01643_replicated_merge_tree_fsync_smoke.sql b/tests/queries/0_stateless/01643_replicated_merge_tree_fsync_smoke.sql
index bcce87e11db..54c30fa2b1a 100644
--- a/tests/queries/0_stateless/01643_replicated_merge_tree_fsync_smoke.sql
+++ b/tests/queries/0_stateless/01643_replicated_merge_tree_fsync_smoke.sql
@@ -63,17 +63,6 @@ system sync replica rep_fsync_r2;
 drop table rep_fsync_r1;
 drop table rep_fsync_r2;
 
-select 'memory in_memory_parts_insert_sync';
-create table rep_fsync_r1 (key Int) engine=ReplicatedMergeTree('/clickhouse/tables/{database}/rep_fsync', 'r1') order by key settings min_rows_for_compact_part=2, in_memory_parts_insert_sync=1, fsync_after_insert=1, fsync_part_directory=1;
-create table rep_fsync_r2 (key Int) engine=ReplicatedMergeTree('/clickhouse/tables/{database}/rep_fsync', 'r2') order by key settings min_rows_for_compact_part=2, in_memory_parts_insert_sync=1, fsync_after_insert=1, fsync_part_directory=1;
-insert into rep_fsync_r1 values (1);
-system sync replica rep_fsync_r2;
-select * from rep_fsync_r2;
-optimize table rep_fsync_r1 final;
-system sync replica rep_fsync_r2;
-drop table rep_fsync_r1;
-drop table rep_fsync_r2;
-
 select 'wide fsync_part_directory,vertical';
 create table rep_fsync_r1 (key Int) engine=ReplicatedMergeTree('/clickhouse/tables/{database}/rep_fsync', 'r1') order by key settings min_bytes_for_wide_part=0, fsync_part_directory=1, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0;
 create table rep_fsync_r2 (key Int) engine=ReplicatedMergeTree('/clickhouse/tables/{database}/rep_fsync', 'r2') order by key settings min_bytes_for_wide_part=0, fsync_part_directory=1, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0;
diff --git a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.reference b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.reference
index c66682ca038..cf3e942adfe 100644
--- a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.reference
+++ b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.reference
@@ -4,7 +4,6 @@ ReplacingMergeTree: OK
 JSONEachRow: OK
 clusterAllReplicas: OK
 SimpleAggregateFunction: OK
-write_ahead_log_interval_ms_to_fsync: OK
 max_concurrent_queries_for_all_users: OK
 test_shard_localhost: OK
 default_path_test: OK
diff --git a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh
index 617148de5a3..42ae5e84f44 100755
--- a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh
+++ b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh
@@ -89,8 +89,6 @@ client_compwords_positive=(
     clusterAllReplicas
     # system.data_type_families
     SimpleAggregateFunction
-    # system.merge_tree_settings
-    write_ahead_log_interval_ms_to_fsync
     # system.settings
     max_concurrent_queries_for_all_users
     # system.clusters
diff --git a/tests/queries/0_stateless/01710_projection_fetch_long.reference b/tests/queries/0_stateless/01710_projection_fetch_long.reference
index abce5410b26..c7834c75d02 100644
--- a/tests/queries/0_stateless/01710_projection_fetch_long.reference
+++ b/tests/queries/0_stateless/01710_projection_fetch_long.reference
@@ -10,8 +10,8 @@
 3	3
 4	4
 0
-CREATE TABLE default.tp_2\n(\n    `x` Int32,\n    `y` Int32,\n    PROJECTION p\n    (\n        SELECT \n            x,\n            y\n        ORDER BY x\n    ),\n    PROJECTION pp\n    (\n        SELECT \n            x,\n            count()\n        GROUP BY x\n    )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{shard}/01710_projection_fetch_default\', \'2_{replica}\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192
+CREATE TABLE default.tp_2\n(\n    `x` Int32,\n    `y` Int32,\n    PROJECTION p\n    (\n        SELECT \n            x,\n            y\n        ORDER BY x\n    ),\n    PROJECTION pp\n    (\n        SELECT \n            x,\n            count()\n        GROUP BY x\n    )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{shard}/01710_projection_fetch_default\', \'2_{replica}\')\nORDER BY y\nSETTINGS min_rows_for_wide_part = 4, min_bytes_for_wide_part = 32, index_granularity = 8192
 2
-CREATE TABLE default.tp_2\n(\n    `x` Int32,\n    `y` Int32,\n    PROJECTION p\n    (\n        SELECT \n            x,\n            y\n        ORDER BY x\n    ),\n    PROJECTION pp\n    (\n        SELECT \n            x,\n            count()\n        GROUP BY x\n    )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{shard}/01710_projection_fetch_default\', \'2_{replica}\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192
-CREATE TABLE default.tp_2\n(\n    `x` Int32,\n    `y` Int32,\n    PROJECTION p\n    (\n        SELECT \n            x,\n            y\n        ORDER BY x\n    ),\n    PROJECTION pp\n    (\n        SELECT \n            x,\n            count()\n        GROUP BY x\n    )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{shard}/01710_projection_fetch_default\', \'2_{replica}\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192
-CREATE TABLE default.tp_2\n(\n    `x` Int32,\n    `y` Int32,\n    PROJECTION p\n    (\n        SELECT \n            x,\n            y\n        ORDER BY x\n    )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{shard}/01710_projection_fetch_default\', \'2_{replica}\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192
+CREATE TABLE default.tp_2\n(\n    `x` Int32,\n    `y` Int32,\n    PROJECTION p\n    (\n        SELECT \n            x,\n            y\n        ORDER BY x\n    ),\n    PROJECTION pp\n    (\n        SELECT \n            x,\n            count()\n        GROUP BY x\n    )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{shard}/01710_projection_fetch_default\', \'2_{replica}\')\nORDER BY y\nSETTINGS min_rows_for_wide_part = 4, min_bytes_for_wide_part = 32, index_granularity = 8192
+CREATE TABLE default.tp_2\n(\n    `x` Int32,\n    `y` Int32,\n    PROJECTION p\n    (\n        SELECT \n            x,\n            y\n        ORDER BY x\n    ),\n    PROJECTION pp\n    (\n        SELECT \n            x,\n            count()\n        GROUP BY x\n    )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{shard}/01710_projection_fetch_default\', \'2_{replica}\')\nORDER BY y\nSETTINGS min_rows_for_wide_part = 4, min_bytes_for_wide_part = 32, index_granularity = 8192
+CREATE TABLE default.tp_2\n(\n    `x` Int32,\n    `y` Int32,\n    PROJECTION p\n    (\n        SELECT \n            x,\n            y\n        ORDER BY x\n    )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{shard}/01710_projection_fetch_default\', \'2_{replica}\')\nORDER BY y\nSETTINGS min_rows_for_wide_part = 4, min_bytes_for_wide_part = 32, index_granularity = 8192
diff --git a/tests/queries/0_stateless/01710_projection_fetch_long.sql b/tests/queries/0_stateless/01710_projection_fetch_long.sql
index 6c41c69254e..13f7a293934 100644
--- a/tests/queries/0_stateless/01710_projection_fetch_long.sql
+++ b/tests/queries/0_stateless/01710_projection_fetch_long.sql
@@ -3,9 +3,9 @@
 drop table if exists tp_1;
 drop table if exists tp_2;
 
-create table tp_1 (x Int32, y Int32, projection p (select x, y order by x)) engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/01710_projection_fetch_' || currentDatabase(), '1_{replica}') order by y settings min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32;
+create table tp_1 (x Int32, y Int32, projection p (select x, y order by x)) engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/01710_projection_fetch_' || currentDatabase(), '1_{replica}') order by y settings min_rows_for_wide_part = 4, min_bytes_for_wide_part = 32;
 
-create table tp_2 (x Int32, y Int32, projection p (select x, y order by x)) engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/01710_projection_fetch_' || currentDatabase(), '2_{replica}') order by y settings min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32;
+create table tp_2 (x Int32, y Int32, projection p (select x, y order by x)) engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/01710_projection_fetch_' || currentDatabase(), '2_{replica}') order by y settings min_rows_for_wide_part = 4, min_bytes_for_wide_part = 32;
 
 insert into tp_1 select number, number from numbers(3);
 
diff --git a/tests/queries/0_stateless/01710_projection_part_check.sql b/tests/queries/0_stateless/01710_projection_part_check.sql
index c889bd323a7..b15d9d7525e 100644
--- a/tests/queries/0_stateless/01710_projection_part_check.sql
+++ b/tests/queries/0_stateless/01710_projection_part_check.sql
@@ -1,6 +1,6 @@
 drop table if exists tp;
 
-create table tp (x Int32, y Int32, projection p (select x, y order by x)) engine = MergeTree order by y settings min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32;
+create table tp (x Int32, y Int32, projection p (select x, y order by x)) engine = MergeTree order by y settings min_rows_for_wide_part = 4, min_bytes_for_wide_part = 32;
 
 insert into tp select number, number from numbers(3);
 insert into tp select number, number from numbers(5);
diff --git a/tests/queries/0_stateless/02148_in_memory_part_flush.reference b/tests/queries/0_stateless/02148_in_memory_part_flush.reference
deleted file mode 100644
index 219c5f4b22f..00000000000
--- a/tests/queries/0_stateless/02148_in_memory_part_flush.reference
+++ /dev/null
@@ -1,4 +0,0 @@
-before DETACH TABLE
-500
-after DETACH TABLE
-500
diff --git a/tests/queries/0_stateless/02148_in_memory_part_flush.sql b/tests/queries/0_stateless/02148_in_memory_part_flush.sql
deleted file mode 100644
index ec20721186e..00000000000
--- a/tests/queries/0_stateless/02148_in_memory_part_flush.sql
+++ /dev/null
@@ -1,26 +0,0 @@
-DROP TABLE IF EXISTS mem_part_flush;
-
-CREATE TABLE mem_part_flush
-(
-`key` UInt32,
-`ts` DateTime,
-`db_time` DateTime DEFAULT now()
-)
-ENGINE = MergeTree
-ORDER BY (key, ts)
-SETTINGS min_rows_for_compact_part = 1000000, min_bytes_for_compact_part = 200000000, in_memory_parts_enable_wal = 0;
-
-INSERT INTO mem_part_flush(key, ts) SELECT number % 1000, now() + intDiv(number,1000) FROM numbers(500);
-
-SELECT 'before DETACH TABLE';
-SELECT count(*) FROM mem_part_flush;
-
-DETACH TABLE mem_part_flush;
-
-ATTACH TABLE mem_part_flush;
-
-SELECT 'after DETACH TABLE';
-SELECT count(*) FROM mem_part_flush;
-
-
-DROP TABLE mem_part_flush;
diff --git a/tests/queries/0_stateless/02410_inmemory_wal_cleanup.reference b/tests/queries/0_stateless/02410_inmemory_wal_cleanup.reference
deleted file mode 100644
index 6727d83a6f4..00000000000
--- a/tests/queries/0_stateless/02410_inmemory_wal_cleanup.reference
+++ /dev/null
@@ -1,35 +0,0 @@
--- { echo }
-
-DROP TABLE IF EXISTS in_memory;
-CREATE TABLE in_memory (a UInt32) ENGINE = MergeTree ORDER BY a SETTINGS min_rows_for_compact_part = 1000, min_bytes_for_wide_part = 10485760;
-INSERT INTO in_memory VALUES (1);
-INSERT INTO in_memory VALUES (2);
-SELECT name, active, part_type FROM system.parts WHERE database = currentDatabase() AND table = 'in_memory';
-all_1_1_0	1	InMemory
-all_2_2_0	1	InMemory
-SELECT * FROM in_memory ORDER BY a;
-1
-2
--- no WAL remove since parts are still in use
-DETACH TABLE in_memory;
-ATTACH TABLE in_memory;
-SELECT name, active, part_type FROM system.parts WHERE database = currentDatabase() AND table = 'in_memory';
-all_1_1_0	1	InMemory
-all_2_2_0	1	InMemory
-SELECT * FROM in_memory ORDER BY a;
-1
-2
--- WAL should be removed, since on disk part covers all parts in WAL
-OPTIMIZE TABLE in_memory;
-DETACH TABLE in_memory;
-ATTACH TABLE in_memory;
-SELECT name, active, part_type FROM system.parts WHERE database = currentDatabase() AND table = 'in_memory';
-all_1_2_1	1	Compact
--- check that the WAL will be reinitialized after remove
-INSERT INTO in_memory VALUES (3);
-DETACH TABLE in_memory;
-ATTACH TABLE in_memory;
-SELECT * FROM in_memory ORDER BY a;
-1
-2
-3
diff --git a/tests/queries/0_stateless/02410_inmemory_wal_cleanup.sql b/tests/queries/0_stateless/02410_inmemory_wal_cleanup.sql
deleted file mode 100644
index 7f832d980ba..00000000000
--- a/tests/queries/0_stateless/02410_inmemory_wal_cleanup.sql
+++ /dev/null
@@ -1,29 +0,0 @@
--- Tags: no-s3-storage
-
--- { echo }
-
-DROP TABLE IF EXISTS in_memory;
-
-CREATE TABLE in_memory (a UInt32) ENGINE = MergeTree ORDER BY a SETTINGS min_rows_for_compact_part = 1000, min_bytes_for_wide_part = 10485760;
-INSERT INTO in_memory VALUES (1);
-INSERT INTO in_memory VALUES (2);
-SELECT name, active, part_type FROM system.parts WHERE database = currentDatabase() AND table = 'in_memory';
-SELECT * FROM in_memory ORDER BY a;
-
--- no WAL remove since parts are still in use
-DETACH TABLE in_memory;
-ATTACH TABLE in_memory;
-SELECT name, active, part_type FROM system.parts WHERE database = currentDatabase() AND table = 'in_memory';
-SELECT * FROM in_memory ORDER BY a;
-
--- WAL should be removed, since on disk part covers all parts in WAL
-OPTIMIZE TABLE in_memory;
-DETACH TABLE in_memory;
-ATTACH TABLE in_memory;
-SELECT name, active, part_type FROM system.parts WHERE database = currentDatabase() AND table = 'in_memory';
-
--- check that the WAL will be reinitialized after remove
-INSERT INTO in_memory VALUES (3);
-DETACH TABLE in_memory;
-ATTACH TABLE in_memory;
-SELECT * FROM in_memory ORDER BY a;
diff --git a/tests/queries/0_stateless/02423_drop_memory_parts.reference b/tests/queries/0_stateless/02423_drop_memory_parts.reference
deleted file mode 100644
index d69a5f07a05..00000000000
--- a/tests/queries/0_stateless/02423_drop_memory_parts.reference
+++ /dev/null
@@ -1,14 +0,0 @@
-init state
-30
-0_1_1_0	InMemory	10	1
-1_2_2_0	InMemory	10	1
-2_3_3_0	InMemory	10	1
-drop part 0
-20
-1_2_2_0	InMemory	10	1
-2_3_3_0	InMemory	10	1
-detach table
-attach table
-20
-1_2_2_0	InMemory	10	1
-2_3_3_0	InMemory	10	1
diff --git a/tests/queries/0_stateless/02423_drop_memory_parts.sql b/tests/queries/0_stateless/02423_drop_memory_parts.sql
deleted file mode 100644
index 9326f159b0c..00000000000
--- a/tests/queries/0_stateless/02423_drop_memory_parts.sql
+++ /dev/null
@@ -1,40 +0,0 @@
--- Tags: no-s3-storage
-
-DROP TABLE IF EXISTS table_in_memory;
-
-CREATE TABLE table_in_memory
-(
-    `id` UInt64,
-    `value` UInt64
-)
-ENGINE = MergeTree
-PARTITION BY id
-ORDER BY value
-SETTINGS min_bytes_for_wide_part=1000, min_bytes_for_compact_part=900;
-
-SELECT 'init state';
-INSERT INTO table_in_memory SELECT intDiv(number, 10), number FROM numbers(30);
-
-SELECT count() FROM table_in_memory;
-SELECT name, part_type, rows, active from system.parts
-WHERE table='table_in_memory' AND database=currentDatabase();
-
-SELECT 'drop part 0';
-ALTER TABLE table_in_memory DROP PARTITION 0;
-
-SELECT count() FROM table_in_memory;
-SELECT name, part_type, rows, active from system.parts
-WHERE table='table_in_memory' AND database=currentDatabase() AND active;
-
-SELECT 'detach table';
-DETACH TABLE table_in_memory;
-
-SELECT name, part_type, rows, active from system.parts
-WHERE table='table_in_memory' AND database=currentDatabase();
-
-SELECT 'attach table';
-ATTACH TABLE table_in_memory;
-
-SELECT count() FROM table_in_memory;
-SELECT name, part_type, rows, active from system.parts
-WHERE table='table_in_memory' AND database=currentDatabase() and active;
diff --git a/tests/queries/0_stateless/02559_nested_multiple_levels_default.sql b/tests/queries/0_stateless/02559_nested_multiple_levels_default.sql
index 156af5c7784..9dcdab82acb 100644
--- a/tests/queries/0_stateless/02559_nested_multiple_levels_default.sql
+++ b/tests/queries/0_stateless/02559_nested_multiple_levels_default.sql
@@ -10,26 +10,12 @@ CREATE TABLE data_compact
 )
 ENGINE = MergeTree()
 ORDER BY tuple()
-SETTINGS min_rows_for_compact_part=0, min_bytes_for_compact_part=0, min_rows_for_wide_part=100, min_bytes_for_wide_part=1e9;
+SETTINGS min_rows_for_wide_part=100, min_bytes_for_wide_part=1e9;
 INSERT INTO data_compact VALUES ([0]);
 ALTER TABLE data_compact ADD COLUMN root.nested_array Array(Array(UInt8));
 SELECT table, part_type FROM system.parts WHERE table = 'data_compact' AND database = currentDatabase();
 SELECT root.nested_array FROM data_compact;
 
--- memory
-DROP TABLE IF EXISTS data_memory;
-CREATE TABLE data_memory
-(
-    `root.array` Array(UInt8),
-)
-ENGINE = MergeTree()
-ORDER BY tuple()
-SETTINGS min_rows_for_compact_part=100, min_bytes_for_compact_part=1e9, min_rows_for_wide_part=100, min_bytes_for_wide_part=1e9, in_memory_parts_enable_wal=0;
-INSERT INTO data_memory VALUES ([0]);
-ALTER TABLE data_memory ADD COLUMN root.nested_array Array(Array(UInt8));
-SELECT table, part_type FROM system.parts WHERE table = 'data_memory' AND database = currentDatabase();
-SELECT root.nested_array FROM data_memory;
-
 -- wide
 DROP TABLE IF EXISTS data_wide;
 CREATE TABLE data_wide
@@ -38,7 +24,7 @@ CREATE TABLE data_wide
 )
 ENGINE = MergeTree()
 ORDER BY tuple()
-SETTINGS min_rows_for_wide_part=0, min_bytes_for_wide_part=0, min_rows_for_wide_part=0, min_bytes_for_wide_part=0;
+SETTINGS min_rows_for_wide_part=0, min_bytes_for_wide_part=0;
 INSERT INTO data_wide VALUES ([0]);
 ALTER TABLE data_wide ADD COLUMN root.nested_array Array(Array(UInt8));
 SELECT table, part_type FROM system.parts WHERE table = 'data_wide' AND database = currentDatabase();
diff --git a/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh b/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh
index 4713e4cbe8b..adc9525ef81 100755
--- a/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh
+++ b/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh
@@ -32,11 +32,8 @@ DROP TABLE IF EXISTS times;
 CREATE TABLE times (t DateTime) ENGINE MergeTree ORDER BY t
   SETTINGS
     storage_policy='default',
-    min_rows_for_compact_part = 0,
-    min_bytes_for_compact_part = 0,
     min_rows_for_wide_part = 1000000,
     min_bytes_for_wide_part = 1000000,
-    in_memory_parts_enable_wal = 0,
     ratio_of_defaults_for_sparse_serialization=1.0;
 "
 
@@ -74,4 +71,3 @@ AND ( query LIKE '%SELECT % FROM times%' OR query LIKE '%INSERT INTO times%' )
 AND type = 'QueryFinish'
 ORDER BY query_start_time_microseconds ASC, query DESC;
 "
-
diff --git a/tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.reference b/tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.reference
deleted file mode 100644
index 0cfbf08886f..00000000000
--- a/tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.reference
+++ /dev/null
@@ -1 +0,0 @@
-2
diff --git a/tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.sql b/tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.sql
deleted file mode 100644
index 194ea9bfcc1..00000000000
--- a/tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.sql
+++ /dev/null
@@ -1,15 +0,0 @@
--- Tags: no-s3-storage
-
-DROP TABLE IF EXISTS inmemory_test;
-
-CREATE TABLE inmemory_test (d Date, id String)
-ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/inmemory_test', 'r1')
-PARTITION BY toYYYYMMDD(d) ORDER BY (d, id)
-SETTINGS min_rows_for_compact_part = 10, index_granularity = 8192;
-
-INSERT INTO inmemory_test(d, id) VALUES('2023-01-01', 'abcdefghijklmnopqrstuvwxyz');
-INSERT INTO inmemory_test(d, id) VALUES('2023-01-01', 'a1234567890123456789012345');
-
-SELECT COUNT(1) FROM inmemory_test;
-
-DROP TABLE inmemory_test;
diff --git a/tests/queries/0_stateless/02703_max_local_read_bandwidth.sh b/tests/queries/0_stateless/02703_max_local_read_bandwidth.sh
index bdcdb38846c..130f3a29ade 100755
--- a/tests/queries/0_stateless/02703_max_local_read_bandwidth.sh
+++ b/tests/queries/0_stateless/02703_max_local_read_bandwidth.sh
@@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 $CLICKHOUSE_CLIENT -nm -q "
     drop table if exists data;
-    create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9, min_bytes_for_compact_part=0;
+    create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9;
 "
 
 # reading 1e6*8 bytes with 1M bandwith it should take (8-1)/1=7 seconds
diff --git a/tests/queries/0_stateless/02703_max_local_write_bandwidth.sh b/tests/queries/0_stateless/02703_max_local_write_bandwidth.sh
index 276a15ca6cc..80713e90169 100755
--- a/tests/queries/0_stateless/02703_max_local_write_bandwidth.sh
+++ b/tests/queries/0_stateless/02703_max_local_write_bandwidth.sh
@@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 $CLICKHOUSE_CLIENT -nm -q "
     drop table if exists data;
-    create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9, min_bytes_for_compact_part=0;
+    create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9;
 "
 
 query_id=$(random_str 10)
diff --git a/tests/queries/0_stateless/02704_max_backup_bandwidth.sh b/tests/queries/0_stateless/02704_max_backup_bandwidth.sh
index c9ad23031b6..b5d32d2059d 100755
--- a/tests/queries/0_stateless/02704_max_backup_bandwidth.sh
+++ b/tests/queries/0_stateless/02704_max_backup_bandwidth.sh
@@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 $CLICKHOUSE_CLIENT -nm -q "
     drop table if exists data;
-    create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9, min_bytes_for_compact_part=0;
+    create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9;
 "
 
 # reading 1e6*8 bytes with 1M bandwith it should take (8-1)/1=7 seconds
diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt
index bb9d4c88fa1..e1d5939eb91 100644
--- a/utils/CMakeLists.txt
+++ b/utils/CMakeLists.txt
@@ -23,7 +23,6 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
     add_subdirectory (wikistat-loader)
     add_subdirectory (check-marks)
     add_subdirectory (checksum-for-compressed-block)
-    add_subdirectory (wal-dump)
     add_subdirectory (check-mysql-binlog)
     add_subdirectory (keeper-bench)
 
diff --git a/utils/wal-dump/CMakeLists.txt b/utils/wal-dump/CMakeLists.txt
deleted file mode 100644
index 3d59e95b4ca..00000000000
--- a/utils/wal-dump/CMakeLists.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-clickhouse_add_executable (wal-dump main.cpp)
-target_link_libraries(wal-dump PRIVATE dbms boost::program_options)
diff --git a/utils/wal-dump/main.cpp b/utils/wal-dump/main.cpp
deleted file mode 100644
index 3566936324b..00000000000
--- a/utils/wal-dump/main.cpp
+++ /dev/null
@@ -1,78 +0,0 @@
-#include <iostream>
-
-#include <boost/program_options.hpp>
-
-#include <Compression/CompressedReadBuffer.h>
-#include <Compression/CompressedReadBufferFromFile.h>
-#include <Compression/CompressedWriteBuffer.h>
-#include <Formats/NativeReader.h>
-#include <IO/Operators.h>
-#include <IO/ReadBufferFromFile.h>
-#include <IO/ReadHelpers.h>
-#include <IO/WriteBufferFromFileDescriptor.h>
-#include <Storages/MergeTree/MergeTreeWriteAheadLog.h>
-
-/*
- * Dump the Write Ahead Log file, outputs:
- * Part 0, Version: 0, Action : ADD_PART, Name: 4_1_1_0, Block:
-   a Int32 Int32(size = 2), b Int32 Int32(size = 2), c Int32 Int32(size = 2)
- */
-
-static void dump(const std::string & bin_path)
-{
-    DB::ReadBufferFromFile in(bin_path);
-    DB::NativeReader block_in(in, 0);
-    DB::Block block;
-
-    DB::WriteBufferFromFileDescriptor out(STDOUT_FILENO);
-
-    for (size_t part_num = 0; !in.eof(); ++part_num)
-    {
-        UInt8 version;
-        String part_name;
-        DB::MergeTreeWriteAheadLog::ActionType action_type;
-
-        DB::readIntBinary(version, in);
-        DB::readIntBinary(action_type, in);
-        DB::readStringBinary(part_name, in);
-        block = block_in.read();
-
-        out << "Part " << part_num << ", Version: " << version
-            << ", Action : " << (action_type == DB::MergeTreeWriteAheadLog::ActionType::ADD_PART ? "ADD_PART" : "DROP_PART")
-            << ", Name: " << part_name << ", Block:\n";
-        out << block.dumpStructure() << "\n";
-        out << "\n" << DB::flush;
-    }
-}
-
-
-int main(int argc, char ** argv)
-{
-    boost::program_options::options_description desc("Allowed options");
-    desc.add_options()("help,h", "produce help message");
-
-    boost::program_options::variables_map options;
-    boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options);
-
-    if (options.count("help") || argc != 2)
-    {
-        std::cout << "Usage: " << argv[0] << " wal.bin" << std::endl;
-        std::cout << desc << std::endl;
-        return 1;
-    }
-
-    try
-    {
-        dump(argv[1]);
-    }
-    catch (const DB::Exception & e)
-    {
-        std::cerr << e.what() << ", " << e.message() << std::endl
-                  << std::endl
-                  << "Stack trace:" << std::endl
-                  << e.getStackTraceString() << std::endl;
-        throw;
-    }
-
-    return 0;
-}

From 93e8ecf4ffe6b5766792c2316fc024b4f9c05f84 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Wed, 3 May 2023 01:33:26 +0200
Subject: [PATCH 117/689] Update test

---
 tests/integration/test_merges_memory_limit/test.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_merges_memory_limit/test.py b/tests/integration/test_merges_memory_limit/test.py
index 22bf143f4f7..e663f3280cc 100644
--- a/tests/integration/test_merges_memory_limit/test.py
+++ b/tests/integration/test_merges_memory_limit/test.py
@@ -22,11 +22,15 @@ def test_memory_limit_success():
     )
     node.query("SYSTEM STOP MERGES test_merge_oom")
     node.query(
-        "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(10000)"
+        "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(3000)"
     )
     node.query(
-        "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(10000)"
+        "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(3000)"
     )
+    node.query(
+        "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(3000)"
+    )
+
     _, error = node.query_and_get_answer_with_error(
         "SYSTEM START MERGES test_merge_oom;SET optimize_throw_if_noop=1;OPTIMIZE TABLE test_merge_oom FINAL"
     )

From 551043418ca1463712f3da221ea4528b65af403d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 3 May 2023 02:34:56 +0200
Subject: [PATCH 118/689] Update tests

---
 .../0_stateless/01475_read_subcolumns_storages.reference       | 3 ---
 .../0_stateless/02559_nested_multiple_levels_default.reference | 2 --
 2 files changed, 5 deletions(-)

diff --git a/tests/queries/0_stateless/01475_read_subcolumns_storages.reference b/tests/queries/0_stateless/01475_read_subcolumns_storages.reference
index 4e37b751d5a..ce0b13eabd2 100644
--- a/tests/queries/0_stateless/01475_read_subcolumns_storages.reference
+++ b/tests/queries/0_stateless/01475_read_subcolumns_storages.reference
@@ -7,9 +7,6 @@ TinyLog
 Memory
 100	[1,2,3]	[[[1,2],[],[4]],[[5,6],[7,8]],[[]]]	[1,NULL,2]	('foo',200)	{'foo':1,'bar':42}
 100	0	[1,2,3]	3	[[[1,2],[],[4]],[[5,6],[7,8]],[[]]]	3	[3,2,1]	[[2,0,1],[2,2],[0]]	[1,NULL,2]	3	[0,1,0]	('foo',200)	foo	200	{'foo':1,'bar':42}	['foo','bar']	[1,42]
-MergeTree ORDER BY tuple() SETTINGS min_bytes_for_compact_part='10M'
-100	[1,2,3]	[[[1,2],[],[4]],[[5,6],[7,8]],[[]]]	[1,NULL,2]	('foo',200)	{'foo':1,'bar':42}
-100	0	[1,2,3]	3	[[[1,2],[],[4]],[[5,6],[7,8]],[[]]]	3	[3,2,1]	[[2,0,1],[2,2],[0]]	[1,NULL,2]	3	[0,1,0]	('foo',200)	foo	200	{'foo':1,'bar':42}	['foo','bar']	[1,42]
 MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part='10M'
 100	[1,2,3]	[[[1,2],[],[4]],[[5,6],[7,8]],[[]]]	[1,NULL,2]	('foo',200)	{'foo':1,'bar':42}
 100	0	[1,2,3]	3	[[[1,2],[],[4]],[[5,6],[7,8]],[[]]]	3	[3,2,1]	[[2,0,1],[2,2],[0]]	[1,NULL,2]	3	[0,1,0]	('foo',200)	foo	200	{'foo':1,'bar':42}	['foo','bar']	[1,42]
diff --git a/tests/queries/0_stateless/02559_nested_multiple_levels_default.reference b/tests/queries/0_stateless/02559_nested_multiple_levels_default.reference
index b0214e0e7c7..9ed0fb620a4 100644
--- a/tests/queries/0_stateless/02559_nested_multiple_levels_default.reference
+++ b/tests/queries/0_stateless/02559_nested_multiple_levels_default.reference
@@ -1,6 +1,4 @@
 data_compact	Compact
 [[]]
-data_memory	InMemory
-[[]]
 data_wide	Wide
 [[]]

From ae62b227f074410ae5b2f5db9dac41a60b0565de Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 3 May 2023 08:30:22 +0000
Subject: [PATCH 119/689] More fixes

---
 src/Analyzer/Passes/CNF.cpp                   | 14 +++----
 .../0_stateless/01626_cnf_fuzz_long.python    |  2 +-
 .../02725_cnf_large_check.reference           |  4 ++
 .../0_stateless/02725_cnf_large_check.sql     | 40 ++++++++++++++++++-
 4 files changed, 49 insertions(+), 11 deletions(-)

diff --git a/src/Analyzer/Passes/CNF.cpp b/src/Analyzer/Passes/CNF.cpp
index 4448cda8ddd..91e973c7573 100644
--- a/src/Analyzer/Passes/CNF.cpp
+++ b/src/Analyzer/Passes/CNF.cpp
@@ -162,14 +162,13 @@ private:
 class PushOrVisitor
 {
 public:
-    PushOrVisitor(ContextPtr context, size_t max_atoms_, size_t num_atoms_)
+    PushOrVisitor(ContextPtr context, size_t max_atoms_)
         : max_atoms(max_atoms_)
-        , num_atoms(num_atoms_)
         , and_resolver(FunctionFactory::instance().get("and", context))
         , or_resolver(FunctionFactory::instance().get("or", context))
     {}
 
-    bool visit(QueryTreeNodePtr & node)
+    bool visit(QueryTreeNodePtr & node, size_t num_atoms)
     {
         if (max_atoms && num_atoms > max_atoms)
             return false;
@@ -188,7 +187,7 @@ public:
             auto & arguments = function_node->getArguments().getNodes();
             for (auto & argument : arguments)
             {
-                if (!visit(argument))
+                if (!visit(argument, num_atoms))
                     return false;
             }
         }
@@ -220,7 +219,7 @@ public:
             auto rhs = createFunctionNode(or_resolver, std::move(other_node), std::move(and_function_arguments[1]));
             node = createFunctionNode(and_resolver, std::move(lhs), std::move(rhs));
 
-            visit(node);
+            return visit(node, num_atoms);
         }
 
         return true;
@@ -228,7 +227,6 @@ public:
 
 private:
     size_t max_atoms;
-    size_t num_atoms;
 
     const FunctionOverloadResolverPtr and_resolver;
     const FunctionOverloadResolverPtr or_resolver;
@@ -519,8 +517,8 @@ std::optional<CNF> CNF::tryBuildCNF(const QueryTreeNodePtr & node, ContextPtr co
         visitor.visit(node_cloned, false);
     }
 
-    if (PushOrVisitor visitor(context, max_atoms, atom_count);
-        !visitor.visit(node_cloned))
+    if (PushOrVisitor visitor(context, max_atoms);
+        !visitor.visit(node_cloned, atom_count))
             return std::nullopt;
 
     CollectGroupsVisitor collect_visitor;
diff --git a/tests/queries/0_stateless/01626_cnf_fuzz_long.python b/tests/queries/0_stateless/01626_cnf_fuzz_long.python
index de9e4a21dbb..87cd748590f 100644
--- a/tests/queries/0_stateless/01626_cnf_fuzz_long.python
+++ b/tests/queries/0_stateless/01626_cnf_fuzz_long.python
@@ -56,7 +56,7 @@ def generate_dnf():
 
 
 select_query = (
-    "SELECT count() FROM t_cnf_fuzz WHERE {} SETTINGS convert_query_to_cnf = {}"
+    "SELECT count() FROM t_cnf_fuzz WHERE {} SETTINGS convert_query_to_cnf = {}, allow_experimental_analyzer=1"
 )
 
 fail_report = """
diff --git a/tests/queries/0_stateless/02725_cnf_large_check.reference b/tests/queries/0_stateless/02725_cnf_large_check.reference
index 226aaf8af79..edea3168553 100644
--- a/tests/queries/0_stateless/02725_cnf_large_check.reference
+++ b/tests/queries/0_stateless/02725_cnf_large_check.reference
@@ -1,2 +1,6 @@
 8
+105
 8
+2
+1120
+2
diff --git a/tests/queries/0_stateless/02725_cnf_large_check.sql b/tests/queries/0_stateless/02725_cnf_large_check.sql
index 881d562ab6d..bc237dffe75 100644
--- a/tests/queries/0_stateless/02725_cnf_large_check.sql
+++ b/tests/queries/0_stateless/02725_cnf_large_check.sql
@@ -3,7 +3,43 @@ DROP TABLE IF EXISTS 02725_cnf;
 CREATE TABLE 02725_cnf (c0 UInt8, c1 UInt8, c2 UInt8, c3 UInt8, c4 UInt8, c5 UInt8, c6 UInt8, c7 UInt8, c8 UInt8, c9 UInt8) ENGINE = Memory;
 
 INSERT INTO 02725_cnf VALUES (0, 0, 0, 0, 0, 0, 0, 0, 0, 0), (0, 0, 0, 0, 0, 0, 0, 0, 0, 1), (0, 0, 0, 0, 0, 0, 0, 0, 1, 0), (0, 0, 0, 0, 0, 0, 0, 0, 1, 1), (0, 0, 0, 0, 0, 0, 0, 1, 0, 0), (0, 0, 0, 0, 0, 0, 0, 1, 0, 1), (0, 0, 0, 0, 0, 0, 0, 1, 1, 0), (0, 0, 0, 0, 0, 0, 0, 1, 1, 1);
-SELECT count() FROM 02725_cnf WHERE (c5 AND (NOT c0)) OR ((NOT c3) AND (NOT c6) AND (NOT c1) AND (NOT c6)) OR (c7 AND (NOT c3) AND (NOT c5) AND (NOT c7)) OR ((NOT c8) AND c5) OR ((NOT c0)) OR ((NOT c8) AND (NOT c5) AND c1 AND c6 AND c3) OR (c7 AND (NOT c0) AND c6 AND c1 AND (NOT c2)) OR (c3 AND (NOT c9) AND c1) SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer=1;
-SELECT count() FROM 02725_cnf WHERE (c5 AND (NOT c0)) OR ((NOT c3) AND (NOT c6) AND (NOT c1) AND (NOT c6)) OR (c7 AND (NOT c3) AND (NOT c5) AND (NOT c7)) OR ((NOT c8) AND c5) OR ((NOT c0)) OR ((NOT c8) AND (NOT c5) AND c1 AND c6 AND c3) OR (c7 AND (NOT c0) AND c6 AND c1 AND (NOT c2)) OR (c3 AND (NOT c9) AND c1) SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer=0;
+
+SELECT count()
+FROM 02725_cnf
+WHERE (c5 AND (NOT c0)) OR ((NOT c3) AND (NOT c6) AND (NOT c1) AND (NOT c6)) OR (c7 AND (NOT c3) AND (NOT c5) AND (NOT c7)) OR ((NOT c8) AND c5) OR ((NOT c0)) OR ((NOT c8) AND (NOT c5) AND c1 AND c6 AND c3) OR (c7 AND (NOT c0) AND c6 AND c1 AND (NOT c2)) OR (c3 AND (NOT c9) AND c1)
+SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 1;
+
+-- make sure we use CNF form
+SELECT count()
+FROM (
+    EXPLAIN QUERY TREE
+    SELECT count()
+    FROM 02725_cnf
+    WHERE (c5 AND (NOT c0)) OR ((NOT c3) AND (NOT c6) AND (NOT c1) AND (NOT c6)) OR (c7 AND (NOT c3) AND (NOT c5) AND (NOT c7)) OR ((NOT c8) AND c5) OR ((NOT c0)) OR ((NOT c8) AND (NOT c5) AND c1 AND c6 AND c3) OR (c7 AND (NOT c0) AND c6 AND c1 AND (NOT c2)) OR (c3 AND (NOT c9) AND c1)
+    SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 1);
+
+SELECT count()
+FROM 02725_cnf
+WHERE (c5 AND (NOT c0)) OR ((NOT c3) AND (NOT c6) AND (NOT c1) AND (NOT c6)) OR (c7 AND (NOT c3) AND (NOT c5) AND (NOT c7)) OR ((NOT c8) AND c5) OR ((NOT c0)) OR ((NOT c8) AND (NOT c5) AND c1 AND c6 AND c3) OR (c7 AND (NOT c0) AND c6 AND c1 AND (NOT c2)) OR (c3 AND (NOT c9) AND c1)
+SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 0;
+
+SELECT count()
+FROM 02725_cnf
+WHERE ((NOT c2) AND c2 AND (NOT c1)) OR ((NOT c2) AND c3 AND (NOT c5)) OR ((NOT c7) AND (NOT c8)) OR (c9 AND c6 AND c8 AND (NOT c8) AND (NOT c7))
+SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 1;
+
+-- make sure we use CNF form
+SELECT count()
+FROM (
+    EXPLAIN QUERY TREE
+    SELECT count()
+    FROM 02725_cnf
+    WHERE ((NOT c2) AND c2 AND (NOT c1)) OR ((NOT c2) AND c3 AND (NOT c5)) OR ((NOT c7) AND (NOT c8)) OR (c9 AND c6 AND c8 AND (NOT c8) AND (NOT c7))
+    SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 1);
+
+SELECT count()
+FROM 02725_cnf
+WHERE ((NOT c2) AND c2 AND (NOT c1)) OR ((NOT c2) AND c3 AND (NOT c5)) OR ((NOT c7) AND (NOT c8)) OR (c9 AND c6 AND c8 AND (NOT c8) AND (NOT c7))
+SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 0;
 
 DROP TABLE 02725_cnf;

From 3ba490e4d63a2a3301e8923d897393c8b20eaef5 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Wed, 3 May 2023 03:05:38 +0200
Subject: [PATCH 120/689] test when merge canceled with s3

---
 tests/integration/helpers/wait_for_helpers.py |   9 ++
 .../configs/config.d/storage_conf.xml         |  25 ++++
 .../test_merge_tree_s3/s3_mocks/broken_s3.py  | 111 ++++++++++++++
 tests/integration/test_merge_tree_s3/test.py  | 136 ++++++++++++++++++
 4 files changed, 281 insertions(+)
 create mode 100644 tests/integration/test_merge_tree_s3/s3_mocks/broken_s3.py

diff --git a/tests/integration/helpers/wait_for_helpers.py b/tests/integration/helpers/wait_for_helpers.py
index b041de99282..d240c71bc86 100644
--- a/tests/integration/helpers/wait_for_helpers.py
+++ b/tests/integration/helpers/wait_for_helpers.py
@@ -28,3 +28,12 @@ def wait_for_delete_empty_parts(node, table, database=None, **kwargs):
         f"WHERE active AND rows = 0 AND table = '{table}' AND database = '{database}'"
     )
     assert_eq_with_retry(node, empty_parts_query, "0\n", **kwargs)
+
+
+def wait_for_merges(node, table, database=None, **kwargs):
+    table, database = _parse_table_database(table, database)
+    merges_count_query = (
+        f"SELECT count() > 0 FROM system.merges "
+        f"WHERE table = '{table}' AND database = '{database}'"
+    )
+    assert_eq_with_retry(node, merges_count_query, "1\n", **kwargs)
diff --git a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml
index c2060f32fe5..68b9b6bf1bf 100644
--- a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml
+++ b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml
@@ -22,6 +22,14 @@
                 <secret_access_key>minio123</secret_access_key>
                 <s3_max_single_read_retries>10</s3_max_single_read_retries>
             </no_delete_objects_s3>
+            <broken_s3>
+                <type>s3</type>
+                <endpoint>http://resolver:8083/root/data/</endpoint>
+                <access_key_id>minio</access_key_id>
+                <secret_access_key>minio123</secret_access_key>
+                <s3_retry_attempts>1</s3_retry_attempts>
+                <connect_timeout_ms>20000</connect_timeout_ms>
+            </broken_s3>
             <hdd>
                 <type>local</type>
                 <path>/</path>
@@ -100,6 +108,23 @@
                     </main>
                 </volumes>
             </s3_cache_r>
+            <external_broken_s3>
+                 <volumes>
+                    <main>
+                        <disk>hdd</disk>
+                    </main>
+                    <external>
+                        <disk>broken_s3</disk>
+                    </external>
+                </volumes>
+            </external_broken_s3>
+            <broken_s3>
+                 <volumes>
+                    <main>
+                        <disk>broken_s3</disk>
+                    </main>
+                </volumes>
+            </broken_s3>
         </policies>
     </storage_configuration>
 
diff --git a/tests/integration/test_merge_tree_s3/s3_mocks/broken_s3.py b/tests/integration/test_merge_tree_s3/s3_mocks/broken_s3.py
new file mode 100644
index 00000000000..1c2465f6d44
--- /dev/null
+++ b/tests/integration/test_merge_tree_s3/s3_mocks/broken_s3.py
@@ -0,0 +1,111 @@
+import sys
+import urllib.parse
+import http.server
+import socketserver
+
+
+UPSTREAM_HOST = "minio1"
+UPSTREAM_PORT = 9001
+
+
+class ServerRuntime:
+    def __init__(self):
+        self.error_at_put_when_length_bigger = None
+
+    def reset(self):
+        self.error_at_put_when_length_bigger = None
+
+
+runtime = ServerRuntime()
+
+
+class RequestHandler(http.server.BaseHTTPRequestHandler):
+    def _ok(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain")
+        self.end_headers()
+        self.wfile.write(b"OK")
+
+    def _ping(self):
+        self._ok()
+
+    def _redirect(self):
+        content_length = int(self.headers.get("Content-Length", 0))
+        to_read = content_length
+        while to_read > 0:
+            # read content in order to avoid error on client
+            # Poco::Exception. Code: 1000, e.code() = 32, I/O error: Broken pipe
+            # do it piece by piece in order to avoid big allocation
+            size = min(to_read, 1024)
+            str(self.rfile.read(size))
+            to_read -= size
+
+        self.send_response(307)
+        url = f"http://{UPSTREAM_HOST}:{UPSTREAM_PORT}{self.path}"
+        self.send_header("Location", url)
+        self.end_headers()
+        self.wfile.write(b"Redirected")
+
+    def _error(self, data):
+        self.send_response(500)
+        self.send_header("Content-Type", "text/xml")
+        self.end_headers()
+        self.wfile.write(data)
+
+    def _broken_settings(self):
+        parts = urllib.parse.urlsplit(self.path)
+        path = [x for x in parts.path.split("/") if x]
+        assert path[0] == "mock_settings", path
+        if path[1] == "error_at_put":
+            params = urllib.parse.parse_qs(parts.query, keep_blank_values=False)
+            runtime.error_at_put_when_length_bigger = int(
+                params.get("when_length_bigger", [1024*1024])[0]
+            )
+            self._ok()
+        elif path[1] == "reset":
+            runtime.reset()
+            self._ok()
+        else:
+            self._error("mock_settings, wrong command")
+
+    def do_GET(self):
+        if self.path == "/":
+            self._ping()
+        elif self.path.startswith("/mock_settings"):
+            self._broken_settings()
+        else:
+            self._redirect()
+
+    def do_PUT(self):
+        if runtime.error_at_put_when_length_bigger is not None:
+            content_length = int(self.headers.get("Content-Length", 0))
+            if content_length > runtime.error_at_put_when_length_bigger:
+                self._error(
+                    b'<?xml version="1.0" encoding="UTF-8"?>'
+                    b"<Error>"
+                    b"<Code>ExpectedError</Code>"
+                    b"<Message>mock s3 injected error</Message>"
+                    b"<RequestId>txfbd566d03042474888193-00608d7537</RequestId>"
+                    b"</Error>"
+                )
+            else:
+                self._redirect()
+        else:
+            self._redirect()
+
+    def do_POST(self):
+        self._redirect()
+
+    def do_HEAD(self):
+        self._redirect()
+
+    def do_DELETE(self):
+        self._redirect()
+
+
+class ThreadedHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
+    """Handle requests in a separate thread."""
+
+
+httpd = ThreadedHTTPServer(("0.0.0.0", int(sys.argv[1])), RequestHandler)
+httpd.serve_forever()
diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py
index 9e9903c36c7..0244eeecba6 100644
--- a/tests/integration/test_merge_tree_s3/test.py
+++ b/tests/integration/test_merge_tree_s3/test.py
@@ -8,6 +8,8 @@ from helpers.mock_servers import start_mock_servers
 from helpers.utility import generate_values, replace_config, SafeThread
 from helpers.wait_for_helpers import wait_for_delete_inactive_parts
 from helpers.wait_for_helpers import wait_for_delete_empty_parts
+from helpers.wait_for_helpers import wait_for_merges
+from helpers.test_tools import assert_eq_with_retry
 
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -97,6 +99,7 @@ def run_s3_mocks(cluster):
         [
             ("unstable_proxy.py", "resolver", "8081"),
             ("no_delete_objects.py", "resolver", "8082"),
+            ("broken_s3.py", "resolver", "8083"),
         ],
     )
 
@@ -136,6 +139,18 @@ def clear_minio(cluster):
     yield
 
 
+@pytest.fixture(autouse=True, scope="function")
+def reset_mock_broken_s3(cluster):
+    response = cluster.exec_in_container(
+        cluster.get_container_id("resolver"),
+        ["curl", "-s", f"http://localhost:8083/mock_settings/reset"],
+        nothrow=True,
+    )
+    assert response == "OK"
+
+    yield
+
+
 def check_no_objects_after_drop(cluster, table_name="s3_test", node_name="node"):
     node = cluster.instances[node_name]
     node.query(f"DROP TABLE IF EXISTS {table_name} NO DELAY")
@@ -825,3 +840,124 @@ def test_cache_setting_compatibility(cluster, node_name):
     assert not node.contains_in_log("No such file or directory: Cache info:")
 
     check_no_objects_after_drop(cluster)
+
+
+@pytest.mark.parametrize("node_name", ["node"])
+def test_merge_canceled_by_drop(cluster, node_name):
+    node = cluster.instances[node_name]
+    node.query("DROP TABLE IF EXISTS test_merge_canceled_by_drop NO DELAY")
+    node.query(
+        "CREATE TABLE test_merge_canceled_by_drop "
+        " (key UInt32, value String)"
+        " Engine=MergeTree() "
+        " ORDER BY value "
+        " SETTINGS storage_policy='s3'"
+    )
+    node.query("SYSTEM STOP MERGES test_merge_canceled_by_drop")
+    node.query(
+        "INSERT INTO test_merge_canceled_by_drop SELECT number, toString(number) FROM numbers(100000000)"
+    )
+    node.query("SYSTEM START MERGES test_merge_canceled_by_drop")
+
+    wait_for_merges(node, "test_merge_canceled_by_drop")
+    check_no_objects_after_drop(
+        cluster, table_name="test_merge_canceled_by_drop", node_name=node_name
+    )
+
+
+@pytest.mark.parametrize("node_name", ["node"])
+def test_merge_canceled_by_s3_errors(cluster, node_name):
+    node = cluster.instances[node_name]
+    node.query("DROP TABLE IF EXISTS test_merge_canceled_by_s3_errors NO DELAY")
+    node.query(
+        "CREATE TABLE test_merge_canceled_by_s3_errors "
+        " (key UInt32, value String)"
+        " Engine=MergeTree() "
+        " ORDER BY value "
+        " SETTINGS storage_policy='broken_s3'"
+    )
+    node.query("SYSTEM STOP MERGES test_merge_canceled_by_s3_errors")
+    node.query(
+        "INSERT INTO test_merge_canceled_by_s3_errors SELECT number, toString(number) FROM numbers(10000)"
+    )
+    node.query(
+        "INSERT INTO test_merge_canceled_by_s3_errors SELECT 2*number, toString(number) FROM numbers(10000)"
+    )
+    min_key = node.query("SELECT min(key) FROM test_merge_canceled_by_s3_errors")
+    assert int(min_key) == 0, min_key
+
+    response = cluster.exec_in_container(
+        cluster.get_container_id("resolver"),
+        [
+            "curl",
+            "-s",
+            f"http://localhost:8083/mock_settings/error_at_put?when_length_bigger=50000",
+        ],
+        nothrow=True,
+    )
+    assert response == "OK"
+
+    node.query("SYSTEM START MERGES test_merge_canceled_by_s3_errors")
+
+    error = node.query_and_get_error(
+        "OPTIMIZE TABLE test_merge_canceled_by_s3_errors FINAL",
+    )
+    assert "ExpectedError Message: mock s3 injected error" in error, error
+
+    node.wait_for_log_line("ExpectedError Message: mock s3 injected error")
+
+    check_no_objects_after_drop(
+        cluster, table_name="test_merge_canceled_by_s3_errors", node_name=node_name
+    )
+
+
+@pytest.mark.parametrize("node_name", ["node"])
+def test_merge_canceled_by_s3_errors_when_move(cluster, node_name):
+    node = cluster.instances[node_name]
+    settings = {
+        "storage_policy": "external_broken_s3",
+        "merge_with_ttl_timeout": 1,
+    }
+    create_table(node, "merge_canceled_by_s3_errors_when_move", **settings)
+
+    node.query("SYSTEM STOP MERGES merge_canceled_by_s3_errors_when_move")
+    node.query(
+        "INSERT INTO merge_canceled_by_s3_errors_when_move"
+        " VALUES {}".format(generate_values("2020-01-03", 1000))
+    )
+    node.query(
+        "INSERT INTO merge_canceled_by_s3_errors_when_move"
+        " VALUES {}".format(generate_values("2020-01-03", 1000, -1))
+    )
+
+    node.query(
+        "ALTER TABLE merge_canceled_by_s3_errors_when_move"
+        "    MODIFY TTL"
+        "        dt + INTERVAL 1 DAY "
+        "        TO VOLUME 'external'",
+        settings={"materialize_ttl_after_modify": 0},
+    )
+
+    response = cluster.exec_in_container(
+        cluster.get_container_id("resolver"),
+        [
+            "curl",
+            "-s",
+            f"http://localhost:8083/mock_settings/error_at_put?when_length_bigger=10000",
+        ],
+        nothrow=True,
+    )
+    assert response == "OK"
+
+    node.query("SYSTEM START MERGES merge_canceled_by_s3_errors_when_move")
+
+    node.query("OPTIMIZE TABLE merge_canceled_by_s3_errors_when_move FINAL")
+
+    node.wait_for_log_line("ExpectedError Message: mock s3 injected error")
+
+    count = node.query("SELECT count() FROM merge_canceled_by_s3_errors_when_move")
+    assert int(count) == 2000, count
+
+    check_no_objects_after_drop(
+        cluster, table_name="merge_canceled_by_s3_errors_when_move", node_name=node_name
+    )

From cfa2854ccf85178afc491f05f6cc5e8a2f0f9d72 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Wed, 3 May 2023 08:40:22 +0000
Subject: [PATCH 121/689] Automatic style fix

---
 tests/queries/0_stateless/01626_cnf_fuzz_long.python | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01626_cnf_fuzz_long.python b/tests/queries/0_stateless/01626_cnf_fuzz_long.python
index 87cd748590f..fc8486c27fd 100644
--- a/tests/queries/0_stateless/01626_cnf_fuzz_long.python
+++ b/tests/queries/0_stateless/01626_cnf_fuzz_long.python
@@ -55,9 +55,7 @@ def generate_dnf():
     return " OR ".join(clauses)
 
 
-select_query = (
-    "SELECT count() FROM t_cnf_fuzz WHERE {} SETTINGS convert_query_to_cnf = {}, allow_experimental_analyzer=1"
-)
+select_query = "SELECT count() FROM t_cnf_fuzz WHERE {} SETTINGS convert_query_to_cnf = {}, allow_experimental_analyzer=1"
 
 fail_report = """
 Failed query: '{}'.

From 14baf0e5074117ddd243143eee322677367ebae7 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio2368@users.noreply.github.com>
Date: Wed, 3 May 2023 10:44:07 +0200
Subject: [PATCH 122/689] Revert change

---
 tests/queries/0_stateless/01626_cnf_fuzz_long.python | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01626_cnf_fuzz_long.python b/tests/queries/0_stateless/01626_cnf_fuzz_long.python
index fc8486c27fd..de9e4a21dbb 100644
--- a/tests/queries/0_stateless/01626_cnf_fuzz_long.python
+++ b/tests/queries/0_stateless/01626_cnf_fuzz_long.python
@@ -55,7 +55,9 @@ def generate_dnf():
     return " OR ".join(clauses)
 
 
-select_query = "SELECT count() FROM t_cnf_fuzz WHERE {} SETTINGS convert_query_to_cnf = {}, allow_experimental_analyzer=1"
+select_query = (
+    "SELECT count() FROM t_cnf_fuzz WHERE {} SETTINGS convert_query_to_cnf = {}"
+)
 
 fail_report = """
 Failed query: '{}'.

From 189f276ff5eada947a375038160a88b2f0f1815f Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Wed, 3 May 2023 13:16:08 +0200
Subject: [PATCH 123/689] Fix

---
 src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 2 --
 src/Storages/StorageS3.cpp                    | 8 ++++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
index 7eda102ebc8..3e0a5a4a263 100644
--- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
+++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
@@ -107,8 +107,6 @@ IAsynchronousReader::Result ReadBufferFromRemoteFSGather::readInto(char * data,
     file_offset_of_buffer_end = offset;
     bytes_to_ignore = ignore;
 
-    assert(!bytes_to_ignore || initialized());
-
     auto result = nextImpl();
 
     if (result)
diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index 125135a5d1a..53b2140fff5 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -669,11 +669,15 @@ std::unique_ptr<ReadBuffer> StorageS3Source::createAsyncS3ReadBuffer(
         std::move(read_buffer_creator),
         StoredObjects{StoredObject{key, object_size}},
         read_settings,
-        context->getFilesystemCacheLog());
+        /* cache_log */nullptr);
+
+    auto modified_settings{read_settings};
+    /// FIXME: Changing this setting to default value breaks something around parquet reading
+    modified_settings.remote_read_min_bytes_for_seek = modified_settings.remote_fs_buffer_size;
 
     auto & pool_reader = context->getThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER);
     auto async_reader = std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(
-        pool_reader, read_settings, std::move(s3_impl),
+        pool_reader, modified_settings, std::move(s3_impl),
         context->getAsyncReadCounters(), context->getFilesystemReadPrefetchesLog());
 
     async_reader->setReadUntilEnd();

From eca52c4015211679163a7683cb4fda0a550d17dc Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Wed, 3 May 2023 15:41:12 +0200
Subject: [PATCH 124/689] Update src/Common/AsyncLoader.cpp

Co-authored-by: Antonio Andelic <antonio2368@users.noreply.github.com>
---
 src/Common/AsyncLoader.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
index e6bd6eb12d4..a4fc35788b9 100644
--- a/src/Common/AsyncLoader.cpp
+++ b/src/Common/AsyncLoader.cpp
@@ -24,7 +24,7 @@ void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, Atomic
 {
     if (processed % PRINT_MESSAGE_EACH_N_OBJECTS == 0 || watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS))
     {
-        LOG_INFO(log, "{}%", processed * 100.0 / total);
+        LOG_INFO(log, "Processed: {}%", processed * 100.0 / total);
         watch.restart();
     }
 }

From 5f2e6fae22e993d73dc19db5ae6adfab10111bcb Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Wed, 3 May 2023 15:42:18 +0200
Subject: [PATCH 125/689] Update src/Common/AsyncLoader.h

Co-authored-by: Antonio Andelic <antonio2368@users.noreply.github.com>
---
 src/Common/AsyncLoader.h | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index d35896942d0..44d60721c02 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -221,13 +221,11 @@ inline LoadJobSet getGoals(const LoadTaskPtrs & tasks)
 
 inline LoadJobSet joinJobs(const LoadJobSet & jobs1, const LoadJobSet & jobs2)
 {
-    if (jobs1.empty())
-        return jobs2;
-    if (jobs2.empty())
-        return jobs1;
     LoadJobSet result;
-    result.insert(jobs1.begin(), jobs1.end());
-    result.insert(jobs2.begin(), jobs2.end());
+    if (!jobs1.empty())
+        result.insert(jobs1.begin(), jobs1.end());
+    if (!jobs2.empty())
+        result.insert(jobs2.begin(), jobs2.end());
     return result;
 }
 

From eeedc5d3e7832bc6ec23349aa2563d73faa8d8f9 Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Wed, 3 May 2023 15:43:31 +0200
Subject: [PATCH 126/689] Update src/Common/AsyncLoader.h

Co-authored-by: Antonio Andelic <antonio2368@users.noreply.github.com>
---
 src/Common/AsyncLoader.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 44d60721c02..7d55802a258 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -264,7 +264,7 @@ inline LoadTaskPtrs joinTasks(const LoadTaskPtrs & tasks1, const LoadTaskPtrs &
 // 3b) If at least one of the job dependencies is failed or canceled, then this job is canceled (with all it's dependent jobs as well).
 //     On cancellation an ASYNC_LOAD_CANCELED exception is generated and saved inside LoadJob object. The job status is changed to CANCELED.
 //     Exception is rethrown by any existing or new `wait()` call. The job is moved to the set of the finished jobs.
-// 4)  The scheduled pending ready job starts execution by a worker. The job is dequeuedCallback `job_func` is called.
+// 4)  The scheduled pending ready job starts execution by a worker. The job is dequeued. Callback `job_func` is called.
 //     Status of an executing job is PENDING. And it is still considered as a scheduled job by AsyncLoader.
 //     Note that `job_func` of a CANCELED job is never executed.
 // 5a) On successful execution the job status is changed to OK and all existing and new `wait()` calls finish w/o exceptions.

From 356bb15e96f11583e25f361f800bb22bf9c02470 Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Wed, 3 May 2023 15:43:52 +0200
Subject: [PATCH 127/689] Update src/Common/AsyncLoader.h

Co-authored-by: Antonio Andelic <antonio2368@users.noreply.github.com>
---
 src/Common/AsyncLoader.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 7d55802a258..0b0564446b0 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -268,7 +268,7 @@ inline LoadTaskPtrs joinTasks(const LoadTaskPtrs & tasks1, const LoadTaskPtrs &
 //     Status of an executing job is PENDING. And it is still considered as a scheduled job by AsyncLoader.
 //     Note that `job_func` of a CANCELED job is never executed.
 // 5a) On successful execution the job status is changed to OK and all existing and new `wait()` calls finish w/o exceptions.
-// 5b) Any exception thrown out of `job_func` is wrapped into an ASYNC_LOAD_FAILED exception and save inside LoadJob.
+// 5b) Any exception thrown out of `job_func` is wrapped into an ASYNC_LOAD_FAILED exception and saved inside LoadJob.
 //     The job status is changed to FAILED. All the dependent jobs are canceled. The exception is rethrown from all existing and new `wait()` calls.
 // 6)  The job is no longer considered as scheduled and is instead moved to the finished jobs set. This is just for introspection of the finished jobs.
 // 7)  The task containing this job is destructed or `remove()` is explicitly called. The job is removed from the finished job set.

From 236a0d9da0a8e7f48d941e8e9761bd2485106d29 Mon Sep 17 00:00:00 2001
From: flynn <fenglv15@mails.ucas.ac.cn>
Date: Wed, 3 May 2023 15:46:17 +0000
Subject: [PATCH 128/689] add alias str_to_map and mapFromString for
 extractKeyValuePairs

---
 .../keyvaluepair/extractKeyValuePairs.cpp     |  2 ++
 ...t_key_value_pairs_multiple_input.reference | 22 +++++++++++++++++++
 ...extract_key_value_pairs_multiple_input.sql | 22 +++++++++++++++++++
 3 files changed, 46 insertions(+)

diff --git a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
index 4078b88c23e..a1b140001ac 100644
--- a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
+++ b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
@@ -228,6 +228,8 @@ REGISTER_FUNCTION(ExtractKeyValuePairs)
             └──────────────────┘
             ```)")
     );
+    factory.registerAlias("str_to_map", NameExtractKeyValuePairs::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("mapFromString", NameExtractKeyValuePairs::name);
 }
 
 }
diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
index 9daeafff779..3160825eb89 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
@@ -304,3 +304,25 @@ WITH
 SELECT
     x;
 {}
+WITH
+    str_to_map('name:neymar, age:31 team:psg,nationality:brazil') AS s_map,
+    CAST(
+            arrayMap(
+                    (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
+                ),
+            'Map(String,String)'
+        ) AS x
+SELECT
+    x;
+{'age':'31','name':'neymar','nationality':'brazil','team':'psg'}
+WITH
+    mapFromString('name:neymar, age:31 team:psg,nationality:brazil') AS s_map,
+    CAST(
+            arrayMap(
+                    (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
+                ),
+            'Map(String,String)'
+        ) AS x
+SELECT
+    x;
+{'age':'31','name':'neymar','nationality':'brazil','team':'psg'}
diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
index 7d681e36a4b..eeb004ecd36 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
@@ -426,3 +426,25 @@ WITH
         ) AS x
 SELECT
     x;
+
+WITH
+    str_to_map('name:neymar, age:31 team:psg,nationality:brazil') AS s_map,
+    CAST(
+            arrayMap(
+                    (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
+                ),
+            'Map(String,String)'
+        ) AS x
+SELECT
+    x;
+
+WITH
+    mapFromString('name:neymar, age:31 team:psg,nationality:brazil') AS s_map,
+    CAST(
+            arrayMap(
+                    (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
+                ),
+            'Map(String,String)'
+        ) AS x
+SELECT
+    x;

From 90a56950295da8ef6c62b3177dad971c09a99b9e Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Wed, 3 May 2023 17:13:40 +0000
Subject: [PATCH 129/689] add AsyncLoader introspection

---
 src/Common/AsyncLoader.cpp | 41 +++++++++++++++++++++++++++++++++-----
 src/Common/AsyncLoader.h   | 31 ++++++++++++++++++++++++----
 2 files changed, 63 insertions(+), 9 deletions(-)

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
index a4fc35788b9..69cc145d8c6 100644
--- a/src/Common/AsyncLoader.cpp
+++ b/src/Common/AsyncLoader.cpp
@@ -96,19 +96,25 @@ void LoadJob::canceled(const std::exception_ptr & ptr)
 void LoadJob::finish()
 {
     func = {}; // To ensure job function is destructed before `AsyncLoader::wait()` and `LoadJob::wait()` return
-    finished_ns = clock_gettime_ns();
+    finish_time = std::chrono::system_clock::now();
     if (waiters > 0)
         finished.notify_all();
 }
 
 void LoadJob::scheduled()
 {
-    scheduled_ns = clock_gettime_ns();
+    schedule_time = std::chrono::system_clock::now();
+}
+
+void LoadJob::enqueued()
+{
+    if (enqueue_time.load() == TimePoint{}) // Do not rewrite in case of requeue
+        enqueue_time = std::chrono::system_clock::now();
 }
 
 void LoadJob::execute(const LoadJobPtr & self)
 {
-    started_ns = clock_gettime_ns();
+    start_time = std::chrono::system_clock::now();
     func(self);
 }
 
@@ -217,7 +223,7 @@ void AsyncLoader::scheduleImpl(const LoadJobSet & input_jobs)
     // Restart watches after idle period
     if (scheduled_jobs.empty())
     {
-        busy_period_started_ns = clock_gettime_ns();
+        busy_period_start_time = std::chrono::system_clock::now();
         stopwatch.restart();
         old_jobs = finished_jobs.size();
     }
@@ -367,7 +373,7 @@ void AsyncLoader::remove(const LoadJobSet & jobs)
     for (const auto & job : jobs)
     {
         size_t erased = finished_jobs.erase(job);
-        if (old_jobs >= erased && job->finished_ns && job->finished_ns < busy_period_started_ns)
+        if (old_jobs >= erased && job->finishTime() != LoadJob::TimePoint{} && job->finishTime() < busy_period_start_time)
             old_jobs -= erased;
     }
 }
@@ -397,6 +403,29 @@ size_t AsyncLoader::getScheduledJobCount() const
     return scheduled_jobs.size();
 }
 
+std::vector<AsyncLoader::JobState> AsyncLoader::getJobStates() const
+{
+    std::unique_lock lock{mutex};
+    std::multimap<String, JobState> states;
+    for (const auto & [job, info] : scheduled_jobs)
+        states.emplace(job->name, JobState{
+            .job = job,
+            .dependencies_left = info.dependencies_left,
+            .is_executing = info.is_executing(),
+            .is_blocked = info.is_blocked(),
+            .is_ready = info.is_ready(),
+            .initial_priority = info.initial_priority,
+            .ready_seqno = last_ready_seqno
+        });
+    for (const auto & job : finished_jobs)
+        states.emplace(job->name, JobState{.job = job});
+    lock.unlock();
+    std::vector<JobState> result;
+    for (auto && [_, state] : states)
+        result.emplace_back(std::move(state));
+    return result;
+}
+
 void AsyncLoader::checkCycle(const LoadJobSet & jobs, std::unique_lock<std::mutex> & lock)
 {
     LoadJobSet left = jobs;
@@ -536,6 +565,8 @@ void AsyncLoader::enqueue(Info & info, const LoadJobPtr & job, std::unique_lock<
         ready_queue.emplace(info.key(), job);
     });
 
+    job->enqueued();
+
     if (is_running && workers < max_threads)
         spawn(lock);
 }
diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 0b0564446b0..4a27cddd9a9 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -70,6 +70,13 @@ public:
     // Returns number of threads blocked by `wait()` or `waitNoThrow()` calls.
     size_t waitersCount() const;
 
+    // Introspection
+    using TimePoint = std::chrono::system_clock::time_point;
+    TimePoint scheduleTime() const { return schedule_time; }
+    TimePoint enqueueTime() const { return enqueue_time; }
+    TimePoint startTime() const { return start_time; }
+    TimePoint finishTime() const { return finish_time; }
+
     const LoadJobSet dependencies; // Jobs to be done before this one (with ownership), it is `const` to make creation of cycles hard
     const String name;
 
@@ -82,6 +89,7 @@ private:
     void finish();
 
     void scheduled();
+    void enqueued();
     void execute(const LoadJobPtr & self);
 
     std::function<void(const LoadJobPtr & self)> func;
@@ -93,9 +101,10 @@ private:
     LoadStatus load_status{LoadStatus::PENDING};
     std::exception_ptr load_exception;
 
-    UInt64 scheduled_ns = 0;
-    UInt64 started_ns = 0;
-    UInt64 finished_ns = 0;
+    std::atomic<TimePoint> schedule_time{TimePoint{}};
+    std::atomic<TimePoint> enqueue_time{TimePoint{}};
+    std::atomic<TimePoint> start_time{TimePoint{}};
+    std::atomic<TimePoint> finish_time{TimePoint{}};
 };
 
 struct EmptyJobFunc
@@ -371,6 +380,20 @@ public:
     size_t getMaxThreads() const;
     size_t getScheduledJobCount() const;
 
+    // Helper class for introspection
+    struct JobState {
+        LoadJobPtr job;
+        size_t dependencies_left = 0;
+        bool is_executing = false;
+        bool is_blocked = false;
+        bool is_ready = false;
+        std::optional<ssize_t> initial_priority;
+        std::optional<UInt64> ready_seqno;
+    };
+
+    // For introspection and debug only, see `system.async_loader` table
+    std::vector<JobState> getJobStates() const;
+
 private:
     void checkCycle(const LoadJobSet & jobs, std::unique_lock<std::mutex> & lock);
     String checkCycleImpl(const LoadJobPtr & job, LoadJobSet & left, LoadJobSet & visited, std::unique_lock<std::mutex> & lock);
@@ -385,7 +408,7 @@ private:
     // Logging
     const bool log_failures; // Worker should log all exceptions caught from job functions.
     Poco::Logger * log;
-    UInt64 busy_period_started_ns;
+    std::chrono::system_clock::time_point busy_period_start_time;
     AtomicStopwatch stopwatch;
     size_t old_jobs = 0; // Number of jobs that were finished in previous busy period (for correct progress indication)
 

From 94dca90219a5f0fa23124354a1adc041e26517f5 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Wed, 3 May 2023 18:43:29 +0000
Subject: [PATCH 130/689] do not polute unit test output

---
 src/Common/AsyncLoader.cpp              | 6 ++++--
 src/Common/AsyncLoader.h                | 3 ++-
 src/Common/tests/gtest_async_loader.cpp | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
index 69cc145d8c6..6c19b1910eb 100644
--- a/src/Common/AsyncLoader.cpp
+++ b/src/Common/AsyncLoader.cpp
@@ -156,8 +156,9 @@ void LoadTask::detach()
     jobs.clear();
 }
 
-AsyncLoader::AsyncLoader(Metric metric_threads, Metric metric_active_threads, size_t max_threads_, bool log_failures_)
+AsyncLoader::AsyncLoader(Metric metric_threads, Metric metric_active_threads, size_t max_threads_, bool log_failures_, bool log_progress_)
     : log_failures(log_failures_)
+    , log_progress(log_progress_)
     , log(&Poco::Logger::get("AsyncLoader"))
     , max_threads(max_threads_)
     , pool(metric_threads, metric_active_threads, max_threads)
@@ -525,7 +526,8 @@ void AsyncLoader::finish(std::unique_lock<std::mutex> & lock, const LoadJobPtr &
     NOEXCEPT_SCOPE({
         ALLOW_ALLOCATIONS_IN_SCOPE;
         finished_jobs.insert(job);
-        logAboutProgress(log, finished_jobs.size() - old_jobs, finished_jobs.size() + scheduled_jobs.size() - old_jobs, stopwatch);
+        if (log_progress)
+            logAboutProgress(log, finished_jobs.size() - old_jobs, finished_jobs.size() + scheduled_jobs.size() - old_jobs, stopwatch);
     });
 }
 
diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 4a27cddd9a9..5726e843c37 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -339,7 +339,7 @@ private:
 public:
     using Metric = CurrentMetrics::Metric;
 
-    AsyncLoader(Metric metric_threads, Metric metric_active_threads, size_t max_threads_, bool log_failures_);
+    AsyncLoader(Metric metric_threads, Metric metric_active_threads, size_t max_threads_, bool log_failures_, bool log_progress_);
 
     // WARNING: all tasks instances should be destructed before associated AsyncLoader.
     ~AsyncLoader();
@@ -407,6 +407,7 @@ private:
 
     // Logging
     const bool log_failures; // Worker should log all exceptions caught from job functions.
+    const bool log_progress; // Periodically log total progress
     Poco::Logger * log;
     std::chrono::system_clock::time_point busy_period_start_time;
     AtomicStopwatch stopwatch;
diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 96b6e981c65..2f6c74a0f96 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -38,7 +38,7 @@ struct AsyncLoaderTest
     pcg64 rng{randomSeed()};
 
     explicit AsyncLoaderTest(size_t max_threads = 1)
-        : loader(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, max_threads, /* log_failures = */ false)
+        : loader(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, max_threads, /* log_failures = */ false, /* log_progress = */ false)
     {}
 
     template <typename T>

From 5c3e9b9e858fbd1560f6b6c94d190e920273a4c1 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Wed, 3 May 2023 18:57:55 +0000
Subject: [PATCH 131/689] fix style

---
 src/Common/AsyncLoader.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h
index 5726e843c37..d4a3218a541 100644
--- a/src/Common/AsyncLoader.h
+++ b/src/Common/AsyncLoader.h
@@ -381,7 +381,8 @@ public:
     size_t getScheduledJobCount() const;
 
     // Helper class for introspection
-    struct JobState {
+    struct JobState
+    {
         LoadJobPtr job;
         size_t dependencies_left = 0;
         bool is_executing = false;

From 8ae594fdd1f320f6e3b421f78a4e345dd4d4e049 Mon Sep 17 00:00:00 2001
From: Eridanus <maodingxuan@foxmail.com>
Date: Thu, 4 May 2023 03:05:11 +0800
Subject: [PATCH 132/689] empty commit


From 913b63edc93c80f8bfaedfe2332859fc5dab83d3 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 3 May 2023 21:28:33 +0200
Subject: [PATCH 133/689] Fix another zero copy bug

---
 src/Storages/MergeTree/MergeTreeData.cpp    |  4 +--
 src/Storages/MergeTree/ZeroCopyLock.cpp     |  2 +-
 src/Storages/MergeTree/ZeroCopyLock.h       |  2 ++
 src/Storages/StorageReplicatedMergeTree.cpp | 39 ++++++++++++++++++---
 4 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index fd98db7962e..e4181a5f9de 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -7202,8 +7202,8 @@ std::pair<MergeTreeData::MutableDataPartPtr, scope_guard> MergeTreeData::cloneAn
         copy_instead_of_hardlink,
         files_to_copy_instead_of_hardlinks);
 
-    LOG_DEBUG(log, "Clone {} part {} to {}{}",
-              src_flushed_tmp_part ? "flushed" : "",
+    LOG_DEBUG(log, "Clone{} part {} to {}{}",
+              src_flushed_tmp_part ? " flushed" : "",
               src_part_storage->getFullPath(),
               std::string(fs::path(dst_part_storage->getFullRootPath()) / tmp_dst_part_name),
               with_copy);
diff --git a/src/Storages/MergeTree/ZeroCopyLock.cpp b/src/Storages/MergeTree/ZeroCopyLock.cpp
index 53dfe0c769f..cca005dd7c0 100644
--- a/src/Storages/MergeTree/ZeroCopyLock.cpp
+++ b/src/Storages/MergeTree/ZeroCopyLock.cpp
@@ -3,7 +3,7 @@
 namespace DB
 {
     ZeroCopyLock::ZeroCopyLock(const zkutil::ZooKeeperPtr & zookeeper, const std::string & lock_path, const std::string & lock_message)
-        : lock(zkutil::createSimpleZooKeeperLock(zookeeper, lock_path, "part_exclusive_lock", lock_message))
+        : lock(zkutil::createSimpleZooKeeperLock(zookeeper, lock_path, ZERO_COPY_LOCK_NAME, lock_message))
     {
     }
 }
diff --git a/src/Storages/MergeTree/ZeroCopyLock.h b/src/Storages/MergeTree/ZeroCopyLock.h
index 4400ea55b8f..2803952af18 100644
--- a/src/Storages/MergeTree/ZeroCopyLock.h
+++ b/src/Storages/MergeTree/ZeroCopyLock.h
@@ -12,6 +12,8 @@ namespace DB
 /// because due to bad abstraction we use it in MergeTreeData.
 struct ZeroCopyLock
 {
+    static inline const std::string_view ZERO_COPY_LOCK_NAME = "part_exclusive_lock";
+
     ZeroCopyLock(const zkutil::ZooKeeperPtr & zookeeper, const std::string & lock_path, const std::string & lock_message);
 
     bool isLocked() const { return lock->isLocked(); }
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 20839a61c92..94abc1422fd 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -8249,7 +8249,7 @@ void StorageReplicatedMergeTree::lockSharedData(
     {
         String zookeeper_node = fs::path(zc_zookeeper_path) / id / replica_name;
 
-        LOG_TRACE(log, "Trying to create zookeeper persistent lock {}", zookeeper_node);
+        LOG_TRACE(log, "Trying to create zookeeper persistent lock {} with hardlinks [{}]", zookeeper_node, fmt::join(hardlinks, ", "));
 
         createZeroCopyLockNode(
             zookeeper, zookeeper_node, zkutil::CreateMode::Persistent,
@@ -8362,7 +8362,7 @@ namespace
 /// But sometimes we need an opposite. When we deleting all_0_0_0_1 it can be non replicated to other replicas, so we are the only owner of this part.
 /// In this case when we will drop all_0_0_0_1 we will drop blobs for all_0_0_0. But it will lead to dataloss. For such case we need to check that other replicas
 /// still need parent part.
-std::pair<bool, NameSet> getParentLockedBlobs(const ZooKeeperWithFaultInjectionPtr & zookeeper_ptr, const std::string & zero_copy_part_path_prefix, const MergeTreePartInfo & part_info, MergeTreeDataFormatVersion format_version, Poco::Logger * log)
+std::pair<bool, std::optional<NameSet>> getParentLockedBlobs(const ZooKeeperWithFaultInjectionPtr & zookeeper_ptr, const std::string & zero_copy_part_path_prefix, const MergeTreePartInfo & part_info, MergeTreeDataFormatVersion format_version, Poco::Logger * log)
 {
     NameSet files_not_to_remove;
 
@@ -8404,15 +8404,40 @@ std::pair<bool, NameSet> getParentLockedBlobs(const ZooKeeperWithFaultInjectionP
             /// Get hardlinked files
             String files_not_to_remove_str;
             Coordination::Error code;
-            zookeeper_ptr->tryGet(fs::path(zero_copy_part_path_prefix) / part_candidate_info_str, files_not_to_remove_str, nullptr, nullptr, &code);
+            zookeeper_ptr->tryGet(fs::path(zero_copy_part_path_prefix) / part_candidate_info_str, files_not_to_remove_str, nullptr, &code);
             if (code != Coordination::Error::ZOK)
+            {
                 LOG_TRACE(log, "Cannot get parent files from ZooKeeper on path ({}), error {}", (fs::path(zero_copy_part_path_prefix) / part_candidate_info_str).string(), errorMessage(code));
+                return {true, std::nullopt};
+            }
 
             if (!files_not_to_remove_str.empty())
             {
                 boost::split(files_not_to_remove, files_not_to_remove_str, boost::is_any_of("\n "));
                 LOG_TRACE(log, "Found files not to remove from parent part {}: [{}]", part_candidate_info_str, fmt::join(files_not_to_remove, ", "));
             }
+            else
+            {
+                std::vector<std::string> children;
+                code = zookeeper_ptr->tryGetChildren(fs::path(zero_copy_part_path_prefix) / part_candidate_info_str, children);
+                if (code != Coordination::Error::ZOK)
+                {
+                    LOG_TRACE(log, "Cannot get parent locks in ZooKeeper on path ({}), error {}", (fs::path(zero_copy_part_path_prefix) / part_candidate_info_str).string(), errorMessage(code));
+                    return {true, std::nullopt};
+                }
+
+                if (children.size() > 1 || children.size() == 1 && children[0] != ZeroCopyLock::ZERO_COPY_LOCK_NAME)
+                {
+                    LOG_TRACE(log, "No files not to remove found for part {} from parent {}", part_info_str, part_candidate_info_str);
+                }
+                else
+                {
+                    /// The case when part is actually removed, but some stale replica trying to execute merge/mutation.
+                    /// We shouldn't use the part to check hardlinked blobs, it just doesn't exist.
+                    LOG_TRACE(log, "Part {} is not parent (only merge/mutation locks exist), refusing to use as parent", part_candidate_info_str);
+                    continue;
+                }
+            }
 
             return {true, files_not_to_remove};
         }
@@ -8448,6 +8473,12 @@ std::pair<bool, NameSet> StorageReplicatedMergeTree::unlockSharedDataByID(
 
         auto [has_parent, parent_not_to_remove] = getParentLockedBlobs(
             zookeeper_ptr, fs::path(zc_zookeeper_path).parent_path(), part_info, data_format_version, logger);
+        if (has_parent && parent_not_to_remove == std::nullopt)
+        {
+            LOG_TRACE(logger, "Failed to get mutation parent on {} for part {}, refusing to remove blobs", zookeeper_part_replica_node, part_name);
+            return {false, {}};
+        }
+
         files_not_to_remove.insert(parent_not_to_remove.begin(), parent_not_to_remove.end());
 
         String zookeeper_part_uniq_node = fs::path(zc_zookeeper_path) / part_id;
@@ -8527,7 +8558,7 @@ std::pair<bool, NameSet> StorageReplicatedMergeTree::unlockSharedDataByID(
 
             if (error_code == Coordination::Error::ZOK)
             {
-                LOG_TRACE(logger, "Removed last parent zookeeper lock {} for part {} (part is finally unlocked)", zookeeper_part_uniq_node, part_name);
+                LOG_TRACE(logger, "Removed last parent zookeeper lock {} for part {} (part is finally unlocked)", zookeeper_part_node, part_name);
             }
             else if (error_code == Coordination::Error::ZNOTEMPTY)
             {

From 610e63bfd2b21e06c312c625e54279c31d06853c Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 3 May 2023 23:54:24 +0200
Subject: [PATCH 134/689] Fix build

---
 src/Storages/MergeTree/ZeroCopyLock.h       |  2 +-
 src/Storages/StorageReplicatedMergeTree.cpp | 19 +++++++++++--------
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/Storages/MergeTree/ZeroCopyLock.h b/src/Storages/MergeTree/ZeroCopyLock.h
index 2803952af18..d4c829a3652 100644
--- a/src/Storages/MergeTree/ZeroCopyLock.h
+++ b/src/Storages/MergeTree/ZeroCopyLock.h
@@ -12,7 +12,7 @@ namespace DB
 /// because due to bad abstraction we use it in MergeTreeData.
 struct ZeroCopyLock
 {
-    static inline const std::string_view ZERO_COPY_LOCK_NAME = "part_exclusive_lock";
+    static inline const auto ZERO_COPY_LOCK_NAME = "part_exclusive_lock";
 
     ZeroCopyLock(const zkutil::ZooKeeperPtr & zookeeper, const std::string & lock_path, const std::string & lock_message);
 
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 94abc1422fd..280150f27ad 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -8404,7 +8404,7 @@ std::pair<bool, std::optional<NameSet>> getParentLockedBlobs(const ZooKeeperWith
             /// Get hardlinked files
             String files_not_to_remove_str;
             Coordination::Error code;
-            zookeeper_ptr->tryGet(fs::path(zero_copy_part_path_prefix) / part_candidate_info_str, files_not_to_remove_str, nullptr, &code);
+            zookeeper_ptr->tryGet(fs::path(zero_copy_part_path_prefix) / part_candidate_info_str, files_not_to_remove_str, nullptr, nullptr, &code);
             if (code != Coordination::Error::ZOK)
             {
                 LOG_TRACE(log, "Cannot get parent files from ZooKeeper on path ({}), error {}", (fs::path(zero_copy_part_path_prefix) / part_candidate_info_str).string(), errorMessage(code));
@@ -8426,7 +8426,7 @@ std::pair<bool, std::optional<NameSet>> getParentLockedBlobs(const ZooKeeperWith
                     return {true, std::nullopt};
                 }
 
-                if (children.size() > 1 || children.size() == 1 && children[0] != ZeroCopyLock::ZERO_COPY_LOCK_NAME)
+                if (children.size() > 1 || (children.size() == 1 && children[0] != ZeroCopyLock::ZERO_COPY_LOCK_NAME))
                 {
                     LOG_TRACE(log, "No files not to remove found for part {} from parent {}", part_info_str, part_candidate_info_str);
                 }
@@ -8471,20 +8471,23 @@ std::pair<bool, NameSet> StorageReplicatedMergeTree::unlockSharedDataByID(
         if (!files_not_to_remove_str.empty())
             boost::split(files_not_to_remove, files_not_to_remove_str, boost::is_any_of("\n "));
 
+        String zookeeper_part_uniq_node = fs::path(zc_zookeeper_path) / part_id;
+
+        /// Delete our replica node for part from zookeeper (we are not interested in it anymore)
+        String zookeeper_part_replica_node = fs::path(zookeeper_part_uniq_node) / replica_name_;
+
         auto [has_parent, parent_not_to_remove] = getParentLockedBlobs(
             zookeeper_ptr, fs::path(zc_zookeeper_path).parent_path(), part_info, data_format_version, logger);
-        if (has_parent && parent_not_to_remove == std::nullopt)
+
+        // parent_not_to_remove == std::nullopt means that we were unable to retrieve parts set
+        if (has_parent || parent_not_to_remove == std::nullopt)
         {
             LOG_TRACE(logger, "Failed to get mutation parent on {} for part {}, refusing to remove blobs", zookeeper_part_replica_node, part_name);
             return {false, {}};
         }
 
-        files_not_to_remove.insert(parent_not_to_remove.begin(), parent_not_to_remove.end());
+        files_not_to_remove.insert(parent_not_to_remove->begin(), parent_not_to_remove->end());
 
-        String zookeeper_part_uniq_node = fs::path(zc_zookeeper_path) / part_id;
-
-        /// Delete our replica node for part from zookeeper (we are not interested in it anymore)
-        String zookeeper_part_replica_node = fs::path(zookeeper_part_uniq_node) / replica_name_;
 
         LOG_TRACE(logger, "Remove zookeeper lock {} for part {}", zookeeper_part_replica_node, part_name);
 

From c8cccb393e33ce1b5be5301db6d172a99da54420 Mon Sep 17 00:00:00 2001
From: xmy <xumovens@gmail.com>
Date: Thu, 4 May 2023 16:02:23 +0800
Subject: [PATCH 135/689] fix

---
 .../AggregateFunctionBitwise.cpp              |  2 +-
 .../AggregateFunctionGroupBitmap.cpp          |  1 -
 .../AggregateFunctionSequenceMatch.cpp        |  2 +-
 .../AggregateFunctionWindowFunnel.cpp         |  2 +-
 src/AggregateFunctions/Helpers.h              | 36 ++++++++++++-------
 ...0498_bitwise_aggregate_functions.reference |  8 +++++
 .../00498_bitwise_aggregate_functions.sql     |  2 ++
 7 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionBitwise.cpp b/src/AggregateFunctions/AggregateFunctionBitwise.cpp
index 82cb3b327f0..f5c2deb4588 100644
--- a/src/AggregateFunctions/AggregateFunctionBitwise.cpp
+++ b/src/AggregateFunctions/AggregateFunctionBitwise.cpp
@@ -27,7 +27,7 @@ AggregateFunctionPtr createAggregateFunctionBitwise(const std::string & name, co
                         "is illegal, because it cannot be used in bitwise operations",
                         argument_types[0]->getName(), name);
 
-    AggregateFunctionPtr res(createWithOptionSignedIntegerType<AggregateFunctionBitwise, Data, true>(*argument_types[0], argument_types[0]));
+    AggregateFunctionPtr res(createWithIntegerType<AggregateFunctionBitwise, Data>(*argument_types[0], argument_types[0]));
 
     if (!res)
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
diff --git a/src/AggregateFunctions/AggregateFunctionGroupBitmap.cpp b/src/AggregateFunctions/AggregateFunctionGroupBitmap.cpp
index beabfe83c47..fd350b47026 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupBitmap.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupBitmap.cpp
@@ -1,6 +1,5 @@
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 #include <AggregateFunctions/FactoryHelpers.h>
-#include <AggregateFunctions/Helpers.h>
 #include <DataTypes/DataTypeAggregateFunction.h>
 
 // TODO include this last because of a broken roaring header. See the comment inside.
diff --git a/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp b/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp
index f2fe9014ceb..3dd9a8b658d 100644
--- a/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp
+++ b/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp
@@ -53,7 +53,7 @@ AggregateFunctionPtr createAggregateFunctionSequenceBase(
 
     String pattern = params.front().safeGet<std::string>();
 
-    AggregateFunctionPtr res(createWithOptionSignedIntegerType<AggregateFunction, Data>(*argument_types[0], argument_types, params, pattern));
+    AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunction, Data>(*argument_types[0], argument_types, params, pattern));
     if (res)
         return res;
 
diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp
index 71c675f7a3b..d80d683fd04 100644
--- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp
+++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp
@@ -48,7 +48,7 @@ createAggregateFunctionWindowFunnel(const std::string & name, const DataTypes &
                             cond_arg->getName(), toString(i + 1), name);
     }
 
-    AggregateFunctionPtr res(createWithOptionSignedIntegerType<AggregateFunctionWindowFunnel, Data>(*arguments[0], arguments, params));
+    AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunctionWindowFunnel, Data>(*arguments[0], arguments, params));
     WhichDataType which(arguments.front().get());
     if (res)
         return res;
diff --git a/src/AggregateFunctions/Helpers.h b/src/AggregateFunctions/Helpers.h
index 4131279a897..e5cfc3034b0 100644
--- a/src/AggregateFunctions/Helpers.h
+++ b/src/AggregateFunctions/Helpers.h
@@ -87,8 +87,8 @@ static IAggregateFunction * createWithNumericType(const IDataType & argument_typ
     return nullptr;
 }
 
-template <template <typename, typename> class AggregateFunctionTemplate, template <typename> class Data, bool support_signed = false, typename... TArgs>
-static IAggregateFunction * createWithOptionSignedIntegerType(const IDataType & argument_type, TArgs && ... args)
+template <template <typename, typename> class AggregateFunctionTemplate, template <typename> class Data, typename... TArgs>
+static IAggregateFunction * createWithUnsignedIntegerType(const IDataType & argument_type, TArgs && ... args)
 {
     WhichDataType which(argument_type);
     if (which.idx == TypeIndex::UInt8) return new AggregateFunctionTemplate<UInt8, Data<UInt8>>(std::forward<TArgs>(args)...);
@@ -97,19 +97,31 @@ static IAggregateFunction * createWithOptionSignedIntegerType(const IDataType &
     if (which.idx == TypeIndex::UInt64) return new AggregateFunctionTemplate<UInt64, Data<UInt64>>(std::forward<TArgs>(args)...);
     if (which.idx == TypeIndex::UInt128) return new AggregateFunctionTemplate<UInt128, Data<UInt128>>(std::forward<TArgs>(args)...);
     if (which.idx == TypeIndex::UInt256) return new AggregateFunctionTemplate<UInt256, Data<UInt256>>(std::forward<TArgs>(args)...);
-
-    if constexpr (support_signed)
-    {
-        if (which.idx == TypeIndex::Int8) return new AggregateFunctionTemplate<Int8, Data<Int8>>(std::forward<TArgs>(args)...);
-        if (which.idx == TypeIndex::Int16) return new AggregateFunctionTemplate<Int16, Data<Int16>>(std::forward<TArgs>(args)...);
-        if (which.idx == TypeIndex::Int32) return new AggregateFunctionTemplate<Int32, Data<Int32>>(std::forward<TArgs>(args)...);
-        if (which.idx == TypeIndex::Int64) return new AggregateFunctionTemplate<Int64, Data<Int64>>(std::forward<TArgs>(args)...);
-        if (which.idx == TypeIndex::Int128) return new AggregateFunctionTemplate<Int128, Data<Int128>>(std::forward<TArgs>(args)...);
-        if (which.idx == TypeIndex::Int256) return new AggregateFunctionTemplate<Int256, Data<Int256>>(std::forward<TArgs>(args)...);
-    }
     return nullptr;
 }
 
+template <template <typename, typename> class AggregateFunctionTemplate, template <typename> class Data, typename... TArgs>
+static IAggregateFunction * createWithSignedIntegerType(const IDataType & argument_type, TArgs && ... args)
+{
+    WhichDataType which(argument_type);
+    if (which.idx == TypeIndex::Int8) return new AggregateFunctionTemplate<Int8, Data<Int8>>(std::forward<TArgs>(args)...);
+    if (which.idx == TypeIndex::Int16) return new AggregateFunctionTemplate<Int16, Data<Int16>>(std::forward<TArgs>(args)...);
+    if (which.idx == TypeIndex::Int32) return new AggregateFunctionTemplate<Int32, Data<Int32>>(std::forward<TArgs>(args)...);
+    if (which.idx == TypeIndex::Int64) return new AggregateFunctionTemplate<Int64, Data<Int64>>(std::forward<TArgs>(args)...);
+    if (which.idx == TypeIndex::Int128) return new AggregateFunctionTemplate<Int128, Data<Int128>>(std::forward<TArgs>(args)...);
+    if (which.idx == TypeIndex::Int256) return new AggregateFunctionTemplate<Int256, Data<Int256>>(std::forward<TArgs>(args)...);
+    return nullptr;
+}
+
+template <template <typename, typename> class AggregateFunctionTemplate, template <typename> class Data, typename... TArgs>
+static IAggregateFunction * createWithIntegerType(const IDataType & argument_type, TArgs && ... args)
+{
+    IAggregateFunction * f = createWithUnsignedIntegerType<AggregateFunctionTemplate, Data>(argument_type, std::forward<TArgs>(args)...);
+    if (f)
+        return f;
+    return createWithSignedIntegerType<AggregateFunctionTemplate, Data>(argument_type, std::forward<TArgs>(args)...);
+}
+
 template <template <typename, typename> class AggregateFunctionTemplate, template <typename> class Data, typename... TArgs>
 static IAggregateFunction * createWithBasicNumberOrDateOrDateTime(const IDataType & argument_type, TArgs &&... args)
 {
diff --git a/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference b/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference
index fb5cfb57d39..7986080ace3 100644
--- a/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference
+++ b/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference
@@ -2,3 +2,11 @@
 1	[1,5,9,13,17]	29	1	17
 2	[2,6,10,14,18]	30	2	18
 3	[3,7,11,15,19]	31	3	19
+0   [0,-4,-8,-12,-16]   -4  0   0
+1   [-1,-5,-9,-13,-17]  -1  -29 -17
+2   [-2,-6,-10,-14,-18] -2  -30 -18
+3   [-3,-7,-11,-15,-19] -3  -31 -19
+0   [-10,-6,-2,2,6] -2  2   -10
+1   [-9,-5,-1,3,7]  -1  3   -9
+2   [-8,-4,0,4,8]   -4  0   8
+3   [-7,-3,1,5,9]   -3  1   9
\ No newline at end of file
diff --git a/tests/queries/0_stateless/00498_bitwise_aggregate_functions.sql b/tests/queries/0_stateless/00498_bitwise_aggregate_functions.sql
index 07cd0b3630f..2c4610e6c4a 100644
--- a/tests/queries/0_stateless/00498_bitwise_aggregate_functions.sql
+++ b/tests/queries/0_stateless/00498_bitwise_aggregate_functions.sql
@@ -1 +1,3 @@
 SELECT number % 4 AS k, groupArray(number), groupBitOr(number), groupBitAnd(number), groupBitXor(number) FROM (SELECT * FROM system.numbers LIMIT 20) GROUP BY k ORDER BY k;
+SELECT number % 4 AS k, groupArray(-number), groupBitOr(-number), groupBitAnd(-number), groupBitXor(-number) FROM (SELECT * FROM system.numbers LIMIT 20) GROUP BY k ORDER BY k;
+SELECT number % 4 AS k, groupArray(number-10), groupBitOr(number-10), groupBitAnd(number-10), groupBitXor(number-10) FROM (SELECT * FROM system.numbers LIMIT 20) GROUP BY k ORDER BY k;

From dfd1c4d8e698106a28983edc5cadbe71737553c3 Mon Sep 17 00:00:00 2001
From: xmy <xumovens@gmail.com>
Date: Thu, 4 May 2023 16:11:39 +0800
Subject: [PATCH 136/689] fix

---
 .../00498_bitwise_aggregate_functions.reference  | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference b/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference
index 7986080ace3..400db09c9f1 100644
--- a/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference
+++ b/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference
@@ -2,11 +2,11 @@
 1	[1,5,9,13,17]	29	1	17
 2	[2,6,10,14,18]	30	2	18
 3	[3,7,11,15,19]	31	3	19
-0   [0,-4,-8,-12,-16]   -4  0   0
-1   [-1,-5,-9,-13,-17]  -1  -29 -17
-2   [-2,-6,-10,-14,-18] -2  -30 -18
-3   [-3,-7,-11,-15,-19] -3  -31 -19
-0   [-10,-6,-2,2,6] -2  2   -10
-1   [-9,-5,-1,3,7]  -1  3   -9
-2   [-8,-4,0,4,8]   -4  0   8
-3   [-7,-3,1,5,9]   -3  1   9
\ No newline at end of file
+0       [0,-4,-8,-12,-16]       -4      0       0
+1       [-1,-5,-9,-13,-17]      -1      -29     -17
+2       [-2,-6,-10,-14,-18]     -2      -30     -18
+3       [-3,-7,-11,-15,-19]     -3      -31     -19
+0       [-10,-6,-2,2,6] -2      2       -10
+1       [-9,-5,-1,3,7]  -1      3       -9
+2       [-8,-4,0,4,8]   -4      0       8
+3       [-7,-3,1,5,9]   -3      1       9
\ No newline at end of file

From a788d0b124d9f20bc0e05e8a825920925f371460 Mon Sep 17 00:00:00 2001
From: ivan-klass <ivan.klass@performi.ca>
Date: Thu, 4 May 2023 13:03:27 +0200
Subject: [PATCH 137/689] Note for multiple arrayJoin of same array expression

---
 docs/en/sql-reference/functions/array-join.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/docs/en/sql-reference/functions/array-join.md b/docs/en/sql-reference/functions/array-join.md
index 14968eb1092..98675f02f43 100644
--- a/docs/en/sql-reference/functions/array-join.md
+++ b/docs/en/sql-reference/functions/array-join.md
@@ -78,6 +78,22 @@ GROUP BY
 │           1 │ Bobruisk │ Firefox │
 └─────────────┴──────────┴─────────┘
 ```
+### Important note!
+Using multiple `arrayJoin` with same expression may not produce expected results due optimizations.
+For that cases, consider changing array expression that is repeated with extra operations that do not affect join result - e.g. `arrayJoin(arraySort(arr))`, `arrayJoin(arrayConcat(arr, []))`
+
+Example:
+```sql
+SELECT
+    arrayJoin(dice) as first_throw,
+    /* arrayJoin(dice) as second_throw */ -- is technically correct, but will annihilate result set
+    arrayJoin(arrayConcat(dice, [])) as second_throw -- intentionally changed expression to force re-evaluation
+FROM (
+    SELECT [1, 2, 3, 4, 5, 6] as dice
+);
+```
+
+
 
 Note the [ARRAY JOIN](../statements/select/array-join.md) syntax in the SELECT query, which provides broader possibilities.
 `ARRAY JOIN` allows you to convert multiple arrays with the same number of elements at a time.

From 1da30e283049aa2d3948e89569bda5988bc36c39 Mon Sep 17 00:00:00 2001
From: ivan-klass <ivan.klass@performi.ca>
Date: Thu, 4 May 2023 13:06:07 +0200
Subject: [PATCH 138/689] Better text

---
 docs/en/sql-reference/functions/array-join.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/sql-reference/functions/array-join.md b/docs/en/sql-reference/functions/array-join.md
index 98675f02f43..bbb9ca1ba04 100644
--- a/docs/en/sql-reference/functions/array-join.md
+++ b/docs/en/sql-reference/functions/array-join.md
@@ -79,8 +79,8 @@ GROUP BY
 └─────────────┴──────────┴─────────┘
 ```
 ### Important note!
-Using multiple `arrayJoin` with same expression may not produce expected results due optimizations.
-For that cases, consider changing array expression that is repeated with extra operations that do not affect join result - e.g. `arrayJoin(arraySort(arr))`, `arrayJoin(arrayConcat(arr, []))`
+Using multiple `arrayJoin` with same expression may not produce expected results due to optimizations.
+For that cases, consider modifying repeated array expression with extra operations that do not affect join result - e.g. `arrayJoin(arraySort(arr))`, `arrayJoin(arrayConcat(arr, []))`
 
 Example:
 ```sql

From 9bb681c104e4652dc149c76306753f73edf3eaa0 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Thu, 4 May 2023 13:31:07 +0200
Subject: [PATCH 139/689] Fix build

---
 src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp
index f2b892b96eb..8cf0b27a517 100644
--- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp
@@ -51,6 +51,7 @@ std::unique_ptr<ReadBufferFromFileBase> LocalObjectStorage::readObjects( /// NOL
     std::optional<size_t> file_size) const
 {
     auto modified_settings = patchSettings(read_settings);
+    auto global_context = Context::getGlobalContextInstance();
     auto read_buffer_creator =
         [=] (const std::string & file_path, size_t /* read_until_position */)
         -> std::unique_ptr<ReadBufferFromFileBase>
@@ -59,14 +60,18 @@ std::unique_ptr<ReadBufferFromFileBase> LocalObjectStorage::readObjects( /// NOL
     };
 
     auto impl = std::make_unique<ReadBufferFromRemoteFSGather>(
-        std::move(read_buffer_creator), objects, modified_settings);
+        std::move(read_buffer_creator), objects, modified_settings,
+        global_context->getFilesystemCacheLog());
 
     /// We use `remove_fs_method` (not `local_fs_method`) because we are about to use
     /// AsynchronousReadIndirectBufferFromRemoteFS which works by the remote_fs_* settings.
     if (modified_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
     {
-        auto & reader = getThreadPoolReader();
-        return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, modified_settings, std::move(impl));
+        auto & reader = global_context->getThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER);
+        return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(
+            reader, modified_settings, std::move(impl),
+            global_context->getAsyncReadCounters(),
+            global_context->getFilesystemReadPrefetchesLog());
     }
     else
     {

From 98c68535df60497f0a9674937ef2e3c81516a735 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Thu, 4 May 2023 12:01:50 +0000
Subject: [PATCH 140/689] Remove from broken tests

---
 tests/broken_tests.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/broken_tests.txt b/tests/broken_tests.txt
index b0a8e0fe535..48e546cbaaa 100644
--- a/tests/broken_tests.txt
+++ b/tests/broken_tests.txt
@@ -127,7 +127,6 @@
 02324_map_combinator_bug
 02241_join_rocksdb_bs
 02003_WithMergeableStateAfterAggregationAndLimit_LIMIT_BY_LIMIT_OFFSET
-01626_cnf_fuzz_long
 01115_join_with_dictionary
 01009_global_array_join_names
 00917_multiple_joins_denny_crane
@@ -136,4 +135,4 @@
 00261_storage_aliases_and_array_join
 02701_non_parametric_function
 01825_type_json_multiple_files
-01281_group_by_limit_memory_tracking
\ No newline at end of file
+01281_group_by_limit_memory_tracking

From 291bcc0e544e0f1acae7675500b889055d0704e5 Mon Sep 17 00:00:00 2001
From: Mike Kot <myrrc@double.cloud>
Date: Thu, 4 May 2023 12:38:50 +0000
Subject: [PATCH 141/689] allow passing password in URL

---
 programs/server/play.html | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/programs/server/play.html b/programs/server/play.html
index 323ba7d0b9e..b36e6c10d8d 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -517,11 +517,12 @@
     let previous_query = '';
 
     const current_url = new URL(window.location);
+    const opened_locally = document.location.href.startsWith("file://");
 
     const server_address = current_url.searchParams.get('url');
     if (server_address) {
         document.getElementById('url').value = server_address;
-    } else if (location.protocol != 'file:') {
+    } else if (!opened_locally) {
         /// Substitute the address of the server where the page is served.
         document.getElementById('url').value = location.origin;
     }
@@ -532,6 +533,18 @@
         document.getElementById('user').value = user_from_url;
     }
 
+    const pass_from_url = current_url.searchParams.get('password');
+    if (pass_from_url) {
+        document.getElementById('password').value = pass_from_url;
+        if (!opened_locally) {
+            let replaced_pass = current_url.searchParams;
+            replaced_pass.delete('password');
+            window.history.replaceState(null, '',
+                window.location.origin + window.location.pathname + '?'
+                + replaced_pass.toString() + window.location.hash);
+        }
+    }
+
     function postImpl(posted_request_num, query)
     {
         const user = document.getElementById('user').value;
@@ -548,7 +561,7 @@
             '&max_result_rows=1000&max_result_bytes=10000000&result_overflow_mode=break';
 
         // If play.html is opened locally, append username and password to the URL parameter to avoid CORS issue.
-        if (document.location.href.startsWith("file://")) {
+        if (opened_locally) {
             url += '&user=' + encodeURIComponent(user) +
             '&password=' + encodeURIComponent(password)
         }
@@ -557,7 +570,7 @@
 
         xhr.open('POST', url, true);
         // If play.html is open normally, use Basic auth to prevent username and password being exposed in URL parameters
-        if (!document.location.href.startsWith("file://")) {
+        if (!opened_locally) {
             xhr.setRequestHeader("Authorization", "Basic " + btoa(user+":"+password));
         }
         xhr.onreadystatechange = function()

From 3aaea2b91d34a98651e62f9cedd263d9333530d5 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Thu, 4 May 2023 13:07:28 +0000
Subject: [PATCH 142/689] join thread even exception happened

---
 src/Interpreters/TraceCollector.cpp | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/TraceCollector.cpp b/src/Interpreters/TraceCollector.cpp
index 49588d490f5..bedc27f7d15 100644
--- a/src/Interpreters/TraceCollector.cpp
+++ b/src/Interpreters/TraceCollector.cpp
@@ -56,9 +56,17 @@ TraceCollector::~TraceCollector()
   */
 void TraceCollector::stop()
 {
-    WriteBufferFromFileDescriptor out(TraceSender::pipe.fds_rw[1]);
-    writeChar(true, out);
-    out.next();
+    try
+    {
+        WriteBufferFromFileDescriptor out(TraceSender::pipe.fds_rw[1]);
+        writeChar(true, out);
+        out.next();
+    }
+    catch (...)
+    {
+        tryLogCurrentException("TraceCollector");
+    }
+
     thread.join();
 }
 

From 9825531cb1d9c7178e5ba924e0db8036dbe1d63f Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Thu, 4 May 2023 13:38:28 +0000
Subject: [PATCH 143/689] Better

---
 src/IO/S3/Client.cpp       | 49 +++++++++++++++++++++-----------------
 src/IO/S3/Client.h         |  9 +++++--
 src/IO/S3/ProviderType.cpp | 12 ++++++++--
 src/IO/S3/ProviderType.h   |  8 ++++++-
 src/IO/S3/Requests.cpp     |  2 +-
 src/IO/S3/Requests.h       |  6 ++---
 src/IO/S3/copyS3File.cpp   |  2 +-
 7 files changed, 56 insertions(+), 32 deletions(-)

diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp
index c1d09f60101..3c0a8122a91 100644
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@@ -115,22 +115,8 @@ std::unique_ptr<Client> Client::create(const Client & other)
 namespace
 {
 
-ProviderType deduceProviderType(const std::string & url, const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> & credentials_provider)
+ProviderType deduceProviderType(const std::string & url)
 {
-    /// GCS can operate in 2 modes for header and query params names:
-    /// - with both x-amz and x-goog prefixes allowed (but cannot mix different prefixes in same request)
-    /// - only with x-goog prefix
-    /// first mode is allowed only with HMAC (or unsigned requests) so when we
-    /// find credential keys we can simply behave as the underlying storage is S3
-    /// otherwise, we need to be aware we are making requests to GCS
-    /// and replace all headers with a valid prefix when needed
-    if (credentials_provider)
-    {
-        auto credentials = credentials_provider->GetAWSCredentials();
-        if (!credentials.IsEmpty())
-            return ProviderType::AWS;
-    }
-
     if (url.find(".amazonaws.com") != std::string::npos)
         return ProviderType::AWS;
 
@@ -158,10 +144,29 @@ Client::Client(
     endpoint_provider->GetBuiltInParameters().GetParameter("Region").GetString(explicit_region);
     endpoint_provider->GetBuiltInParameters().GetParameter("Endpoint").GetString(initial_endpoint);
 
-    provider_type = deduceProviderType(initial_endpoint, credentials_provider);
+    provider_type = deduceProviderType(initial_endpoint);
     LOG_TRACE(log, "Provider type: {}", toString(provider_type));
 
-    detect_region = initial_endpoint.find(".amazonaws.com") != std::string::npos && explicit_region == Aws::Region::AWS_GLOBAL;
+    if (provider_type == ProviderType::GCS)
+    {
+        /// GCS can operate in 2 modes for header and query params names:
+        /// - with both x-amz and x-goog prefixes allowed (but cannot mix different prefixes in same request)
+        /// - only with x-goog prefix
+        /// first mode is allowed only with HMAC (or unsigned requests) so when we
+        /// find credential keys we can simply behave as the underlying storage is S3
+        /// otherwise, we need to be aware we are making requests to GCS
+        /// and replace all headers with a valid prefix when needed
+        if (credentials_provider)
+        {
+            auto credentials = credentials_provider->GetAWSCredentials();
+            if (credentials.IsEmpty())
+                api_mode = ApiMode::GCS;
+        }
+    }
+
+    LOG_TRACE(log, "API mode: {}", toString(api_mode));
+
+    detect_region = provider_type == ProviderType::AWS && explicit_region == Aws::Region::AWS_GLOBAL;
 
     cache = std::make_shared<ClientCache>();
     ClientCacheRegistry::instance().registerClient(cache);
@@ -238,7 +243,7 @@ Model::HeadObjectOutcome Client::HeadObject(const HeadObjectRequest & request) c
 {
     const auto & bucket = request.GetBucket();
 
-    request.setProviderType(provider_type);
+    request.setApiMode(api_mode);
 
     if (auto region = getRegionForBucket(bucket); !region.empty())
     {
@@ -378,7 +383,7 @@ std::invoke_result_t<RequestFn, RequestType>
 Client::doRequest(const RequestType & request, RequestFn request_fn) const
 {
     const auto & bucket = request.GetBucket();
-    request.setProviderType(provider_type);
+    request.setApiMode(api_mode);
 
     if (auto region = getRegionForBucket(bucket); !region.empty())
     {
@@ -451,9 +456,9 @@ Client::doRequest(const RequestType & request, RequestFn request_fn) const
     throw Exception(ErrorCodes::TOO_MANY_REDIRECTS, "Too many redirects");
 }
 
-ProviderType Client::getProviderType() const
+bool Client::supportsMultiPartCopy() const
 {
-    return provider_type;
+    return provider_type != ProviderType::GCS;
 }
 
 void Client::BuildHttpRequest(const Aws::AmazonWebServiceRequest& request,
@@ -461,7 +466,7 @@ void Client::BuildHttpRequest(const Aws::AmazonWebServiceRequest& request,
 {
     Aws::S3::S3Client::BuildHttpRequest(request, httpRequest);
 
-    if (provider_type == ProviderType::GCS)
+    if (api_mode == ApiMode::GCS)
     {
         /// some GCS requests don't like S3 specific headers that the client sets
         httpRequest->DeleteHeader("x-amz-api-version");
diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h
index 0bf07578e90..63feb94e593 100644
--- a/src/IO/S3/Client.h
+++ b/src/IO/S3/Client.h
@@ -190,10 +190,10 @@ public:
     using Aws::S3::S3Client::EnableRequestProcessing;
     using Aws::S3::S3Client::DisableRequestProcessing;
 
-    ProviderType getProviderType() const;
-
     void BuildHttpRequest(const Aws::AmazonWebServiceRequest& request,
                           const std::shared_ptr<Aws::Http::HttpRequest>& httpRequest) const override;
+
+    bool supportsMultiPartCopy() const;
 private:
     Client(size_t max_redirects_,
            ServerSideEncryptionKMSConfig sse_kms_config_,
@@ -241,7 +241,12 @@ private:
     std::string explicit_region;
     mutable bool detect_region = true;
 
+    /// provider type can determine if some functionality is supported
+    /// but for same provider, we would need to generate different headers depending on the
+    /// mode
+    /// E.g. GCS can work in AWS mode in some cases and accept headers with x-amz prefix
     ProviderType provider_type{ProviderType::UNKNOWN};
+    ApiMode api_mode{ApiMode::AWS};
 
     mutable std::shared_ptr<ClientCache> cache;
 
diff --git a/src/IO/S3/ProviderType.cpp b/src/IO/S3/ProviderType.cpp
index 39be90ff63d..5987701db6b 100644
--- a/src/IO/S3/ProviderType.cpp
+++ b/src/IO/S3/ProviderType.cpp
@@ -22,9 +22,17 @@ std::string_view toString(ProviderType provider_type)
     }
 }
 
-bool supportsMultiPartCopy(ProviderType provider_type)
+std::string_view toString(ApiMode api_mode)
 {
-    return provider_type != ProviderType::GCS;
+    using enum ApiMode;
+
+    switch (api_mode)
+    {
+        case AWS:
+            return "AWS";
+        case GCS:
+            return "GCS";
+    }
 }
 
 }
diff --git a/src/IO/S3/ProviderType.h b/src/IO/S3/ProviderType.h
index ff02357e081..eab36da1d30 100644
--- a/src/IO/S3/ProviderType.h
+++ b/src/IO/S3/ProviderType.h
@@ -19,7 +19,13 @@ enum class ProviderType : uint8_t
 
 std::string_view toString(ProviderType provider_type);
 
-bool supportsMultiPartCopy(ProviderType provider_type);
+enum class ApiMode : uint8_t
+{
+    AWS,
+    GCS
+};
+
+std::string_view toString(ApiMode api_mode);
 
 }
 
diff --git a/src/IO/S3/Requests.cpp b/src/IO/S3/Requests.cpp
index a2e7fa72e47..1972b39d2af 100644
--- a/src/IO/S3/Requests.cpp
+++ b/src/IO/S3/Requests.cpp
@@ -10,7 +10,7 @@ namespace DB::S3
 Aws::Http::HeaderValueCollection CopyObjectRequest::GetRequestSpecificHeaders() const
 {
     auto headers = Model::CopyObjectRequest::GetRequestSpecificHeaders();
-    if (provider_type != ProviderType::GCS)
+    if (api_mode != ApiMode::GCS)
         return headers;
 
     /// GCS supports same headers as S3 but with a prefix x-goog instead of x-amz
diff --git a/src/IO/S3/Requests.h b/src/IO/S3/Requests.h
index fb90e7207d0..7b6e911c104 100644
--- a/src/IO/S3/Requests.h
+++ b/src/IO/S3/Requests.h
@@ -62,15 +62,15 @@ public:
         return uri_override;
     }
 
-    void setProviderType(ProviderType provider_type_) const
+    void setApiMode(ApiMode api_mode_) const
     {
-        provider_type = provider_type_;
+        api_mode = api_mode_;
     }
 
 protected:
     mutable std::string region_override;
     mutable std::optional<S3::URI> uri_override;
-    mutable ProviderType provider_type{ProviderType::UNKNOWN};
+    mutable ApiMode api_mode{ApiMode::AWS};
 };
 
 class CopyObjectRequest : public ExtendedRequest<Model::CopyObjectRequest>
diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp
index bf7294ce4aa..b8b3cbbc54e 100644
--- a/src/IO/S3/copyS3File.cpp
+++ b/src/IO/S3/copyS3File.cpp
@@ -595,7 +595,7 @@ namespace
             , src_key(src_key_)
             , offset(src_offset_)
             , size(src_size_)
-            , supports_multipart_copy(S3::supportsMultiPartCopy(client_ptr_->getProviderType()))
+            , supports_multipart_copy(client_ptr_->supportsMultiPartCopy())
         {
         }
 

From 6d3eb75a548b21ff4b3473d8664956d6c7efa31c Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Thu, 4 May 2023 17:35:24 +0200
Subject: [PATCH 144/689] Allow zero objects in ReadBufferFromRemoteFSGather

---
 src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 32 ++++++++-----------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
index 344abcbc95b..b154f5c5a96 100644
--- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
+++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
@@ -31,10 +31,8 @@ ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather(
     , log(&Poco::Logger::get("ReadBufferFromRemoteFSGather"))
     , enable_cache_log(!query_id.empty() && settings.enable_filesystem_cache_log)
 {
-    if (blobs_to_read.empty())
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read zero number of objects");
-
-    current_object = blobs_to_read.front();
+    if (!blobs_to_read.empty())
+        current_object = blobs_to_read.front();
 
     with_cache = settings.remote_fs_cache
         && settings.enable_filesystem_cache
@@ -76,7 +74,9 @@ SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(c
 
 void ReadBufferFromRemoteFSGather::appendFilesystemCacheLog()
 {
-    chassert(!current_object.absolute_path.empty());
+    if (current_object.absolute_path.empty())
+        return;
+
     FilesystemCacheLogElement elem
     {
         .event_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()),
@@ -103,9 +103,7 @@ IAsynchronousReader::Result ReadBufferFromRemoteFSGather::readInto(char * data,
     file_offset_of_buffer_end = offset;
     bytes_to_ignore = ignore;
 
-    assert(!bytes_to_ignore || initialized());
-
-    auto result = nextImpl();
+    const auto result = nextImpl();
 
     if (result)
         return { working_buffer.size(), BufferBase::offset(), nullptr };
@@ -115,6 +113,9 @@ IAsynchronousReader::Result ReadBufferFromRemoteFSGather::readInto(char * data,
 
 void ReadBufferFromRemoteFSGather::initialize()
 {
+    if (blobs_to_read.empty())
+        return;
+
     /// One clickhouse file can be split into multiple files in remote fs.
     auto current_buf_offset = file_offset_of_buffer_end;
     for (size_t i = 0; i < blobs_to_read.size(); ++i)
@@ -146,21 +147,14 @@ bool ReadBufferFromRemoteFSGather::nextImpl()
     if (!current_buf)
         initialize();
 
-    /// If current buffer has remaining data - use it.
-    if (current_buf)
-    {
-        if (readImpl())
-            return true;
-    }
-    else
-    {
+    if (!current_buf)
         return false;
-    }
+
+    if (readImpl())
+        return true;
 
     if (!moveToNextBuffer())
-    {
         return false;
-    }
 
     return readImpl();
 }

From 09a7b9bc1738efc475aef2853555072cab9dd937 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Thu, 4 May 2023 18:16:11 +0200
Subject: [PATCH 145/689] Add test

---
 .../02731_zero_objects_in_metadata.reference  |  3 +++
 .../02731_zero_objects_in_metadata.sh         | 26 +++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 tests/queries/0_stateless/02731_zero_objects_in_metadata.reference
 create mode 100755 tests/queries/0_stateless/02731_zero_objects_in_metadata.sh

diff --git a/tests/queries/0_stateless/02731_zero_objects_in_metadata.reference b/tests/queries/0_stateless/02731_zero_objects_in_metadata.reference
new file mode 100644
index 00000000000..125915f4f65
--- /dev/null
+++ b/tests/queries/0_stateless/02731_zero_objects_in_metadata.reference
@@ -0,0 +1,3 @@
+1	[]
+1	[]
+[]
diff --git a/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh
new file mode 100755
index 00000000000..31f7474089a
--- /dev/null
+++ b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} -n --query "
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (id Int32, empty Array(Int32))
+    ENGINE=MergeTree ORDER BY id
+    SETTINGS min_rows_for_wide_part=0, min_bytes_for_wide_part=0, storage_policy='s3';
+
+INSERT INTO test (id) VALUES (1);
+SELECT * FROM test;
+"
+
+${CLICKHOUSE_CLIENT} -n --query "
+BACKUP TABLE test TO Disk('backups', 'test_s3_backup');
+DROP TABLE test;
+RESTORE TABLE test FROM Disk('backups', 'test_s3_backup');
+" &>/dev/null
+
+${CLICKHOUSE_CLIENT} -n --query "
+SELECT * FROM test;
+SELECT empty FROM test;
+"

From 322b5a4da0ed5a70cf5d4eabb08bcd69e08b2501 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Thu, 4 May 2023 18:59:08 +0200
Subject: [PATCH 146/689] Update ReadBufferFromRemoteFSGather.cpp

---
 src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
index 12aa4920c19..24b3840e3d5 100644
--- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
+++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
@@ -14,11 +14,6 @@
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int LOGICAL_ERROR;
-}
-
 ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather(
     ReadBufferCreator && read_buffer_creator_,
     const StoredObjects & blobs_to_read_,

From 9a0bc891e47d5bf9d91815777c44ed01faa97bf4 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Thu, 4 May 2023 19:39:16 +0200
Subject: [PATCH 147/689] fix can_remove_callback for Temporary parts

---
 src/Storages/MergeTree/IMergeTreeDataPart.cpp |  6 -----
 src/Storages/StorageReplicatedMergeTree.cpp   | 23 +++++++++++++++++++
 src/Storages/StorageReplicatedMergeTree.h     |  2 +-
 3 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
index d3bc3cc70d9..f7ed6c5f278 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
@@ -1684,12 +1684,6 @@ void IMergeTreeDataPart::remove()
             return CanRemoveDescription{.can_remove_anything = true, .files_not_to_remove = {} };
         }
 
-        if (getState() == MergeTreeDataPartState::Temporary)
-        {
-            LOG_TRACE(storage.log, "Part {} in temporary state can be removed without unlocking shared state", name);
-            return CanRemoveDescription{.can_remove_anything = false, .files_not_to_remove = {} };
-        }
-
         auto [can_remove, files_not_to_remove] = canRemovePart();
         if (!can_remove)
             LOG_TRACE(storage.log, "Blobs of part {} cannot be removed", name);
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index f0d0dec3cc3..70ebdf21973 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -8263,6 +8263,29 @@ StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, co
         return std::make_pair(true, NameSet{});
     }
 
+    if (part.getState() == MergeTreeDataPartState::Temporary && part.is_temp)
+    {
+        /// Part {} is in temporary state and has it_temp flag. it means that it is under construction.
+        /// That path hasn't been added to active set, no commit procedure has begun.
+        /// The metadata files is about to delete now.
+        /// However remote data might be shared and has to be unlocked in the keper before removing.
+        /// Actually there is some cases when it is clear without keper:
+        /// When the part has been fetched then remote data has to be  preserved, part doesn't own it.
+        /// When the part has been merged then remote data can to removed, part own it.
+        /// In opposition, when the part has been mutated in generally it hardlinks the files.
+        /// Therefore remote data is shared, it has to be unlocked in the keper.
+
+        std::lock_guard lock(currently_fetching_parts_mutex);
+        if (!currently_fetching_parts.contains(part.name))
+        {
+            LOG_TRACE(log, "Part {} is temporary result of failed fetch."
+                           "Remove it without unlocking shared state preserving shared data.", part.name);
+            return std::make_pair(false, NameSet{});
+        }
+
+        /// The other cases (merge/mutate) is harder to distinguish. Fall back to unlock part in the keper.
+    }
+
     if (has_metadata_in_zookeeper.has_value() && !has_metadata_in_zookeeper)
     {
         if (zookeeper->exists(zookeeper_path))
diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h
index ade4e4f0b4b..77ab976d0fd 100644
--- a/src/Storages/StorageReplicatedMergeTree.h
+++ b/src/Storages/StorageReplicatedMergeTree.h
@@ -703,7 +703,7 @@ private:
 
     /// Required only to avoid races between executeLogEntry and fetchPartition
     std::unordered_set<String> currently_fetching_parts;
-    std::mutex currently_fetching_parts_mutex;
+    mutable std::mutex currently_fetching_parts_mutex;
 
     /// With the quorum being tracked, add a replica to the quorum for the part.
     void updateQuorum(const String & part_name, bool is_parallel);

From 93c1209ea503a4397560f934240eb9cbefa9b766 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Thu, 4 May 2023 19:40:41 +0200
Subject: [PATCH 148/689] fix black style

---
 .../test_merge_tree_s3/s3_mocks/broken_s3.py      | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/tests/integration/test_merge_tree_s3/s3_mocks/broken_s3.py b/tests/integration/test_merge_tree_s3/s3_mocks/broken_s3.py
index 1c2465f6d44..e6def138371 100644
--- a/tests/integration/test_merge_tree_s3/s3_mocks/broken_s3.py
+++ b/tests/integration/test_merge_tree_s3/s3_mocks/broken_s3.py
@@ -29,7 +29,7 @@ class RequestHandler(http.server.BaseHTTPRequestHandler):
     def _ping(self):
         self._ok()
 
-    def _redirect(self):
+    def _read_out(self):
         content_length = int(self.headers.get("Content-Length", 0))
         to_read = content_length
         while to_read > 0:
@@ -40,6 +40,9 @@ class RequestHandler(http.server.BaseHTTPRequestHandler):
             str(self.rfile.read(size))
             to_read -= size
 
+    def _redirect(self):
+        self._read_out()
+
         self.send_response(307)
         url = f"http://{UPSTREAM_HOST}:{UPSTREAM_PORT}{self.path}"
         self.send_header("Location", url)
@@ -47,32 +50,34 @@ class RequestHandler(http.server.BaseHTTPRequestHandler):
         self.wfile.write(b"Redirected")
 
     def _error(self, data):
+        self._read_out()
+
         self.send_response(500)
         self.send_header("Content-Type", "text/xml")
         self.end_headers()
         self.wfile.write(data)
 
-    def _broken_settings(self):
+    def _mock_settings(self):
         parts = urllib.parse.urlsplit(self.path)
         path = [x for x in parts.path.split("/") if x]
         assert path[0] == "mock_settings", path
         if path[1] == "error_at_put":
             params = urllib.parse.parse_qs(parts.query, keep_blank_values=False)
             runtime.error_at_put_when_length_bigger = int(
-                params.get("when_length_bigger", [1024*1024])[0]
+                params.get("when_length_bigger", [1024 * 1024])[0]
             )
             self._ok()
         elif path[1] == "reset":
             runtime.reset()
             self._ok()
         else:
-            self._error("mock_settings, wrong command")
+            self._error("_mock_settings: wrong command")
 
     def do_GET(self):
         if self.path == "/":
             self._ping()
         elif self.path.startswith("/mock_settings"):
-            self._broken_settings()
+            self._mock_settings()
         else:
             self._redirect()
 

From 309032fb6700f251b8a4d29a66a1b157c47bd327 Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Thu, 4 May 2023 13:42:14 -0400
Subject: [PATCH 149/689] Check if grantees are empty

---
 src/Access/UsersConfigAccessStorage.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp
index e25d2422179..a1840b8451c 100644
--- a/src/Access/UsersConfigAccessStorage.cpp
+++ b/src/Access/UsersConfigAccessStorage.cpp
@@ -256,6 +256,9 @@ namespace
                 if (!query.cluster.empty())
                     throw Exception(ErrorCodes::BAD_ARGUMENTS, "Can't grant on cluster using config file");
 
+                if (query.grantees)
+                    throw Exception(ErrorCodes::BAD_ARGUMENTS, "You can't specify grantees in query using config file");
+
                 for (auto & element : query.access_rights_elements)
                 {
                     if (query.is_revoke)

From 4ba7530c236d80743e3c9917f493224cd2b1fc4d Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 5 May 2023 00:21:13 +0200
Subject: [PATCH 150/689] Fix thread MemoryTracker parent after detach

---
 src/Interpreters/ThreadStatusExt.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp
index fd4a6b5e996..5a37d324fc3 100644
--- a/src/Interpreters/ThreadStatusExt.cpp
+++ b/src/Interpreters/ThreadStatusExt.cpp
@@ -223,7 +223,8 @@ void ThreadStatus::detachFromGroup()
     performance_counters.setParent(&ProfileEvents::global_counters);
 
     memory_tracker.reset();
-    memory_tracker.setParent(thread_group->memory_tracker.getParent());
+    /// Extract MemoryTracker out from query and user context
+    memory_tracker.setParent(&total_memory_tracker);
 
     thread_group.reset();
 

From c5184ed3299c4c7a2b27afa07abacb7faf0dd11d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 5 May 2023 01:36:01 +0200
Subject: [PATCH 151/689] Optimize the system.query_log and
 system.query_thread_log tables by applying LowCardinality when appropriate

---
 src/Interpreters/QueryLog.cpp       | 70 ++++++++++++++---------------
 src/Interpreters/QueryThreadLog.cpp | 26 ++++++-----
 2 files changed, 50 insertions(+), 46 deletions(-)

diff --git a/src/Interpreters/QueryLog.cpp b/src/Interpreters/QueryLog.cpp
index 5015f3e4a6e..4746954edf2 100644
--- a/src/Interpreters/QueryLog.cpp
+++ b/src/Interpreters/QueryLog.cpp
@@ -41,6 +41,9 @@ NamesAndTypesList QueryLogElement::getNamesAndTypes()
             {"ExceptionWhileProcessing",    static_cast<Int8>(EXCEPTION_WHILE_PROCESSING)}
         });
 
+    auto low_cardinality_string = std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>());
+    auto array_low_cardinality_string = std::make_shared<DataTypeArray>(low_cardinality_string);
+
     return
     {
         {"type", std::move(query_status_datatype)},
@@ -59,31 +62,26 @@ NamesAndTypesList QueryLogElement::getNamesAndTypes()
         {"result_bytes", std::make_shared<DataTypeUInt64>()},
         {"memory_usage", std::make_shared<DataTypeUInt64>()},
 
-        {"current_database", std::make_shared<DataTypeString>()},
+        {"current_database", low_cardinality_string},
         {"query", std::make_shared<DataTypeString>()},
         {"formatted_query", std::make_shared<DataTypeString>()},
         {"normalized_query_hash", std::make_shared<DataTypeUInt64>()},
-        {"query_kind", std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())},
-        {"databases", std::make_shared<DataTypeArray>(
-            std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()))},
-        {"tables", std::make_shared<DataTypeArray>(
-            std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()))},
-        {"columns", std::make_shared<DataTypeArray>(
-            std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()))},
-        {"projections", std::make_shared<DataTypeArray>(
-            std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()))},
-        {"views", std::make_shared<DataTypeArray>(
-            std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()))},
+        {"query_kind", low_cardinality_string},
+        {"databases", array_low_cardinality_string},
+        {"tables", array_low_cardinality_string},
+        {"columns", array_low_cardinality_string},
+        {"projections", array_low_cardinality_string},
+        {"views", array_low_cardinality_string},
         {"exception_code", std::make_shared<DataTypeInt32>()},
         {"exception", std::make_shared<DataTypeString>()},
         {"stack_trace", std::make_shared<DataTypeString>()},
 
         {"is_initial_query", std::make_shared<DataTypeUInt8>()},
-        {"user", std::make_shared<DataTypeString>()},
+        {"user", low_cardinality_string},
         {"query_id", std::make_shared<DataTypeString>()},
         {"address", DataTypeFactory::instance().get("IPv6")},
         {"port", std::make_shared<DataTypeUInt16>()},
-        {"initial_user", std::make_shared<DataTypeString>()},
+        {"initial_user", low_cardinality_string},
         {"initial_query_id", std::make_shared<DataTypeString>()},
         {"initial_address", DataTypeFactory::instance().get("IPv6")},
         {"initial_port", std::make_shared<DataTypeUInt16>()},
@@ -91,15 +89,15 @@ NamesAndTypesList QueryLogElement::getNamesAndTypes()
         {"initial_query_start_time_microseconds", std::make_shared<DataTypeDateTime64>(6)},
         {"interface", std::make_shared<DataTypeUInt8>()},
         {"is_secure", std::make_shared<DataTypeUInt8>()},
-        {"os_user", std::make_shared<DataTypeString>()},
-        {"client_hostname", std::make_shared<DataTypeString>()},
-        {"client_name", std::make_shared<DataTypeString>()},
+        {"os_user", low_cardinality_string},
+        {"client_hostname", low_cardinality_string},
+        {"client_name", low_cardinality_string},
         {"client_revision", std::make_shared<DataTypeUInt32>()},
         {"client_version_major", std::make_shared<DataTypeUInt32>()},
         {"client_version_minor", std::make_shared<DataTypeUInt32>()},
         {"client_version_patch", std::make_shared<DataTypeUInt32>()},
         {"http_method", std::make_shared<DataTypeUInt8>()},
-        {"http_user_agent", std::make_shared<DataTypeString>()},
+        {"http_user_agent", low_cardinality_string},
         {"http_referer", std::make_shared<DataTypeString>()},
         {"forwarded_for", std::make_shared<DataTypeString>()},
         {"quota_key", std::make_shared<DataTypeString>()},
@@ -110,36 +108,38 @@ NamesAndTypesList QueryLogElement::getNamesAndTypes()
         {"log_comment", std::make_shared<DataTypeString>()},
 
         {"thread_ids", std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>())},
-        {"ProfileEvents", std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(), std::make_shared<DataTypeUInt64>())},
-        {"Settings", std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(), std::make_shared<DataTypeString>())},
+        {"ProfileEvents", std::make_shared<DataTypeMap>(low_cardinality_string, std::make_shared<DataTypeUInt64>())},
+        {"Settings", std::make_shared<DataTypeMap>(low_cardinality_string, low_cardinality_string)},
 
-        {"used_aggregate_functions", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
-        {"used_aggregate_function_combinators", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
-        {"used_database_engines", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
-        {"used_data_type_families", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
-        {"used_dictionaries", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
-        {"used_formats", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
-        {"used_functions", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
-        {"used_storages", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
-        {"used_table_functions", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
+        {"used_aggregate_functions", array_low_cardinality_string},
+        {"used_aggregate_function_combinators", array_low_cardinality_string},
+        {"used_database_engines", array_low_cardinality_string},
+        {"used_data_type_families", array_low_cardinality_string},
+        {"used_dictionaries", array_low_cardinality_string},
+        {"used_formats", array_low_cardinality_string},
+        {"used_functions", array_low_cardinality_string},
+        {"used_storages", array_low_cardinality_string},
+        {"used_table_functions", array_low_cardinality_string},
 
-        {"used_row_policies", std::make_shared<DataTypeArray>(std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()))},
+        {"used_row_policies", array_low_cardinality_string},
 
         {"transaction_id", getTransactionIDDataType()},
 
-        {"AsyncReadCounters", std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(), std::make_shared<DataTypeUInt64>())},
+        {"asynchronous_read_counters", std::make_shared<DataTypeMap>(low_cardinality_string, std::make_shared<DataTypeUInt64>())},
     };
-
 }
 
 NamesAndAliases QueryLogElement::getNamesAndAliases()
 {
+    auto low_cardinality_string = std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>());
+    auto array_low_cardinality_string = std::make_shared<DataTypeArray>(low_cardinality_string);
+
     return
     {
-        {"ProfileEvents.Names", {std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())}, "mapKeys(ProfileEvents)"},
+        {"ProfileEvents.Names", array_low_cardinality_string, "mapKeys(ProfileEvents)"},
         {"ProfileEvents.Values", {std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>())}, "mapValues(ProfileEvents)"},
-        {"Settings.Names", {std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())}, "mapKeys(Settings)" },
-        {"Settings.Values", {std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())}, "mapValues(Settings)"}
+        {"Settings.Names", array_low_cardinality_string, "mapKeys(Settings)" },
+        {"Settings.Values", array_low_cardinality_string, "mapValues(Settings)"}
     };
 }
 
diff --git a/src/Interpreters/QueryThreadLog.cpp b/src/Interpreters/QueryThreadLog.cpp
index 3b28df17999..fe9c8f91d30 100644
--- a/src/Interpreters/QueryThreadLog.cpp
+++ b/src/Interpreters/QueryThreadLog.cpp
@@ -10,6 +10,7 @@
 #include <DataTypes/DataTypeDateTime64.h>
 #include <DataTypes/DataTypeFactory.h>
 #include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeLowCardinality.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <Interpreters/ProfileEventsExt.h>
 #include <Interpreters/QueryLog.h>
@@ -22,7 +23,10 @@ namespace DB
 
 NamesAndTypesList QueryThreadLogElement::getNamesAndTypes()
 {
-    return {
+    auto low_cardinality_string = std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>());
+
+    return
+    {
         {"event_date", std::make_shared<DataTypeDate>()},
         {"event_time", std::make_shared<DataTypeDateTime>()},
         {"event_time_microseconds", std::make_shared<DataTypeDateTime64>(6)},
@@ -37,19 +41,19 @@ NamesAndTypesList QueryThreadLogElement::getNamesAndTypes()
         {"memory_usage", std::make_shared<DataTypeInt64>()},
         {"peak_memory_usage", std::make_shared<DataTypeInt64>()},
 
-        {"thread_name", std::make_shared<DataTypeString>()},
+        {"thread_name", low_cardinality_string},
         {"thread_id", std::make_shared<DataTypeUInt64>()},
         {"master_thread_id", std::make_shared<DataTypeUInt64>()},
-        {"current_database", std::make_shared<DataTypeString>()},
+        {"current_database", low_cardinality_string},
         {"query", std::make_shared<DataTypeString>()},
         {"normalized_query_hash", std::make_shared<DataTypeUInt64>()},
 
         {"is_initial_query", std::make_shared<DataTypeUInt8>()},
-        {"user", std::make_shared<DataTypeString>()},
+        {"user", low_cardinality_string},
         {"query_id", std::make_shared<DataTypeString>()},
         {"address", DataTypeFactory::instance().get("IPv6")},
         {"port", std::make_shared<DataTypeUInt16>()},
-        {"initial_user", std::make_shared<DataTypeString>()},
+        {"initial_user", low_cardinality_string},
         {"initial_query_id", std::make_shared<DataTypeString>()},
         {"initial_address", DataTypeFactory::instance().get("IPv6")},
         {"initial_port", std::make_shared<DataTypeUInt16>()},
@@ -57,15 +61,15 @@ NamesAndTypesList QueryThreadLogElement::getNamesAndTypes()
         {"initial_query_start_time_microseconds", std::make_shared<DataTypeDateTime64>(6)},
         {"interface", std::make_shared<DataTypeUInt8>()},
         {"is_secure", std::make_shared<DataTypeUInt8>()},
-        {"os_user", std::make_shared<DataTypeString>()},
-        {"client_hostname", std::make_shared<DataTypeString>()},
-        {"client_name", std::make_shared<DataTypeString>()},
+        {"os_user", low_cardinality_string},
+        {"client_hostname", low_cardinality_string},
+        {"client_name", low_cardinality_string},
         {"client_revision", std::make_shared<DataTypeUInt32>()},
         {"client_version_major", std::make_shared<DataTypeUInt32>()},
         {"client_version_minor", std::make_shared<DataTypeUInt32>()},
         {"client_version_patch", std::make_shared<DataTypeUInt32>()},
         {"http_method", std::make_shared<DataTypeUInt8>()},
-        {"http_user_agent", std::make_shared<DataTypeString>()},
+        {"http_user_agent", low_cardinality_string},
         {"http_referer", std::make_shared<DataTypeString>()},
         {"forwarded_for", std::make_shared<DataTypeString>()},
         {"quota_key", std::make_shared<DataTypeString>()},
@@ -73,7 +77,7 @@ NamesAndTypesList QueryThreadLogElement::getNamesAndTypes()
 
         {"revision", std::make_shared<DataTypeUInt32>()},
 
-        {"ProfileEvents", std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(), std::make_shared<DataTypeUInt64>())},
+        {"ProfileEvents", std::make_shared<DataTypeMap>(low_cardinality_string, std::make_shared<DataTypeUInt64>())},
     };
 }
 
@@ -81,7 +85,7 @@ NamesAndAliases QueryThreadLogElement::getNamesAndAliases()
 {
     return
     {
-        {"ProfileEvents.Names", {std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())}, "mapKeys(ProfileEvents)"},
+        {"ProfileEvents.Names", {std::make_shared<DataTypeArray>(std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()))}, "mapKeys(ProfileEvents)"},
         {"ProfileEvents.Values", {std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>())}, "mapValues(ProfileEvents)"}
     };
 }

From 5cf3863e7e306de9e6f98fcfa91ef67f036aa2f9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 5 May 2023 03:37:13 +0300
Subject: [PATCH 152/689] Update play.html

---
 programs/server/play.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/server/play.html b/programs/server/play.html
index b36e6c10d8d..193d824d594 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -517,7 +517,7 @@
     let previous_query = '';
 
     const current_url = new URL(window.location);
-    const opened_locally = document.location.href.startsWith("file://");
+    const opened_locally = location.protocol == 'file:';
 
     const server_address = current_url.searchParams.get('url');
     if (server_address) {

From 0fa311ee38dceb793f301f24c818de15e85f63fd Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Wed, 3 May 2023 00:29:29 +0000
Subject: [PATCH 153/689] Make INSERT do more things in parallel to avoid
 getting bottlenecked on one thread

---
 src/Interpreters/InterpreterInsertQuery.cpp   | 110 +++++++++++++-----
 src/Interpreters/InterpreterInsertQuery.h     |  10 +-
 src/QueryPipeline/Chain.cpp                   |   8 ++
 src/QueryPipeline/Chain.h                     |  11 +-
 src/Storages/IStorage.h                       |   2 +
 .../00341_squashing_insert_select2.sql        |   1 +
 6 files changed, 110 insertions(+), 32 deletions(-)

diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index e78a61831a1..b5b159f37d5 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -241,13 +241,17 @@ Chain InterpreterInsertQuery::buildChain(
         running_group = std::make_shared<ThreadGroup>(getContext());
 
     auto sample = getSampleBlock(columns, table, metadata_snapshot);
-    return buildChainImpl(table, metadata_snapshot, sample, thread_status_holder, running_group, elapsed_counter_ms);
+
+    Chain sink = buildSink(table, metadata_snapshot, thread_status_holder, running_group, elapsed_counter_ms);
+    Chain chain = buildPreSinkChain(sink.getInputHeader(), table, metadata_snapshot, sample, thread_status_holder);
+
+    chain.appendChain(std::move(sink));
+    return chain;
 }
 
-Chain InterpreterInsertQuery::buildChainImpl(
+Chain InterpreterInsertQuery::buildSink(
     const StoragePtr & table,
     const StorageMetadataPtr & metadata_snapshot,
-    const Block & query_sample_block,
     ThreadStatusesHolderPtr thread_status_holder,
     ThreadGroupPtr running_group,
     std::atomic_uint64_t * elapsed_counter_ms)
@@ -258,14 +262,7 @@ Chain InterpreterInsertQuery::buildChainImpl(
         thread_status = nullptr;
 
     auto context_ptr = getContext();
-    const ASTInsertQuery * query = nullptr;
-    if (query_ptr)
-        query = query_ptr->as<ASTInsertQuery>();
 
-    const Settings & settings = context_ptr->getSettingsRef();
-    bool null_as_default = query && query->select && context_ptr->getSettingsRef().insert_null_as_default;
-
-    /// We create a pipeline of several streams, into which we will write data.
     Chain out;
 
     /// Keep a reference to the context to make sure it stays alive until the chain is executed and destroyed
@@ -286,16 +283,48 @@ Chain InterpreterInsertQuery::buildChainImpl(
             thread_status_holder, running_group, elapsed_counter_ms);
     }
 
+    return out;
+}
+
+Chain InterpreterInsertQuery::buildPreSinkChain(
+    const Block & subsequent_header,
+    const StoragePtr & table,
+    const StorageMetadataPtr & metadata_snapshot,
+    const Block & query_sample_block,
+    ThreadStatusesHolderPtr thread_status_holder)
+{
+    ThreadStatus * thread_status = current_thread;
+
+    if (!thread_status_holder)
+        thread_status = nullptr;
+
+    auto context_ptr = getContext();
+
+    const ASTInsertQuery * query = nullptr;
+    if (query_ptr)
+        query = query_ptr->as<ASTInsertQuery>();
+
+    const Settings & settings = context_ptr->getSettingsRef();
+    bool null_as_default = query && query->select && context_ptr->getSettingsRef().insert_null_as_default;
+
+    /// We create a pipeline of several streams, into which we will write data.
+    Chain out;
+
+    auto input_header = [&]() -> const Block &
+    {
+        return out.empty() ? subsequent_header : out.getInputHeader();
+    };
+
     /// Note that we wrap transforms one on top of another, so we write them in reverse of data processing order.
 
     /// Checking constraints. It must be done after calculation of all defaults, so we can check them on calculated columns.
     if (const auto & constraints = metadata_snapshot->getConstraints(); !constraints.empty())
         out.addSource(std::make_shared<CheckConstraintsTransform>(
-            table->getStorageID(), out.getInputHeader(), metadata_snapshot->getConstraints(), context_ptr));
+            table->getStorageID(), input_header(), metadata_snapshot->getConstraints(), context_ptr));
 
     auto adding_missing_defaults_dag = addMissingDefaults(
         query_sample_block,
-        out.getInputHeader().getNamesAndTypesList(),
+        input_header().getNamesAndTypesList(),
         metadata_snapshot->getColumns(),
         context_ptr,
         null_as_default);
@@ -316,12 +345,12 @@ Chain InterpreterInsertQuery::buildChainImpl(
         bool table_prefers_large_blocks = table->prefersLargeBlocks();
 
         out.addSource(std::make_shared<SquashingChunksTransform>(
-            out.getInputHeader(),
+            input_header(),
             table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
             table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL));
     }
 
-    auto counting = std::make_shared<CountingTransform>(out.getInputHeader(), thread_status, getContext()->getQuota());
+    auto counting = std::make_shared<CountingTransform>(input_header(), thread_status, getContext()->getQuota());
     counting->setProcessListElement(context_ptr->getProcessListElement());
     counting->setProgressCallback(context_ptr->getProgressCallback());
     out.addSource(std::move(counting));
@@ -362,10 +391,20 @@ BlockIO InterpreterInsertQuery::execute()
         // Distributed INSERT SELECT
         distributed_pipeline = table->distributedWrite(query, getContext());
 
-    std::vector<Chain> out_chains;
+    std::vector<Chain> presink_chains;
+    std::vector<Chain> sink_chains;
     if (!distributed_pipeline || query.watch)
     {
-        size_t out_streams_size = 1;
+        /// Number of streams works like this:
+        ///  * For the SELECT, use `max_threads`, or `max_insert_threads`, or whatever
+        ///    InterpreterSelectQuery ends up with.
+        ///  * Use `max_insert_threads` streams for various insert-preparation steps, e.g.
+        ///    materializing and squashing (too slow to do in one thread). That's `presink_chains`.
+        ///  * If the table supports parallel inserts, use the same streams for writing to IStorage.
+        ///    Otherwise ResizeProcessor them down to 1 stream.
+        ///  * If it's not an INSERT SELECT, forget all that and use one stream.
+        size_t pre_streams_size = 1;
+        size_t sink_streams_size = 1;
 
         if (query.select)
         {
@@ -441,10 +480,14 @@ BlockIO InterpreterInsertQuery::execute()
 
             pipeline.dropTotalsAndExtremes();
 
-            if (table->supportsParallelInsert() && settings.max_insert_threads > 1)
-                out_streams_size = std::min(static_cast<size_t>(settings.max_insert_threads), pipeline.getNumStreams());
+            if (settings.max_insert_threads > 1)
+            {
+                pre_streams_size = std::min(static_cast<size_t>(settings.max_insert_threads), pipeline.getNumStreams());
+                if (table->supportsParallelInsert())
+                    sink_streams_size = pre_streams_size;
+            }
 
-            pipeline.resize(out_streams_size);
+            pipeline.resize(pre_streams_size);
 
             /// Allow to insert Nullable into non-Nullable columns, NULL values will be added as defaults values.
             if (getContext()->getSettingsRef().insert_null_as_default)
@@ -476,13 +519,17 @@ BlockIO InterpreterInsertQuery::execute()
             running_group = current_thread->getThreadGroup();
         if (!running_group)
             running_group = std::make_shared<ThreadGroup>(getContext());
-        for (size_t i = 0; i < out_streams_size; ++i)
+        for (size_t i = 0; i < sink_streams_size; ++i)
         {
-            auto out = buildChainImpl(table, metadata_snapshot, query_sample_block,
-                /* thread_status_holder= */ nullptr,
-                running_group,
-                /* elapsed_counter_ms= */ nullptr);
-            out_chains.emplace_back(std::move(out));
+            auto out = buildSink(table, metadata_snapshot, /* thread_status_holder= */ nullptr,
+                running_group, /* elapsed_counter_ms= */ nullptr);
+            sink_chains.emplace_back(std::move(out));
+        }
+        for (size_t i = 0; i < pre_streams_size; ++i)
+        {
+            auto out = buildPreSinkChain(sink_chains[0].getInputHeader(), table, metadata_snapshot,
+                query_sample_block, /* thread_status_holder= */ nullptr);
+            presink_chains.emplace_back(std::move(out));
         }
     }
 
@@ -495,7 +542,7 @@ BlockIO InterpreterInsertQuery::execute()
     }
     else if (query.select || query.watch)
     {
-        const auto & header = out_chains.at(0).getInputHeader();
+        const auto & header = presink_chains.at(0).getInputHeader();
         auto actions_dag = ActionsDAG::makeConvertingActions(
                 pipeline.getHeader().getColumnsWithTypeAndName(),
                 header.getColumnsWithTypeAndName(),
@@ -516,10 +563,14 @@ BlockIO InterpreterInsertQuery::execute()
 
         size_t num_select_threads = pipeline.getNumThreads();
 
-        for (auto & chain : out_chains)
+        for (auto & chain : presink_chains)
+            resources = chain.detachResources();
+        for (auto & chain : sink_chains)
             resources = chain.detachResources();
 
-        pipeline.addChains(std::move(out_chains));
+        pipeline.addChains(std::move(presink_chains));
+        pipeline.resize(sink_chains.size());
+        pipeline.addChains(std::move(sink_chains));
 
         if (!settings.parallel_view_processing)
         {
@@ -552,7 +603,8 @@ BlockIO InterpreterInsertQuery::execute()
     }
     else
     {
-        res.pipeline = QueryPipeline(std::move(out_chains.at(0)));
+        presink_chains.at(0).appendChain(std::move(sink_chains.at(0)));
+        res.pipeline = QueryPipeline(std::move(presink_chains[0]));
         res.pipeline.setNumThreads(std::min<size_t>(res.pipeline.getNumThreads(), settings.max_threads));
 
         if (query.hasInlinedData() && !async_insert)
diff --git a/src/Interpreters/InterpreterInsertQuery.h b/src/Interpreters/InterpreterInsertQuery.h
index 5bff472270d..b9a146e5338 100644
--- a/src/Interpreters/InterpreterInsertQuery.h
+++ b/src/Interpreters/InterpreterInsertQuery.h
@@ -66,13 +66,19 @@ private:
 
     std::vector<std::unique_ptr<ReadBuffer>> owned_buffers;
 
-    Chain buildChainImpl(
+    Chain buildSink(
         const StoragePtr & table,
         const StorageMetadataPtr & metadata_snapshot,
-        const Block & query_sample_block,
         ThreadStatusesHolderPtr thread_status_holder,
         ThreadGroupPtr running_group,
         std::atomic_uint64_t * elapsed_counter_ms);
+
+    Chain buildPreSinkChain(
+        const Block & subsequent_header,
+        const StoragePtr & table,
+        const StorageMetadataPtr & metadata_snapshot,
+        const Block & query_sample_block,
+        ThreadStatusesHolderPtr thread_status_holder);
 };
 
 
diff --git a/src/QueryPipeline/Chain.cpp b/src/QueryPipeline/Chain.cpp
index 6122517432a..eaa36071542 100644
--- a/src/QueryPipeline/Chain.cpp
+++ b/src/QueryPipeline/Chain.cpp
@@ -99,6 +99,14 @@ void Chain::addSink(ProcessorPtr processor)
     processors.emplace_back(std::move(processor));
 }
 
+void Chain::appendChain(Chain chain)
+{
+    connect(getOutputPort(), chain.getInputPort());
+    processors.splice(processors.end(), std::move(chain.processors));
+    attachResources(chain.detachResources());
+    num_threads += chain.num_threads;
+}
+
 IProcessor & Chain::getSource()
 {
     checkInitialized(processors);
diff --git a/src/QueryPipeline/Chain.h b/src/QueryPipeline/Chain.h
index d6139281990..322e49d0d49 100644
--- a/src/QueryPipeline/Chain.h
+++ b/src/QueryPipeline/Chain.h
@@ -7,6 +7,10 @@
 namespace DB
 {
 
+/// Has one unconnected input port and one unconnected output port.
+/// There may be other ports on the processors, but they must all be connected.
+/// The unconnected input must be on the first processor, output - on the last.
+/// The processors don't necessarily form an actual chain.
 class Chain
 {
 public:
@@ -27,6 +31,7 @@ public:
 
     void addSource(ProcessorPtr processor);
     void addSink(ProcessorPtr processor);
+    void appendChain(Chain chain);
 
     IProcessor & getSource();
     IProcessor & getSink();
@@ -44,7 +49,11 @@ public:
     void addStorageHolder(StoragePtr storage) { holder.storage_holders.emplace_back(std::move(storage)); }
     void addInterpreterContext(ContextPtr context) { holder.interpreter_context.emplace_back(std::move(context)); }
 
-    void attachResources(QueryPlanResourceHolder holder_) { holder = std::move(holder_); }
+    void attachResources(QueryPlanResourceHolder holder_)
+    {
+        /// This operator "=" actually merges holder_ into holder, doesn't replace.
+        holder = std::move(holder_);
+    }
     QueryPlanResourceHolder detachResources() { return std::move(holder); }
 
     void reset();
diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h
index 5743d903630..43f993d8f7c 100644
--- a/src/Storages/IStorage.h
+++ b/src/Storages/IStorage.h
@@ -146,6 +146,8 @@ public:
     virtual bool supportsReplication() const { return false; }
 
     /// Returns true if the storage supports parallel insert.
+    /// If false, each INSERT query will call write() only once.
+    /// Different INSERT queries may write in parallel regardless of this value.
     virtual bool supportsParallelInsert() const { return false; }
 
     /// Returns true if the storage supports deduplication of inserted data blocks.
diff --git a/tests/queries/0_stateless/00341_squashing_insert_select2.sql b/tests/queries/0_stateless/00341_squashing_insert_select2.sql
index 3eb5a2682e0..e973a146d5b 100644
--- a/tests/queries/0_stateless/00341_squashing_insert_select2.sql
+++ b/tests/queries/0_stateless/00341_squashing_insert_select2.sql
@@ -3,6 +3,7 @@ CREATE TABLE numbers_squashed (number UInt8) ENGINE = StripeLog;
 
 SET min_insert_block_size_rows = 100;
 SET min_insert_block_size_bytes = 0;
+SET max_insert_threads = 1;
 SET max_threads = 1;
 
 INSERT INTO numbers_squashed

From 3bd1489f18d9812053a9c1b67601bb23188a3cba Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Fri, 5 May 2023 04:18:46 +0000
Subject: [PATCH 154/689] Propagate input_format_parquet_preserve_order to
 parallelizeOutputAfterReading()

---
 src/Formats/FormatFactory.cpp              | 8 ++++++++
 src/Formats/FormatFactory.h                | 2 ++
 src/Storages/IStorage.cpp                  | 2 +-
 src/Storages/IStorage.h                    | 2 +-
 src/Storages/StorageDictionary.h           | 2 +-
 src/Storages/StorageFile.cpp               | 5 +++++
 src/Storages/StorageFile.h                 | 2 ++
 src/Storages/StorageNull.h                 | 2 +-
 src/Storages/StorageS3.cpp                 | 5 +++++
 src/Storages/StorageS3.h                   | 4 +++-
 src/Storages/StorageURL.cpp                | 5 +++++
 src/Storages/StorageURL.h                  | 2 ++
 src/Storages/System/StorageSystemNumbers.h | 2 +-
 src/Storages/System/StorageSystemOne.h     | 2 +-
 src/Storages/System/StorageSystemZeros.h   | 2 +-
 15 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
index c8fc6c5e702..082ccb85970 100644
--- a/src/Formats/FormatFactory.cpp
+++ b/src/Formats/FormatFactory.cpp
@@ -809,6 +809,14 @@ bool FormatFactory::checkIfOutputFormatPrefersLargeBlocks(const String & name) c
     return target.prefers_large_blocks;
 }
 
+bool FormatFactory::checkParallelizeOutputAfterReading(const String & name, ContextPtr context) const
+{
+    if (name == "Parquet" && context->getSettingsRef().input_format_parquet_preserve_order)
+        return false;
+
+    return true;
+}
+
 void FormatFactory::checkFormatName(const String & name) const
 {
     auto it = dict.find(name);
diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h
index edf20a92b00..677e34845d8 100644
--- a/src/Formats/FormatFactory.h
+++ b/src/Formats/FormatFactory.h
@@ -250,6 +250,8 @@ public:
     bool checkIfFormatHasAnySchemaReader(const String & name) const;
     bool checkIfOutputFormatPrefersLargeBlocks(const String & name) const;
 
+    bool checkParallelizeOutputAfterReading(const String & name, ContextPtr context) const;
+
     void registerAdditionalInfoForSchemaCacheGetter(const String & name, AdditionalInfoForSchemaCacheGetter additional_info_for_schema_cache_getter);
     String getAdditionalInfoForSchemaCache(const String & name, ContextPtr context, const std::optional<FormatSettings> & format_settings_ = std::nullopt);
 
diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp
index 98990b9e37b..8cf708acd8b 100644
--- a/src/Storages/IStorage.cpp
+++ b/src/Storages/IStorage.cpp
@@ -137,7 +137,7 @@ void IStorage::read(
     /// parallelize processing if not yet
     const size_t output_ports = pipe.numOutputPorts();
     const bool parallelize_output = context->getSettingsRef().parallelize_output_from_storages;
-    if (parallelize_output && parallelizeOutputAfterReading() && output_ports > 0 && output_ports < num_streams)
+    if (parallelize_output && parallelizeOutputAfterReading(context) && output_ports > 0 && output_ports < num_streams)
         pipe.resize(num_streams);
 
     readFromPipe(query_plan, std::move(pipe), column_names, storage_snapshot, query_info, context, getName());
diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h
index 5743d903630..0070791d5de 100644
--- a/src/Storages/IStorage.h
+++ b/src/Storages/IStorage.h
@@ -377,7 +377,7 @@ private:
     /// This is enabled by default, but in some cases shouldn't be done.
     /// For example, when you read from system.numbers instead of system.numbers_mt,
     /// you still expect the data to be processed sequentially.
-    virtual bool parallelizeOutputAfterReading() const { return true; }
+    virtual bool parallelizeOutputAfterReading(ContextPtr) const { return true; }
 
 public:
     /// Other version of read which adds reading step to query plan.
diff --git a/src/Storages/StorageDictionary.h b/src/Storages/StorageDictionary.h
index 02efa609337..48230dcfa9f 100644
--- a/src/Storages/StorageDictionary.h
+++ b/src/Storages/StorageDictionary.h
@@ -76,7 +76,7 @@ public:
 
     /// FIXME: processing after reading from dictionaries are not parallelized due to some bug:
     /// count() can return wrong result, see test_dictionaries_redis/test_long.py::test_redis_dict_long
-    bool parallelizeOutputAfterReading() const override { return false; }
+    bool parallelizeOutputAfterReading(ContextPtr) const override { return false; }
 
     std::shared_ptr<const IDictionary> getDictionary() const;
 
diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index 16769d72e4c..26d4aee8cfe 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -412,6 +412,11 @@ bool StorageFile::prefersLargeBlocks() const
     return FormatFactory::instance().checkIfOutputFormatPrefersLargeBlocks(format_name);
 }
 
+bool StorageFile::parallelizeOutputAfterReading(ContextPtr context) const
+{
+    return FormatFactory::instance().checkParallelizeOutputAfterReading(format_name, context);
+}
+
 StorageFile::StorageFile(int table_fd_, CommonArguments args)
     : StorageFile(args)
 {
diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h
index 2c901ce1573..53ce7eeaaf6 100644
--- a/src/Storages/StorageFile.h
+++ b/src/Storages/StorageFile.h
@@ -75,6 +75,8 @@ public:
 
     bool prefersLargeBlocks() const override;
 
+    bool parallelizeOutputAfterReading(ContextPtr context) const override;
+
     bool supportsPartitionBy() const override { return true; }
 
     ColumnsDescription getTableStructureFromFileDescriptor(ContextPtr context);
diff --git a/src/Storages/StorageNull.h b/src/Storages/StorageNull.h
index 24ddb311b65..d35c6a0b8b5 100644
--- a/src/Storages/StorageNull.h
+++ b/src/Storages/StorageNull.h
@@ -42,7 +42,7 @@ public:
             std::make_shared<NullSource>(storage_snapshot->getSampleBlockForColumns(column_names)));
     }
 
-    bool parallelizeOutputAfterReading() const override { return false; }
+    bool parallelizeOutputAfterReading(ContextPtr) const override { return false; }
 
     bool supportsParallelInsert() const override { return true; }
 
diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index 8add005f7f5..d1b168d3f7d 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -1008,6 +1008,11 @@ bool StorageS3::prefersLargeBlocks() const
     return FormatFactory::instance().checkIfOutputFormatPrefersLargeBlocks(configuration.format);
 }
 
+bool StorageS3::parallelizeOutputAfterReading(ContextPtr context) const
+{
+    return FormatFactory::instance().checkParallelizeOutputAfterReading(configuration.format, context);
+}
+
 Pipe StorageS3::read(
     const Names & column_names,
     const StorageSnapshotPtr & storage_snapshot,
diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h
index 3148206a743..c3b862f6bbd 100644
--- a/src/Storages/StorageS3.h
+++ b/src/Storages/StorageS3.h
@@ -238,7 +238,7 @@ private:
  * It sends HTTP GET to server when select is called and
  * HTTP PUT when insert is called.
  */
-class StorageS3 : public IStorage, WithContext
+class StorageS3 : public IStorage
 {
 public:
     struct Configuration : public StatelessTableEngineConfiguration
@@ -366,6 +366,8 @@ private:
 
     bool prefersLargeBlocks() const override;
 
+    bool parallelizeOutputAfterReading(ContextPtr context) const override;
+
     static std::optional<ColumnsDescription> tryGetColumnsFromCache(
         const KeysWithInfo::const_iterator & begin,
         const KeysWithInfo::const_iterator & end,
diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp
index 59543b3b4ef..b76d6e7e382 100644
--- a/src/Storages/StorageURL.cpp
+++ b/src/Storages/StorageURL.cpp
@@ -600,6 +600,11 @@ bool IStorageURLBase::prefersLargeBlocks() const
     return FormatFactory::instance().checkIfOutputFormatPrefersLargeBlocks(format_name);
 }
 
+bool IStorageURLBase::parallelizeOutputAfterReading(ContextPtr context) const
+{
+    return FormatFactory::instance().checkParallelizeOutputAfterReading(format_name, context);
+}
+
 Pipe IStorageURLBase::read(
     const Names & column_names,
     const StorageSnapshotPtr & storage_snapshot,
diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h
index 65bf86da594..a21a5c59298 100644
--- a/src/Storages/StorageURL.h
+++ b/src/Storages/StorageURL.h
@@ -102,6 +102,8 @@ protected:
 
     bool prefersLargeBlocks() const override;
 
+    bool parallelizeOutputAfterReading(ContextPtr context) const override;
+
 private:
     virtual Block getHeaderBlock(const Names & column_names, const StorageSnapshotPtr & storage_snapshot) const = 0;
 
diff --git a/src/Storages/System/StorageSystemNumbers.h b/src/Storages/System/StorageSystemNumbers.h
index f7277f147f9..80590718d03 100644
--- a/src/Storages/System/StorageSystemNumbers.h
+++ b/src/Storages/System/StorageSystemNumbers.h
@@ -40,7 +40,7 @@ public:
         size_t max_block_size,
         size_t num_streams) override;
 
-    bool parallelizeOutputAfterReading() const override { return false; }
+    bool parallelizeOutputAfterReading(ContextPtr) const override { return false; }
 
     bool hasEvenlyDistributedRead() const override { return true; }
     bool isSystemStorage() const override { return true; }
diff --git a/src/Storages/System/StorageSystemOne.h b/src/Storages/System/StorageSystemOne.h
index eaf44b635cc..e3b6e1d5f78 100644
--- a/src/Storages/System/StorageSystemOne.h
+++ b/src/Storages/System/StorageSystemOne.h
@@ -30,7 +30,7 @@ public:
         size_t max_block_size,
         size_t num_streams) override;
 
-    bool parallelizeOutputAfterReading() const override { return false; }
+    bool parallelizeOutputAfterReading(ContextPtr) const override { return false; }
 
     bool isSystemStorage() const override { return true; }
 
diff --git a/src/Storages/System/StorageSystemZeros.h b/src/Storages/System/StorageSystemZeros.h
index 936fb772453..84b4b752aae 100644
--- a/src/Storages/System/StorageSystemZeros.h
+++ b/src/Storages/System/StorageSystemZeros.h
@@ -31,7 +31,7 @@ public:
         size_t max_block_size,
         size_t num_streams) override;
 
-    bool parallelizeOutputAfterReading() const override { return false; }
+    bool parallelizeOutputAfterReading(ContextPtr) const override { return false; }
 
     bool hasEvenlyDistributedRead() const override { return true; }
     bool isSystemStorage() const override { return true; }

From 90b0de5677ef0d4e08bebe5507dfe193695d5dae Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 5 May 2023 06:36:53 +0200
Subject: [PATCH 155/689] Make Pretty prettier

---
 src/Core/Settings.h                           |  1 +
 src/Formats/FormatFactory.cpp                 |  1 +
 src/Formats/FormatSettings.h                  |  1 +
 .../Formats/Impl/PrettyBlockOutputFormat.cpp  | 26 ++++++++++++-------
 .../Formats/Impl/PrettyBlockOutputFormat.h    | 10 +++----
 .../Impl/PrettySpaceBlockOutputFormat.cpp     |  2 +-
 ...2731_pretty_time_based_squashing.reference | 14 ++++++++++
 .../02731_pretty_time_based_squashing.sql     |  2 ++
 8 files changed, 40 insertions(+), 17 deletions(-)
 create mode 100644 tests/queries/0_stateless/02731_pretty_time_based_squashing.reference
 create mode 100644 tests/queries/0_stateless/02731_pretty_time_based_squashing.sql

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index c7c47b432cd..30781f3c2e8 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -902,6 +902,7 @@ class IColumn;
     M(UInt64, output_format_pretty_max_value_width, 10000, "Maximum width of value to display in Pretty formats. If greater - it will be cut.", 0) \
     M(Bool, output_format_pretty_color, true, "Use ANSI escape sequences to paint colors in Pretty formats", 0) \
     M(String, output_format_pretty_grid_charset, "UTF-8", "Charset for printing grid borders. Available charsets: ASCII, UTF-8 (default one).", 0) \
+    M(Milliseconds, output_format_pretty_squash_ms, 100, "Squash blocks in Pretty formats if the time passed after the previous block is not greater than the specified threshold in milliseconds. This avoids printing miltiple small blocks.", 0) \
     M(UInt64, output_format_parquet_row_group_size, 1000000, "Target row group size in rows.", 0) \
     M(UInt64, output_format_parquet_row_group_size_bytes, 512 * 1024 * 1024, "Target row group size in bytes, before compression.", 0) \
     M(Bool, output_format_parquet_string_as_string, false, "Use Parquet String type instead of Binary for String columns.", 0) \
diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
index c8fc6c5e702..574f78cd06a 100644
--- a/src/Formats/FormatFactory.cpp
+++ b/src/Formats/FormatFactory.cpp
@@ -128,6 +128,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings)
     format_settings.pretty.max_rows = settings.output_format_pretty_max_rows;
     format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width;
     format_settings.pretty.output_format_pretty_row_numbers = settings.output_format_pretty_row_numbers;
+    format_settings.pretty.squash_milliseconds = static_cast<UInt64>(settings.output_format_pretty_squash_ms);
     format_settings.protobuf.input_flatten_google_wrappers = settings.input_format_protobuf_flatten_google_wrappers;
     format_settings.protobuf.output_nullables_with_google_wrappers = settings.output_format_protobuf_nullables_with_google_wrappers;
     format_settings.protobuf.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference;
diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h
index b9a55ffaa83..3a2e818d540 100644
--- a/src/Formats/FormatSettings.h
+++ b/src/Formats/FormatSettings.h
@@ -227,6 +227,7 @@ struct FormatSettings
         UInt64 max_rows = 10000;
         UInt64 max_column_pad_width = 250;
         UInt64 max_value_width = 10000;
+        UInt64 squash_milliseconds = 100;
         bool color = true;
 
         bool output_format_pretty_row_numbers = false;
diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp
index 14648e68f94..bfcc4c101d9 100644
--- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp
+++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp
@@ -137,21 +137,25 @@ void PrettyBlockOutputFormat::write(Chunk chunk, PortKind port_kind)
         total_rows += chunk.getNumRows();
         return;
     }
-    if (mono_block)
+
+    if (mono_block
+        || (format_settings.pretty.squash_milliseconds
+            && time_after_previous_chunk.elapsedMilliseconds() <= format_settings.pretty.squash_milliseconds))
     {
         if (port_kind == PortKind::Main)
         {
-            if (mono_chunk)
-                mono_chunk.append(chunk);
+            if (squashed_chunk)
+                squashed_chunk.append(chunk);
             else
-                mono_chunk = std::move(chunk);
+                squashed_chunk = std::move(chunk);
             return;
         }
 
         /// Should be written from writeSuffix()
-        assert(!mono_chunk);
+        assert(!squashed_chunk);
     }
 
+    writeSquashedChunkIfNeeded();
     writeChunk(chunk, port_kind);
 }
 
@@ -389,18 +393,20 @@ void PrettyBlockOutputFormat::consumeExtremes(Chunk chunk)
 }
 
 
-void PrettyBlockOutputFormat::writeMonoChunkIfNeeded()
+void PrettyBlockOutputFormat::writeSquashedChunkIfNeeded()
 {
-    if (mono_chunk)
+    if (squashed_chunk)
     {
-        writeChunk(mono_chunk, PortKind::Main);
-        mono_chunk.clear();
+        writeChunk(squashed_chunk, PortKind::Main);
+        squashed_chunk.clear();
+        if (format_settings.pretty.squash_milliseconds)
+            time_after_previous_chunk.restart();
     }
 }
 
 void PrettyBlockOutputFormat::writeSuffix()
 {
-    writeMonoChunkIfNeeded();
+    writeSquashedChunkIfNeeded();
 
     if (total_rows >= format_settings.pretty.max_rows)
     {
diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h
index dfb23ac63f9..621f1387196 100644
--- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h
+++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h
@@ -39,7 +39,7 @@ protected:
 
     void write(Chunk chunk, PortKind port_kind);
     virtual void writeChunk(const Chunk & chunk, PortKind port_kind);
-    void writeMonoChunkIfNeeded();
+    void writeSquashedChunkIfNeeded();
     void writeSuffix() override;
 
     void onRowsReadBeforeUpdate() override { total_rows = getRowsReadBefore(); }
@@ -59,11 +59,11 @@ protected:
 
 private:
     bool mono_block;
-    /// For mono_block == true only
-    Chunk mono_chunk;
+    Chunk squashed_chunk;
+    Stopwatch time_after_previous_chunk; /// For squashing.
 };
 
-template <class OutputFormat>
+template <typename OutputFormat>
 void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, const String & base_name)
 {
     auto creator = [&](FormatFactory & fact, const String & name, bool no_escapes, bool mono_block)
@@ -81,8 +81,6 @@ void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, cons
             }
             return std::make_shared<OutputFormat>(buf, sample, format_settings, mono_block);
         });
-        if (!mono_block)
-            factory.markOutputFormatSupportsParallelFormatting(name);
     };
     creator(factory, base_name, false, false);
     creator(factory, base_name + "NoEscapes", true, false);
diff --git a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp
index 0fb1a413a6c..e54d88a11b2 100644
--- a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp
+++ b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp
@@ -100,7 +100,7 @@ void PrettySpaceBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port
 
 void PrettySpaceBlockOutputFormat::writeSuffix()
 {
-    writeMonoChunkIfNeeded();
+    writeSquashedChunkIfNeeded();
 
     if (total_rows >= format_settings.pretty.max_rows)
     {
diff --git a/tests/queries/0_stateless/02731_pretty_time_based_squashing.reference b/tests/queries/0_stateless/02731_pretty_time_based_squashing.reference
new file mode 100644
index 00000000000..653faf33a46
--- /dev/null
+++ b/tests/queries/0_stateless/02731_pretty_time_based_squashing.reference
@@ -0,0 +1,14 @@
+┌─number─┐
+│      0 │
+└────────┘
+┌─number─┐
+│      1 │
+└────────┘
+┌─number─┐
+│      2 │
+└────────┘
+┌─number─┐
+│      0 │
+│      1 │
+│      2 │
+└────────┘
diff --git a/tests/queries/0_stateless/02731_pretty_time_based_squashing.sql b/tests/queries/0_stateless/02731_pretty_time_based_squashing.sql
new file mode 100644
index 00000000000..e5656c8e1ca
--- /dev/null
+++ b/tests/queries/0_stateless/02731_pretty_time_based_squashing.sql
@@ -0,0 +1,2 @@
+SELECT * FROM numbers(3) FORMAT PrettyCompact SETTINGS max_block_size = 1, max_threads = 1, output_format_pretty_squash_ms = 0;
+SELECT * FROM numbers(3) FORMAT PrettyCompact SETTINGS max_block_size = 1, max_threads = 1, output_format_pretty_squash_ms = 600_000;

From efa66521dea78fb3014ec4577cd21fe7222c88ad Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 5 May 2023 06:40:12 +0200
Subject: [PATCH 156/689] Allow to run in Linux without PRCTL support

---
 src/Common/TimerDescriptor.cpp | 1 -
 src/Common/setThreadName.cpp   | 6 ++++--
 src/Daemon/BaseDaemon.cpp      | 1 -
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp
index e0327bc1fc6..1f07f548d85 100644
--- a/src/Common/TimerDescriptor.cpp
+++ b/src/Common/TimerDescriptor.cpp
@@ -1,7 +1,6 @@
 #if defined(OS_LINUX)
 #include <Common/TimerDescriptor.h>
 #include <Common/Exception.h>
-#include <base/defines.h>
 
 #include <sys/timerfd.h>
 #include <fcntl.h>
diff --git a/src/Common/setThreadName.cpp b/src/Common/setThreadName.cpp
index 0e5fb1d72c6..65c4b5c6523 100644
--- a/src/Common/setThreadName.cpp
+++ b/src/Common/setThreadName.cpp
@@ -43,7 +43,8 @@ void setThreadName(const char * name)
 #else
     if (0 != prctl(PR_SET_NAME, name, 0, 0, 0))
 #endif
-        DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME, ...)", DB::ErrorCodes::PTHREAD_ERROR);
+        if (errno != ENOSYS)    /// It's ok if the syscall is unsupported in some environments.
+            DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME, ...)", DB::ErrorCodes::PTHREAD_ERROR);
 
     memcpy(thread_name, name, std::min<size_t>(1 + strlen(name), THREAD_NAME_SIZE - 1));
 }
@@ -62,7 +63,8 @@ const char * getThreadName()
 //        throw DB::Exception(DB::ErrorCodes::PTHREAD_ERROR, "Cannot get thread name with pthread_get_name_np()");
 #else
     if (0 != prctl(PR_GET_NAME, thread_name, 0, 0, 0))
-        DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)", DB::ErrorCodes::PTHREAD_ERROR);
+        if (errno != ENOSYS)    /// It's ok if the syscall is unsupported in some environments.
+            DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)", DB::ErrorCodes::PTHREAD_ERROR);
 #endif
 
     return thread_name;
diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp
index 3d6d9792090..077fa55e8d3 100644
--- a/src/Daemon/BaseDaemon.cpp
+++ b/src/Daemon/BaseDaemon.cpp
@@ -2,7 +2,6 @@
 
 #include <Daemon/BaseDaemon.h>
 #include <Daemon/SentryWriter.h>
-#include <Parsers/toOneLineQuery.h>
 #include <base/errnoToString.h>
 #include <base/defines.h>
 

From 14f5e785d621c2c9a63d1d852e9171605d76eb02 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 5 May 2023 06:42:29 +0200
Subject: [PATCH 157/689] Fix test

---
 tests/integration/test_system_metrics/test.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_system_metrics/test.py b/tests/integration/test_system_metrics/test.py
index 439e8b66db1..d5bf83f6bf2 100644
--- a/tests/integration/test_system_metrics/test.py
+++ b/tests/integration/test_system_metrics/test.py
@@ -122,11 +122,12 @@ def test_metrics_storage_buffer_size(start_cluster):
         )
         == "1\n"
     )
+    # By the way, this metric does not count the LowCardinality's dictionary size.
     assert (
         node1.query(
             "SELECT value FROM system.metrics WHERE metric = 'StorageBufferBytes'"
         )
-        == "24\n"
+        == "1\n"
     )
 
     node1.query("INSERT INTO test.buffer_table VALUES('hello');")
@@ -140,7 +141,7 @@ def test_metrics_storage_buffer_size(start_cluster):
         node1.query(
             "SELECT value FROM system.metrics WHERE metric = 'StorageBufferBytes'"
         )
-        == "25\n"
+        == "2\n"
     )
 
     # flush

From 6b809a18290e251c1d12e1a586774800ac21d78c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 5 May 2023 06:43:21 +0200
Subject: [PATCH 158/689] Remove a test

---
 tests/integration/test_part_uuid/test.py | 45 ------------------------
 1 file changed, 45 deletions(-)

diff --git a/tests/integration/test_part_uuid/test.py b/tests/integration/test_part_uuid/test.py
index b30dd884427..b1eb1280d56 100644
--- a/tests/integration/test_part_uuid/test.py
+++ b/tests/integration/test_part_uuid/test.py
@@ -108,48 +108,3 @@ def test_part_uuid(started_cluster):
         uuids.add(part_merge_uuid)
         assert part_mutate_uuid not in [uuid_zero, part_merge_uuid]
     assert len(uuids) == 1, "expect the same uuid on all the replicas"
-
-
-def test_part_uuid_wal(started_cluster):
-    uuid_zero = uuid.UUID(bytes=b"\x00" * 16)
-
-    for ix, n in enumerate([node1, node2]):
-        n.query(
-            """
-        CREATE TABLE t_wal(key UInt64, value UInt64)
-        ENGINE ReplicatedMergeTree('/clickhouse/tables/t_wal', '{}')
-        ORDER BY tuple()
-        """.format(
-                ix
-            )
-        )
-
-    node2.query("INSERT INTO t_wal VALUES (1, 1)")
-
-    uuids = set()
-    for node in [node1, node2]:
-        node.query("SYSTEM SYNC REPLICA t_wal")
-        part_initial_uuid = uuid.UUID(
-            node.query(
-                "SELECT uuid FROM system.parts WHERE table = 't_wal' AND active ORDER BY name"
-            ).strip()
-        )
-        assert (
-            "InMemory"
-            == node.query(
-                "SELECT part_type FROM system.parts WHERE table = 't_wal' AND active ORDER BY name"
-            ).strip()
-        )
-        uuids.add(part_initial_uuid)
-        assert uuid_zero != part_initial_uuid
-    assert len(uuids) == 1, "expect the same uuid on all the replicas"
-
-    # Test detach / attach table to trigger WAL processing.
-    for node in [node1, node2]:
-        node.query("DETACH TABLE t_wal; ATTACH TABLE t_wal")
-        part_reattach_uuid = uuid.UUID(
-            node.query(
-                "SELECT uuid FROM system.parts WHERE table = 't_wal' AND active ORDER BY name"
-            ).strip()
-        )
-        assert part_initial_uuid == part_reattach_uuid

From ec30b0f2bea83595dcf0c29d2b36cf0eb58c2797 Mon Sep 17 00:00:00 2001
From: Kuba Kaflik <kuba.kaflik@clickhouse.com>
Date: Fri, 5 May 2023 10:14:06 +0200
Subject: [PATCH 159/689] Add function inline documentation

---
 src/TableFunctions/TableFunctionS3.cpp | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp
index 841f5a91bc8..db99daf45c4 100644
--- a/src/TableFunctions/TableFunctionS3.cpp
+++ b/src/TableFunctions/TableFunctionS3.cpp
@@ -254,12 +254,22 @@ private:
 
 void registerTableFunctionGCS(TableFunctionFactory & factory)
 {
-    factory.registerFunction<TableFunctionGCS>();
+    factory.registerFunction<TableFunctionGCS>(
+        {.documentation
+         = {R"(The table function can be used to read the data stored on Google Cloud Storage.)",
+            Documentation::Examples{{"gcs", "SELECT * FROM gcs(url, hmac_key, hmac_secret)"}},
+            Documentation::Categories{"DataLake"}},
+         .allow_readonly = false});
 }
 
 void registerTableFunctionS3(TableFunctionFactory & factory)
 {
-    factory.registerFunction<TableFunctionS3>();
+    factory.registerFunction<TableFunctionS3>(
+        {.documentation
+         = {R"(The table function can be used to read the data stored on AWS S3.)",
+            Documentation::Examples{{"s3", "SELECT * FROM s3(url, access_key_id, secret_access_key)"}},
+            Documentation::Categories{"DataLake"}},
+         .allow_readonly = false}));
 }
 
 void registerTableFunctionCOS(TableFunctionFactory & factory)

From 3fe8c9e6ed0117838ce7689788126a8223aec114 Mon Sep 17 00:00:00 2001
From: Bulat Gaifullin <gaifullinbf@gmail.com>
Date: Fri, 5 May 2023 11:54:11 +0300
Subject: [PATCH 160/689] Remove extra semicolumns

---
 src/Common/logger_useful.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Common/logger_useful.h b/src/Common/logger_useful.h
index 6412d3d1896..3ac950cbdfb 100644
--- a/src/Common/logger_useful.h
+++ b/src/Common/logger_useful.h
@@ -17,10 +17,10 @@ namespace Poco { class Logger; }
 
 namespace
 {
-    [[maybe_unused]] const ::Poco::Logger * getLogger(const ::Poco::Logger * logger) { return logger; };
-    [[maybe_unused]] const ::Poco::Logger * getLogger(const std::atomic<::Poco::Logger *> & logger) { return logger.load(); };
-    [[maybe_unused]] std::unique_ptr<LogToStrImpl> getLogger(std::unique_ptr<LogToStrImpl> && logger) { return logger; };
-    [[maybe_unused]] std::unique_ptr<LogFrequencyLimiterIml> getLogger(std::unique_ptr<LogFrequencyLimiterIml> && logger) { return logger; };
+    [[maybe_unused]] const ::Poco::Logger * getLogger(const ::Poco::Logger * logger) { return logger; }
+    [[maybe_unused]] const ::Poco::Logger * getLogger(const std::atomic<::Poco::Logger *> & logger) { return logger.load(); }
+    [[maybe_unused]] std::unique_ptr<LogToStrImpl> getLogger(std::unique_ptr<LogToStrImpl> && logger) { return logger; }
+    [[maybe_unused]] std::unique_ptr<LogFrequencyLimiterIml> getLogger(std::unique_ptr<LogFrequencyLimiterIml> && logger) { return logger; }
 }
 
 #define LOG_IMPL_FIRST_ARG(X, ...) X

From 82e6ce22fa07a70f7a7c2597ba75a90e9b024a45 Mon Sep 17 00:00:00 2001
From: Bulat Gaifullin <gaifullinbf@gmail.com>
Date: Fri, 5 May 2023 11:59:37 +0300
Subject: [PATCH 161/689] Update IFileCachePriority.h

remove extra semicolumn
---
 src/Interpreters/Cache/IFileCachePriority.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h
index 4d5f67cc10c..ad63dcc7ea5 100644
--- a/src/Interpreters/Cache/IFileCachePriority.h
+++ b/src/Interpreters/Cache/IFileCachePriority.h
@@ -92,4 +92,4 @@ private:
     const size_t max_elements = 0;
 };
 
-};
+}

From a222c4c5a1c9fec66b283e7070453db44c83b946 Mon Sep 17 00:00:00 2001
From: Bulat Gaifullin <gaifullinbf@gmail.com>
Date: Fri, 5 May 2023 11:59:59 +0300
Subject: [PATCH 162/689] Update LRUFileCachePriority.h

---
 src/Interpreters/Cache/LRUFileCachePriority.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h
index 0cb81109fcd..a80e4647f2d 100644
--- a/src/Interpreters/Cache/LRUFileCachePriority.h
+++ b/src/Interpreters/Cache/LRUFileCachePriority.h
@@ -65,4 +65,4 @@ private:
     mutable LRUFileCachePriority::LRUQueueIterator queue_iter;
 };
 
-};
+}

From 8556493c9e26164829e8b86c24e7a8e8a18aeb84 Mon Sep 17 00:00:00 2001
From: Bulat Gaifullin <gaifullinbf@gmail.com>
Date: Fri, 5 May 2023 12:00:24 +0300
Subject: [PATCH 163/689] Update LRUFileCachePriority.cpp

---
 src/Interpreters/Cache/LRUFileCachePriority.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp
index d626edc2481..20a358f9110 100644
--- a/src/Interpreters/Cache/LRUFileCachePriority.cpp
+++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp
@@ -170,4 +170,4 @@ size_t LRUFileCachePriority::LRUFileCacheIterator::use(const CacheGuard::Lock &)
     return ++queue_iter->hits;
 }
 
-};
+}

From 28403dbd034a82b601c2081d53d4e969948b43ce Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 5 May 2023 09:25:06 +0000
Subject: [PATCH 164/689] Remove explain query

---
 .../02725_cnf_large_check.reference            |  2 --
 .../0_stateless/02725_cnf_large_check.sql      | 18 ------------------
 2 files changed, 20 deletions(-)

diff --git a/tests/queries/0_stateless/02725_cnf_large_check.reference b/tests/queries/0_stateless/02725_cnf_large_check.reference
index edea3168553..1c915801174 100644
--- a/tests/queries/0_stateless/02725_cnf_large_check.reference
+++ b/tests/queries/0_stateless/02725_cnf_large_check.reference
@@ -1,6 +1,4 @@
 8
-105
 8
 2
-1120
 2
diff --git a/tests/queries/0_stateless/02725_cnf_large_check.sql b/tests/queries/0_stateless/02725_cnf_large_check.sql
index bc237dffe75..0780e6bcdd3 100644
--- a/tests/queries/0_stateless/02725_cnf_large_check.sql
+++ b/tests/queries/0_stateless/02725_cnf_large_check.sql
@@ -9,15 +9,6 @@ FROM 02725_cnf
 WHERE (c5 AND (NOT c0)) OR ((NOT c3) AND (NOT c6) AND (NOT c1) AND (NOT c6)) OR (c7 AND (NOT c3) AND (NOT c5) AND (NOT c7)) OR ((NOT c8) AND c5) OR ((NOT c0)) OR ((NOT c8) AND (NOT c5) AND c1 AND c6 AND c3) OR (c7 AND (NOT c0) AND c6 AND c1 AND (NOT c2)) OR (c3 AND (NOT c9) AND c1)
 SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 1;
 
--- make sure we use CNF form
-SELECT count()
-FROM (
-    EXPLAIN QUERY TREE
-    SELECT count()
-    FROM 02725_cnf
-    WHERE (c5 AND (NOT c0)) OR ((NOT c3) AND (NOT c6) AND (NOT c1) AND (NOT c6)) OR (c7 AND (NOT c3) AND (NOT c5) AND (NOT c7)) OR ((NOT c8) AND c5) OR ((NOT c0)) OR ((NOT c8) AND (NOT c5) AND c1 AND c6 AND c3) OR (c7 AND (NOT c0) AND c6 AND c1 AND (NOT c2)) OR (c3 AND (NOT c9) AND c1)
-    SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 1);
-
 SELECT count()
 FROM 02725_cnf
 WHERE (c5 AND (NOT c0)) OR ((NOT c3) AND (NOT c6) AND (NOT c1) AND (NOT c6)) OR (c7 AND (NOT c3) AND (NOT c5) AND (NOT c7)) OR ((NOT c8) AND c5) OR ((NOT c0)) OR ((NOT c8) AND (NOT c5) AND c1 AND c6 AND c3) OR (c7 AND (NOT c0) AND c6 AND c1 AND (NOT c2)) OR (c3 AND (NOT c9) AND c1)
@@ -28,15 +19,6 @@ FROM 02725_cnf
 WHERE ((NOT c2) AND c2 AND (NOT c1)) OR ((NOT c2) AND c3 AND (NOT c5)) OR ((NOT c7) AND (NOT c8)) OR (c9 AND c6 AND c8 AND (NOT c8) AND (NOT c7))
 SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 1;
 
--- make sure we use CNF form
-SELECT count()
-FROM (
-    EXPLAIN QUERY TREE
-    SELECT count()
-    FROM 02725_cnf
-    WHERE ((NOT c2) AND c2 AND (NOT c1)) OR ((NOT c2) AND c3 AND (NOT c5)) OR ((NOT c7) AND (NOT c8)) OR (c9 AND c6 AND c8 AND (NOT c8) AND (NOT c7))
-    SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 1);
-
 SELECT count()
 FROM 02725_cnf
 WHERE ((NOT c2) AND c2 AND (NOT c1)) OR ((NOT c2) AND c3 AND (NOT c5)) OR ((NOT c7) AND (NOT c8)) OR (c9 AND c6 AND c8 AND (NOT c8) AND (NOT c7))

From 180bb9f13d22425c0ece1c456f0723f7bfa37618 Mon Sep 17 00:00:00 2001
From: xmy <xumovens@gmail.com>
Date: Fri, 5 May 2023 18:03:07 +0800
Subject: [PATCH 165/689] fix

---
 .../aggregate-functions/reference/groupbitand.md       |  4 ++--
 .../aggregate-functions/reference/groupbitor.md        |  4 ++--
 .../aggregate-functions/reference/groupbitxor.md       |  4 ++--
 .../00498_bitwise_aggregate_functions.reference        | 10 +++++-----
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md
index 5fd5029751a..5f57407a419 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md
@@ -13,11 +13,11 @@ groupBitAnd(expr)
 
 **Arguments**
 
-`expr` – An expression that results in `UInt* or Int*` type.
+`expr` – An expression that results in `UInt*` or `Int*` type.
 
 **Return value**
 
-Value of the `UInt* or Int*` type.
+Value of the `UInt*` or `Int*` type.
 
 **Example**
 
diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md
index 08a5c15da46..59be69540b0 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md
@@ -13,11 +13,11 @@ groupBitOr(expr)
 
 **Arguments**
 
-`expr` – An expression that results in `UInt* or Int*` type.
+`expr` – An expression that results in `UInt*` or `Int*` type.
 
 **Returned value**
 
-Value of the `UInt* or Int*` type.
+Value of the `UInt*` or `Int*` type.
 
 **Example**
 
diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md
index f33e375953c..b00876a2fdf 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md
@@ -13,11 +13,11 @@ groupBitXor(expr)
 
 **Arguments**
 
-`expr` – An expression that results in `UInt* or Int*` type.
+`expr` – An expression that results in `UInt*` or `Int*` type.
 
 **Return value**
 
-Value of the `UInt* or Int*` type.
+Value of the `UInt*` or `Int*` type.
 
 **Example**
 
diff --git a/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference b/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference
index 400db09c9f1..4e5f00f1c25 100644
--- a/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference
+++ b/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference
@@ -1,7 +1,7 @@
-0	[0,4,8,12,16]	28	0	16
-1	[1,5,9,13,17]	29	1	17
-2	[2,6,10,14,18]	30	2	18
-3	[3,7,11,15,19]	31	3	19
+0       [0,4,8,12,16]   28      0       16
+1       [1,5,9,13,17]   29      1       17
+2       [2,6,10,14,18]  30      2       18
+3       [3,7,11,15,19]  31      3       19
 0       [0,-4,-8,-12,-16]       -4      0       0
 1       [-1,-5,-9,-13,-17]      -1      -29     -17
 2       [-2,-6,-10,-14,-18]     -2      -30     -18
@@ -9,4 +9,4 @@
 0       [-10,-6,-2,2,6] -2      2       -10
 1       [-9,-5,-1,3,7]  -1      3       -9
 2       [-8,-4,0,4,8]   -4      0       8
-3       [-7,-3,1,5,9]   -3      1       9
\ No newline at end of file
+3       [-7,-3,1,5,9]   -3      1       9

From 8a6084abd4cf818b1f7797b4d43d279fee9e7df9 Mon Sep 17 00:00:00 2001
From: Kuba Kaflik <kuba.kaflik@clickhouse.com>
Date: Fri, 5 May 2023 12:41:30 +0200
Subject: [PATCH 166/689] Fix syntax error for TableFunctionS3

---
 src/TableFunctions/TableFunctionS3.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp
index db99daf45c4..2369a4f384f 100644
--- a/src/TableFunctions/TableFunctionS3.cpp
+++ b/src/TableFunctions/TableFunctionS3.cpp
@@ -269,7 +269,7 @@ void registerTableFunctionS3(TableFunctionFactory & factory)
          = {R"(The table function can be used to read the data stored on AWS S3.)",
             Documentation::Examples{{"s3", "SELECT * FROM s3(url, access_key_id, secret_access_key)"}},
             Documentation::Categories{"DataLake"}},
-         .allow_readonly = false}));
+         .allow_readonly = false});
 }
 
 void registerTableFunctionCOS(TableFunctionFactory & factory)

From 2bcd1e5847c37659c0fdc6c996a92373101d4128 Mon Sep 17 00:00:00 2001
From: xmy <xumovens@gmail.com>
Date: Fri, 5 May 2023 19:22:19 +0800
Subject: [PATCH 167/689] fix

---
 ...0498_bitwise_aggregate_functions.reference | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference b/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference
index 4e5f00f1c25..b5ca25ed0b0 100644
--- a/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference
+++ b/tests/queries/0_stateless/00498_bitwise_aggregate_functions.reference
@@ -1,12 +1,12 @@
-0       [0,4,8,12,16]   28      0       16
-1       [1,5,9,13,17]   29      1       17
-2       [2,6,10,14,18]  30      2       18
-3       [3,7,11,15,19]  31      3       19
-0       [0,-4,-8,-12,-16]       -4      0       0
-1       [-1,-5,-9,-13,-17]      -1      -29     -17
-2       [-2,-6,-10,-14,-18]     -2      -30     -18
-3       [-3,-7,-11,-15,-19]     -3      -31     -19
-0       [-10,-6,-2,2,6] -2      2       -10
-1       [-9,-5,-1,3,7]  -1      3       -9
-2       [-8,-4,0,4,8]   -4      0       8
-3       [-7,-3,1,5,9]   -3      1       9
+0	[0,4,8,12,16]	28	0	16
+1	[1,5,9,13,17]	29	1	17
+2	[2,6,10,14,18]	30	2	18
+3	[3,7,11,15,19]	31	3	19
+0	[0,-4,-8,-12,-16]	-4	0	0
+1	[-1,-5,-9,-13,-17]	-1	-29	-17
+2	[-2,-6,-10,-14,-18]	-2	-30	-18
+3	[-3,-7,-11,-15,-19]	-3	-31	-19
+0	[-10,-6,-2,2,6]	-2	2	-10
+1	[-9,-5,-1,3,7]	-1	3	-9
+2	[-8,-4,0,4,8]	-4	0	8
+3	[-7,-3,1,5,9]	-3	1	9

From ce35709300852d9be08c0d90391be18d9d9b3f12 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 5 May 2023 11:52:34 +0000
Subject: [PATCH 168/689] Make test stable for old/new analyzer

00597_push_down_predicate_long
---
 .../0_stateless/00597_push_down_predicate_long.sql     | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/queries/0_stateless/00597_push_down_predicate_long.sql b/tests/queries/0_stateless/00597_push_down_predicate_long.sql
index a77f3730ac2..33853d7c025 100644
--- a/tests/queries/0_stateless/00597_push_down_predicate_long.sql
+++ b/tests/queries/0_stateless/00597_push_down_predicate_long.sql
@@ -110,7 +110,7 @@ SELECT * FROM (SELECT * FROM test_00597 UNION ALL SELECT * FROM test_00597) WHER
 
 -- Optimize predicate expression with join query
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id WHERE id = 1;
-SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id WHERE id = 1;
+SELECT date, id, name, value, r.date, r.name, r.value FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) as r USING id WHERE id = 1;
 
 EXPLAIN SYNTAX SELECT * FROM (SELECT toInt8(1) AS id) ANY LEFT JOIN test_00597 USING id WHERE value = 1;
 SELECT * FROM (SELECT toInt8(1) AS id) ANY LEFT JOIN test_00597 USING id WHERE value = 1;
@@ -121,22 +121,22 @@ SELECT b.value FROM (SELECT toInt8(1) AS id) ANY LEFT JOIN test_00597 AS b USING
 
 -- Optimize predicate expression with join and nested subquery
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1;
-SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1;
+SELECT date, id, name, value FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1;
 
 -- Optimize predicate expression with join query and qualified
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1;
-SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1;
+SELECT date, id, name, value, b.date, b.name, b.value FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1;
 
 -- Compatibility test
 EXPLAIN SYNTAX SELECT * FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01');
-SELECT * FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01');
+SELECT id, date, name, value FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01');
 
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) AS a ANY LEFT JOIN (SELECT * FROM test_00597) AS b  ON  a.id = b.id) WHERE id = 1;
 SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) AS a ANY LEFT JOIN (SELECT * FROM test_00597) AS b  ON  a.id = b.id) WHERE id = 1;
 
 -- Explain with join subquery
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1;
-SELECT * FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1;
+SELECT date, id, name, value, r.date, r.name, r.value FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1;
 
 -- issue 20497
 EXPLAIN SYNTAX SELECT value + t1.value AS expr FROM (SELECT t0.value, t1.value FROM test_00597 AS t0 FULL JOIN test_00597 AS t1 USING date) WHERE expr < 3;

From ab6e49ceaf6b096bce83c983f4eded2b2778cf5c Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Fri, 5 May 2023 14:00:18 +0200
Subject: [PATCH 169/689] Update 02731_zero_objects_in_metadata.sh

---
 tests/queries/0_stateless/02731_zero_objects_in_metadata.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh
index 31f7474089a..71c5617795e 100755
--- a/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh
+++ b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: no-fasttest, no-s3-storage
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
@@ -8,7 +9,7 @@ ${CLICKHOUSE_CLIENT} -n --query "
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (id Int32, empty Array(Int32))
     ENGINE=MergeTree ORDER BY id
-    SETTINGS min_rows_for_wide_part=0, min_bytes_for_wide_part=0, storage_policy='s3';
+    SETTINGS min_rows_for_wide_part=0, min_bytes_for_wide_part=0, storage_policy='s3_disk';
 
 INSERT INTO test (id) VALUES (1);
 SELECT * FROM test;

From 427d336e62c3fa3a3834065b5ed382ccf062280a Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 5 May 2023 12:22:37 +0000
Subject: [PATCH 170/689] Update test

---
 .../00597_push_down_predicate_long.reference       | 14 +++++++++-----
 .../0_stateless/00597_push_down_predicate_long.sql |  7 ++++++-
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/00597_push_down_predicate_long.reference b/tests/queries/0_stateless/00597_push_down_predicate_long.reference
index b8fce2a59d7..de2f1dc570d 100644
--- a/tests/queries/0_stateless/00597_push_down_predicate_long.reference
+++ b/tests/queries/0_stateless/00597_push_down_predicate_long.reference
@@ -370,9 +370,9 @@ SELECT
     id,
     name,
     value,
-    date,
-    name,
-    value
+    r.date,
+    r.name,
+    r.value
 FROM
 (
     SELECT
@@ -385,9 +385,13 @@ FROM
 )
 ANY LEFT JOIN
 (
-    SELECT id
+    SELECT
+        date,
+        id,
+        name,
+        value
     FROM test_00597
-) USING (id)
+) AS r USING (id)
 WHERE id = 1
 2000-01-01	1	test string 1	1	2000-01-01	test string 1	1
 SELECT
diff --git a/tests/queries/0_stateless/00597_push_down_predicate_long.sql b/tests/queries/0_stateless/00597_push_down_predicate_long.sql
index 33853d7c025..19437b105af 100644
--- a/tests/queries/0_stateless/00597_push_down_predicate_long.sql
+++ b/tests/queries/0_stateless/00597_push_down_predicate_long.sql
@@ -109,7 +109,8 @@ EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597 UNION ALL SELECT * FROM t
 SELECT * FROM (SELECT * FROM test_00597 UNION ALL SELECT * FROM test_00597) WHERE id = 1;
 
 -- Optimize predicate expression with join query
-EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id WHERE id = 1;
+EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) as r USING id WHERE id = 1;
+-- column names are provided explicitly to make test work with/without new analyzer (#42648)
 SELECT date, id, name, value, r.date, r.name, r.value FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) as r USING id WHERE id = 1;
 
 EXPLAIN SYNTAX SELECT * FROM (SELECT toInt8(1) AS id) ANY LEFT JOIN test_00597 USING id WHERE value = 1;
@@ -121,14 +122,17 @@ SELECT b.value FROM (SELECT toInt8(1) AS id) ANY LEFT JOIN test_00597 AS b USING
 
 -- Optimize predicate expression with join and nested subquery
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1;
+-- column names are provided explicitly to make test work with/without new analyzer, see #49551
 SELECT date, id, name, value FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1;
 
 -- Optimize predicate expression with join query and qualified
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1;
+-- column names are provided explicitly to make test work with/without new analyzer, see #49551
 SELECT date, id, name, value, b.date, b.name, b.value FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1;
 
 -- Compatibility test
 EXPLAIN SYNTAX SELECT * FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01');
+-- column names are provided explicitly to make test work with/without new analyzer, see #49551
 SELECT id, date, name, value FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01');
 
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) AS a ANY LEFT JOIN (SELECT * FROM test_00597) AS b  ON  a.id = b.id) WHERE id = 1;
@@ -136,6 +140,7 @@ SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) AS a ANY LEFT JOIN (SELE
 
 -- Explain with join subquery
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1;
+-- column names are provided explicitly to make test work with/without new analyzer, see #49551
 SELECT date, id, name, value, r.date, r.name, r.value FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1;
 
 -- issue 20497

From d35ff02b8bca501e89a41b71ce12e82116e9eec9 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 5 May 2023 12:35:51 +0000
Subject: [PATCH 171/689] Fix writeAnyEscapedString if quote_character is a
 meta character

Function writeAnyEscapedString() escapes
- quote_character with itself or with a backslash, depending on template
  parameter escape_quote_with_quote
- some metacharacters with backslash.

If quote_character is a metacharacter, the code previously always quoted
with backslash, independently of escape_quote_with_quote. I didn't find
places which do this but better be on the safe side.
---
 src/IO/WriteHelpers.h | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h
index be149394b20..cdbc952690c 100644
--- a/src/IO/WriteHelpers.h
+++ b/src/IO/WriteHelpers.h
@@ -316,6 +316,15 @@ void writeAnyEscapedString(const char * begin, const char * end, WriteBuffer & b
             pos = next_pos;
             switch (*pos)
             {
+                case quote_character:
+                {
+                    if constexpr (escape_quote_with_quote)
+                        writeChar(quote_character, buf);
+                    else
+                        writeChar('\\', buf);
+                    writeChar(quote_character, buf);
+                    break;
+                }
                 case '\b':
                     writeChar('\\', buf);
                     writeChar('b', buf);
@@ -344,15 +353,6 @@ void writeAnyEscapedString(const char * begin, const char * end, WriteBuffer & b
                     writeChar('\\', buf);
                     writeChar('\\', buf);
                     break;
-                case quote_character:
-                {
-                    if constexpr (escape_quote_with_quote)
-                        writeChar(quote_character, buf);
-                    else
-                        writeChar('\\', buf);
-                    writeChar(quote_character, buf);
-                    break;
-                }
                 default:
                     writeChar(*pos, buf);
             }

From 1050754432f723148ae9601cb7463b2c7ab799a1 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Fri, 5 May 2023 14:41:35 +0200
Subject: [PATCH 172/689] Fix

---
 src/Interpreters/Cache/Metadata.cpp |  14 ++-
 src/Interpreters/Cache/Metadata.h   | 129 ++++++++++++++--------------
 2 files changed, 70 insertions(+), 73 deletions(-)

diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp
index 4a975da1616..5d544b7d61b 100644
--- a/src/Interpreters/Cache/Metadata.cpp
+++ b/src/Interpreters/Cache/Metadata.cpp
@@ -255,8 +255,11 @@ void CacheMetadata::doCleanup()
         if (it == end())
             continue;
 
-        auto locked_metadata = std::make_unique<LockedKey>(it->second);
-        const auto key_state = locked_metadata->getKeyState();
+        /// We cannot use LockedKey here, because it submits key for removal
+        /// in destructor (in case key is empty).
+        auto key_metadata = it->second;
+        auto key_lock = key_metadata->guard.lock();
+        const auto key_state = key_metadata->key_state;
 
         if (key_state == KeyMetadata::KeyState::ACTIVE)
         {
@@ -264,7 +267,7 @@ void CacheMetadata::doCleanup()
             continue;
         }
 
-        locked_metadata->markAsRemoved();
+        key_metadata->key_state = KeyMetadata::KeyState::REMOVED;
         erase(it);
         LOG_DEBUG(log, "Key {} is removed from metadata", cleanup_key);
 
@@ -316,11 +319,6 @@ void LockedKey::removeFromCleanupQueue()
     key_metadata->key_state = KeyMetadata::KeyState::ACTIVE;
 }
 
-void LockedKey::markAsRemoved()
-{
-    key_metadata->key_state = KeyMetadata::KeyState::REMOVED;
-}
-
 bool LockedKey::isLastOwnerOfFileSegment(size_t offset) const
 {
     const auto file_segment_metadata = getByOffset(offset);
diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h
index 586c7e5c2a8..d86759ce890 100644
--- a/src/Interpreters/Cache/Metadata.h
+++ b/src/Interpreters/Cache/Metadata.h
@@ -12,69 +12,6 @@ class CleanupQueue;
 using CleanupQueuePtr = std::shared_ptr<CleanupQueue>;
 
 
-struct FileSegmentMetadata : private boost::noncopyable
-{
-    using Priority = IFileCachePriority;
-
-    explicit FileSegmentMetadata(FileSegmentPtr && file_segment_);
-
-    bool releasable() const { return file_segment.unique(); }
-
-    size_t size() const;
-
-    bool valid() const { return !removal_candidate.load(); }
-
-    Priority::Iterator getQueueIterator() { return file_segment->getQueueIterator(); }
-
-    FileSegmentPtr file_segment;
-    std::atomic<bool> removal_candidate{false};
-};
-
-using FileSegmentMetadataPtr = std::shared_ptr<FileSegmentMetadata>;
-
-
-struct KeyMetadata : public std::map<size_t, FileSegmentMetadataPtr>,
-                     private boost::noncopyable,
-                     public std::enable_shared_from_this<KeyMetadata>
-{
-    friend struct LockedKey;
-    using Key = FileCacheKey;
-
-    KeyMetadata(
-        const Key & key_,
-        const std::string & key_path_,
-        CleanupQueue & cleanup_queue_,
-        bool created_base_directory_ = false);
-
-    enum class KeyState
-    {
-        ACTIVE,
-        REMOVING,
-        REMOVED,
-    };
-
-    const Key key;
-    const std::string key_path;
-
-    LockedKeyPtr lock();
-
-    /// Return nullptr if key has non-ACTIVE state.
-    LockedKeyPtr tryLock();
-
-    bool createBaseDirectory();
-
-    std::string getFileSegmentPath(const FileSegment & file_segment);
-
-private:
-    KeyState key_state = KeyState::ACTIVE;
-    KeyGuard guard;
-    CleanupQueue & cleanup_queue;
-    std::atomic<bool> created_base_directory = false;
-};
-
-using KeyMetadataPtr = std::shared_ptr<KeyMetadata>;
-
-
 struct CacheMetadata : public std::unordered_map<FileCacheKey, KeyMetadataPtr>, private boost::noncopyable
 {
 public:
@@ -117,6 +54,70 @@ private:
 };
 
 
+struct FileSegmentMetadata : private boost::noncopyable
+{
+    using Priority = IFileCachePriority;
+
+    explicit FileSegmentMetadata(FileSegmentPtr && file_segment_);
+
+    bool releasable() const { return file_segment.unique(); }
+
+    size_t size() const;
+
+    bool valid() const { return !removal_candidate.load(); }
+
+    Priority::Iterator getQueueIterator() const { return file_segment->getQueueIterator(); }
+
+    FileSegmentPtr file_segment;
+    std::atomic<bool> removal_candidate{false};
+};
+
+using FileSegmentMetadataPtr = std::shared_ptr<FileSegmentMetadata>;
+
+
+struct KeyMetadata : public std::map<size_t, FileSegmentMetadataPtr>,
+                     private boost::noncopyable,
+                     public std::enable_shared_from_this<KeyMetadata>
+{
+    friend struct LockedKey;
+    friend void CacheMetadata::doCleanup();
+    using Key = FileCacheKey;
+
+    KeyMetadata(
+        const Key & key_,
+        const std::string & key_path_,
+        CleanupQueue & cleanup_queue_,
+        bool created_base_directory_ = false);
+
+    enum class KeyState
+    {
+        ACTIVE,
+        REMOVING,
+        REMOVED,
+    };
+
+    const Key key;
+    const std::string key_path;
+
+    LockedKeyPtr lock();
+
+    /// Return nullptr if key has non-ACTIVE state.
+    LockedKeyPtr tryLock();
+
+    bool createBaseDirectory();
+
+    std::string getFileSegmentPath(const FileSegment & file_segment);
+
+private:
+    KeyState key_state = KeyState::ACTIVE;
+    KeyGuard guard;
+    CleanupQueue & cleanup_queue;
+    std::atomic<bool> created_base_directory = false;
+};
+
+using KeyMetadataPtr = std::shared_ptr<KeyMetadata>;
+
+
 /**
  * `LockedKey` is an object which makes sure that as long as it exists the following is true:
  * 1. the key cannot be removed from cache
@@ -163,8 +164,6 @@ struct LockedKey : private boost::noncopyable
 
     void removeFromCleanupQueue();
 
-    void markAsRemoved();
-
     std::string toString() const;
 
 private:

From 6bc302959b45fdf9c23414d96c8cb433a821e30f Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Fri, 5 May 2023 14:55:52 +0200
Subject: [PATCH 173/689] Better

---
 src/Interpreters/Cache/Metadata.cpp |  16 ++--
 src/Interpreters/Cache/Metadata.h   | 129 ++++++++++++++--------------
 2 files changed, 74 insertions(+), 71 deletions(-)

diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp
index 5d544b7d61b..02ec7fdbefa 100644
--- a/src/Interpreters/Cache/Metadata.cpp
+++ b/src/Interpreters/Cache/Metadata.cpp
@@ -255,11 +255,8 @@ void CacheMetadata::doCleanup()
         if (it == end())
             continue;
 
-        /// We cannot use LockedKey here, because it submits key for removal
-        /// in destructor (in case key is empty).
-        auto key_metadata = it->second;
-        auto key_lock = key_metadata->guard.lock();
-        const auto key_state = key_metadata->key_state;
+        auto locked_metadata = std::make_unique<LockedKey>(it->second);
+        const auto key_state = locked_metadata->getKeyState();
 
         if (key_state == KeyMetadata::KeyState::ACTIVE)
         {
@@ -267,7 +264,7 @@ void CacheMetadata::doCleanup()
             continue;
         }
 
-        key_metadata->key_state = KeyMetadata::KeyState::REMOVED;
+        locked_metadata->markAsRemoved();
         erase(it);
         LOG_DEBUG(log, "Key {} is removed from metadata", cleanup_key);
 
@@ -302,7 +299,7 @@ LockedKey::LockedKey(std::shared_ptr<KeyMetadata> key_metadata_)
 
 LockedKey::~LockedKey()
 {
-    if (!key_metadata->empty())
+    if (!key_metadata->empty() || getKeyState() != KeyMetadata::KeyState::ACTIVE)
         return;
 
     key_metadata->key_state = KeyMetadata::KeyState::REMOVING;
@@ -319,6 +316,11 @@ void LockedKey::removeFromCleanupQueue()
     key_metadata->key_state = KeyMetadata::KeyState::ACTIVE;
 }
 
+void LockedKey::markAsRemoved()
+{
+    key_metadata->key_state = KeyMetadata::KeyState::REMOVED;
+}
+
 bool LockedKey::isLastOwnerOfFileSegment(size_t offset) const
 {
     const auto file_segment_metadata = getByOffset(offset);
diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h
index d86759ce890..586c7e5c2a8 100644
--- a/src/Interpreters/Cache/Metadata.h
+++ b/src/Interpreters/Cache/Metadata.h
@@ -12,6 +12,69 @@ class CleanupQueue;
 using CleanupQueuePtr = std::shared_ptr<CleanupQueue>;
 
 
+struct FileSegmentMetadata : private boost::noncopyable
+{
+    using Priority = IFileCachePriority;
+
+    explicit FileSegmentMetadata(FileSegmentPtr && file_segment_);
+
+    bool releasable() const { return file_segment.unique(); }
+
+    size_t size() const;
+
+    bool valid() const { return !removal_candidate.load(); }
+
+    Priority::Iterator getQueueIterator() { return file_segment->getQueueIterator(); }
+
+    FileSegmentPtr file_segment;
+    std::atomic<bool> removal_candidate{false};
+};
+
+using FileSegmentMetadataPtr = std::shared_ptr<FileSegmentMetadata>;
+
+
+struct KeyMetadata : public std::map<size_t, FileSegmentMetadataPtr>,
+                     private boost::noncopyable,
+                     public std::enable_shared_from_this<KeyMetadata>
+{
+    friend struct LockedKey;
+    using Key = FileCacheKey;
+
+    KeyMetadata(
+        const Key & key_,
+        const std::string & key_path_,
+        CleanupQueue & cleanup_queue_,
+        bool created_base_directory_ = false);
+
+    enum class KeyState
+    {
+        ACTIVE,
+        REMOVING,
+        REMOVED,
+    };
+
+    const Key key;
+    const std::string key_path;
+
+    LockedKeyPtr lock();
+
+    /// Return nullptr if key has non-ACTIVE state.
+    LockedKeyPtr tryLock();
+
+    bool createBaseDirectory();
+
+    std::string getFileSegmentPath(const FileSegment & file_segment);
+
+private:
+    KeyState key_state = KeyState::ACTIVE;
+    KeyGuard guard;
+    CleanupQueue & cleanup_queue;
+    std::atomic<bool> created_base_directory = false;
+};
+
+using KeyMetadataPtr = std::shared_ptr<KeyMetadata>;
+
+
 struct CacheMetadata : public std::unordered_map<FileCacheKey, KeyMetadataPtr>, private boost::noncopyable
 {
 public:
@@ -54,70 +117,6 @@ private:
 };
 
 
-struct FileSegmentMetadata : private boost::noncopyable
-{
-    using Priority = IFileCachePriority;
-
-    explicit FileSegmentMetadata(FileSegmentPtr && file_segment_);
-
-    bool releasable() const { return file_segment.unique(); }
-
-    size_t size() const;
-
-    bool valid() const { return !removal_candidate.load(); }
-
-    Priority::Iterator getQueueIterator() const { return file_segment->getQueueIterator(); }
-
-    FileSegmentPtr file_segment;
-    std::atomic<bool> removal_candidate{false};
-};
-
-using FileSegmentMetadataPtr = std::shared_ptr<FileSegmentMetadata>;
-
-
-struct KeyMetadata : public std::map<size_t, FileSegmentMetadataPtr>,
-                     private boost::noncopyable,
-                     public std::enable_shared_from_this<KeyMetadata>
-{
-    friend struct LockedKey;
-    friend void CacheMetadata::doCleanup();
-    using Key = FileCacheKey;
-
-    KeyMetadata(
-        const Key & key_,
-        const std::string & key_path_,
-        CleanupQueue & cleanup_queue_,
-        bool created_base_directory_ = false);
-
-    enum class KeyState
-    {
-        ACTIVE,
-        REMOVING,
-        REMOVED,
-    };
-
-    const Key key;
-    const std::string key_path;
-
-    LockedKeyPtr lock();
-
-    /// Return nullptr if key has non-ACTIVE state.
-    LockedKeyPtr tryLock();
-
-    bool createBaseDirectory();
-
-    std::string getFileSegmentPath(const FileSegment & file_segment);
-
-private:
-    KeyState key_state = KeyState::ACTIVE;
-    KeyGuard guard;
-    CleanupQueue & cleanup_queue;
-    std::atomic<bool> created_base_directory = false;
-};
-
-using KeyMetadataPtr = std::shared_ptr<KeyMetadata>;
-
-
 /**
  * `LockedKey` is an object which makes sure that as long as it exists the following is true:
  * 1. the key cannot be removed from cache
@@ -164,6 +163,8 @@ struct LockedKey : private boost::noncopyable
 
     void removeFromCleanupQueue();
 
+    void markAsRemoved();
+
     std::string toString() const;
 
 private:

From 6cf3e5b77a0622abe25e1938aa3f7a1496d1f9f0 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 5 May 2023 15:25:18 +0200
Subject: [PATCH 174/689] fix race on loading dictionaries

---
 programs/server/Server.cpp                    |  9 +++++++--
 tests/integration/helpers/cluster.py          |  2 +-
 .../test_dictionaries_dependency/test.py      | 20 +++++++++++++++++++
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 8c0d50bae55..c1f5440a371 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -1238,8 +1238,13 @@ try
             global_context->setMacros(std::make_unique<Macros>(*config, "macros", log));
             global_context->setExternalAuthenticatorsConfig(*config);
 
-            global_context->loadOrReloadDictionaries(*config);
-            global_context->loadOrReloadUserDefinedExecutableFunctions(*config);
+            if (global_context->isServerCompletelyStarted())
+            {
+                /// It does not make sense to reload anything before server has started.
+                /// Moreover, it may break initialization order.
+                global_context->loadOrReloadDictionaries(*config);
+                global_context->loadOrReloadUserDefinedExecutableFunctions(*config);
+            }
 
             global_context->setRemoteHostFilter(*config);
 
diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 8adb722e20b..950663cb429 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -17,7 +17,6 @@ import urllib.parse
 import shlex
 import urllib3
 import requests
-import pyspark
 
 try:
     # Please, add modules that required for specific tests only here.
@@ -33,6 +32,7 @@ try:
     import nats
     import ssl
     import meilisearch
+    import pyspark
     from confluent_kafka.avro.cached_schema_registry_client import (
         CachedSchemaRegistryClient,
     )
diff --git a/tests/integration/test_dictionaries_dependency/test.py b/tests/integration/test_dictionaries_dependency/test.py
index 05d6afd35c7..7e73974e8cf 100644
--- a/tests/integration/test_dictionaries_dependency/test.py
+++ b/tests/integration/test_dictionaries_dependency/test.py
@@ -191,3 +191,23 @@ def test_dependent_dict_table_distr(node):
     node.restart_clickhouse()
 
     query("DROP DATABASE IF EXISTS test_db;")
+
+
+def test_no_lazy_load():
+    node2.query("create database no_lazy")
+    node2.query(
+        "create table no_lazy.src (n int, m int) engine=MergeTree order by n partition by n % 100"
+    )
+    node2.query("insert into no_lazy.src select number, number from numbers(0, 99)")
+    node2.query("insert into no_lazy.src select number, number from numbers(100, 99)")
+    node2.query(
+        "create dictionary no_lazy.dict (n int, mm int) primary key n "
+        "source(clickhouse(query 'select n, m + sleepEachRow(0.1) as mm from no_lazy.src')) "
+        "lifetime(min 0 max 0) layout(complex_key_hashed_array(shards 10))"
+    )
+
+    node2.restart_clickhouse()
+
+    assert "42\n" == node2.query("select dictGet('no_lazy.dict', 'mm', 42)")
+
+    node2.query("drop database no_lazy")

From bb1dd0ff8982bd66f44329a434f5e1fb8a1905c7 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 5 May 2023 13:56:48 +0000
Subject: [PATCH 175/689] Remove test from broken_test

---
 tests/broken_tests.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/broken_tests.txt b/tests/broken_tests.txt
index f052faa6515..6c34e09a47e 100644
--- a/tests/broken_tests.txt
+++ b/tests/broken_tests.txt
@@ -1,7 +1,6 @@
 00223_shard_distributed_aggregation_memory_efficient
 00562_in_subquery_merge_tree
 00593_union_all_assert_columns_removed
-00597_push_down_predicate_long
 00673_subquery_prepared_set_performance
 00700_decimal_compare
 00717_merge_and_distributed
@@ -141,4 +140,4 @@
 00261_storage_aliases_and_array_join
 02701_non_parametric_function
 01825_type_json_multiple_files
-01281_group_by_limit_memory_tracking
\ No newline at end of file
+01281_group_by_limit_memory_tracking

From f7e31a9664745ff100029edf3bc1f95b60a9bf13 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 5 May 2023 11:03:47 +0000
Subject: [PATCH 176/689] More protection against SQL injection

---
 .../InterpreterShowColumnsQuery.cpp           | 25 ++++++++++++-------
 .../InterpreterShowIndexesQuery.cpp           | 14 ++++++++---
 .../0_stateless/02706_show_columns.reference  |  5 ++++
 .../0_stateless/02706_show_columns.sql        | 20 +++++++++++++++
 .../0_stateless/02724_show_indexes.reference  |  5 ++++
 .../0_stateless/02724_show_indexes.sql        | 20 +++++++++++++++
 6 files changed, 77 insertions(+), 12 deletions(-)

diff --git a/src/Interpreters/InterpreterShowColumnsQuery.cpp b/src/Interpreters/InterpreterShowColumnsQuery.cpp
index c1a546eae79..c86d3c753c4 100644
--- a/src/Interpreters/InterpreterShowColumnsQuery.cpp
+++ b/src/Interpreters/InterpreterShowColumnsQuery.cpp
@@ -1,11 +1,12 @@
 #include <Interpreters/InterpreterShowColumnsQuery.h>
 
+#include <Common/quoteString.h>
+#include <IO/Operators.h>
 #include <IO/WriteBufferFromString.h>
 #include <Parsers/ASTShowColumnsQuery.h>
 #include <Parsers/formatAST.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/executeQuery.h>
-#include <IO/Operators.h>
 
 
 namespace DB
@@ -23,16 +24,22 @@ String InterpreterShowColumnsQuery::getRewrittenQuery()
 {
     const auto & query = query_ptr->as<ASTShowColumnsQuery &>();
 
-    String database = query.database.empty() ? getContext()->getCurrentDatabase() : query.database;
-    String table = query.table;
+    WriteBufferFromOwnString buf_database;
+    String resolved_database = getContext()->resolveDatabase(query.database);
+    writeEscapedString(resolved_database, buf_database);
+    String database = buf_database.str();
+
+    WriteBufferFromOwnString buf_table;
+    writeEscapedString(query.table, buf_table);
+    String table = buf_table.str();
 
     String rewritten_query = R"(
 SELECT
     name AS field,
     type AS type,
     startsWith(type, 'Nullable') AS null,
-    trim(concatWithSeparator(' ', if(is_in_primary_key, 'PRI', ''), if (is_in_sorting_key, 'SOR', ''))) AS key,
-    if(default_kind IN ('ALIAS', 'DEFAULT', 'MATERIALIZED'), default_expression, NULL) AS default,
+    trim(concatWithSeparator(' ', if (is_in_primary_key, 'PRI', ''), if (is_in_sorting_key, 'SOR', ''))) AS key,
+    if (default_kind IN ('ALIAS', 'DEFAULT', 'MATERIALIZED'), default_expression, NULL) AS default,
     '' AS extra )";
 
     // TODO Interpret query.extended. It is supposed to show internal/virtual columns. Need to fetch virtual column names, see
@@ -51,10 +58,10 @@ SELECT
     }
 
     rewritten_query += fmt::format(R"(
-    FROM system.columns
-    WHERE
-        database = '{}'
-        AND table = '{}' )", database, table);
+FROM system.columns
+WHERE
+    database = '{}'
+    AND table = '{}' )", database, table);
 
     if (!query.like.empty())
     {
diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp
index ca66b32ca80..51311c82eeb 100644
--- a/src/Interpreters/InterpreterShowIndexesQuery.cpp
+++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp
@@ -1,11 +1,12 @@
 #include <Interpreters/InterpreterShowIndexesQuery.h>
 
+#include <Common/quoteString.h>
+#include <IO/Operators.h>
 #include <IO/WriteBufferFromString.h>
 #include <Parsers/ASTShowIndexesQuery.h>
 #include <Parsers/formatAST.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/executeQuery.h>
-#include <IO/Operators.h>
 
 
 namespace DB
@@ -23,8 +24,15 @@ String InterpreterShowIndexesQuery::getRewrittenQuery()
 {
     const auto & query = query_ptr->as<ASTShowIndexesQuery &>();
 
-    String table = query.table;
-    String database = query.database.empty() ? getContext()->getCurrentDatabase() : query.database;
+    WriteBufferFromOwnString buf_table;
+    writeEscapedString(query.table, buf_table);
+    String table = buf_table.str();
+
+    WriteBufferFromOwnString buf_database;
+    String resolved_database = getContext()->resolveDatabase(query.database);
+    writeEscapedString(resolved_database, buf_database);
+    String database = buf_database.str();
+
     String where_expression = query.where_expression ? fmt::format("WHERE ({})", query.where_expression) : "";
 
     String rewritten_query = fmt::format(R"(
diff --git a/tests/queries/0_stateless/02706_show_columns.reference b/tests/queries/0_stateless/02706_show_columns.reference
index fff8d38ede7..9d7e8485e19 100644
--- a/tests/queries/0_stateless/02706_show_columns.reference
+++ b/tests/queries/0_stateless/02706_show_columns.reference
@@ -28,6 +28,11 @@ int32	Nullable(Int32)	1		\N
 uint64	UInt64	0	PRI SOR	\N	
 --- LIMIT
 int32	Nullable(Int32)	1		\N	
+--- Check with weird table names
+c	String	0	PRI SOR	\N	
+c	String	0	PRI SOR	\N	
+c	String	0	PRI SOR	\N	
+c	String	0	PRI SOR	\N	
 --- Original table
 int32	Nullable(Int32)	1		\N	
 str	String	0	SOR	\N	
diff --git a/tests/queries/0_stateless/02706_show_columns.sql b/tests/queries/0_stateless/02706_show_columns.sql
index 522f513b685..b1a907c5c71 100644
--- a/tests/queries/0_stateless/02706_show_columns.sql
+++ b/tests/queries/0_stateless/02706_show_columns.sql
@@ -43,6 +43,26 @@ SHOW COLUMNS FROM tab WHERE field LIKE '%int%';
 SELECT '--- LIMIT';
 SHOW COLUMNS FROM tab LIMIT 1;
 
+SELECT '--- Check with weird table names';
+
+DROP TABLE IF EXISTS `$4@^7`;
+CREATE TABLE `$4@^7` (c String) ENGINE = MergeTree ORDER BY c;
+SHOW COLUMNS FROM `$4@^7`;
+DROP TABLE `$4@^7`;
+
+DROP TABLE IF EXISTS NULL;
+CREATE TABLE NULL (c String) ENGINE = MergeTree ORDER BY c;
+SHOW COLUMNS FROM NULL;
+DROP TABLE NULL;
+
+DROP DATABASE IF EXISTS `'`;
+CREATE DATABASE `'`;
+CREATE TABLE `'`.`'` (c String) ENGINE = MergeTree ORDER BY c;
+SHOW COLUMNS FROM `'` FROM `'`;
+SHOW COLUMNS FROM `'`.`'`; -- abbreviated form
+DROP TABLE `'`.`'`;
+DROP DATABASE `'`;
+
 -- Create a table in a different database. Intentionally useing the same table/column names as above so
 -- we notice if something is buggy in the implementation of SHOW COLUMNS.
 DROP DATABASE IF EXISTS database_123456789abcde;
diff --git a/tests/queries/0_stateless/02724_show_indexes.reference b/tests/queries/0_stateless/02724_show_indexes.reference
index 2ca55c77015..20af3954fa5 100644
--- a/tests/queries/0_stateless/02724_show_indexes.reference
+++ b/tests/queries/0_stateless/02724_show_indexes.reference
@@ -23,6 +23,11 @@ tbl	0	set_idx	\N	\N	\N	\N	\N	\N	\N	set	\N	\N	YES	e
 --- WHERE
 tbl	0	mm1_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	a, c, d
 tbl	0	mm2_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	c, d, e
+--- Check with weird table names
+$4@^7	0	PRIMARY	\N	\N	A	\N	\N	\N	\N	primary	\N	\N	YES	c
+NULL	0	PRIMARY	\N	\N	A	\N	\N	\N	\N	primary	\N	\N	YES	c
+\'	0	PRIMARY	\N	\N	A	\N	\N	\N	\N	primary	\N	\N	YES	c
+\'	0	PRIMARY	\N	\N	A	\N	\N	\N	\N	primary	\N	\N	YES	c
 --- Original table
 tbl	0	blf_idx	\N	\N	\N	\N	\N	\N	\N	bloom_filter	\N	\N	YES	d, b
 tbl	0	mm1_idx	\N	\N	\N	\N	\N	\N	\N	minmax	\N	\N	YES	a, c, d
diff --git a/tests/queries/0_stateless/02724_show_indexes.sql b/tests/queries/0_stateless/02724_show_indexes.sql
index 477e2f049c2..ce8ed67c524 100644
--- a/tests/queries/0_stateless/02724_show_indexes.sql
+++ b/tests/queries/0_stateless/02724_show_indexes.sql
@@ -30,6 +30,26 @@ SHOW EXTENDED INDEX FROM tbl;
 SELECT '--- WHERE';
 SHOW INDEX FROM tbl WHERE index_type LIKE '%minmax%';
 
+SELECT '--- Check with weird table names';
+
+DROP TABLE IF EXISTS `$4@^7`;
+CREATE TABLE `$4@^7` (c String) ENGINE = MergeTree ORDER BY c;
+SHOW INDEX FROM `$4@^7`;
+DROP TABLE `$4@^7`;
+
+DROP TABLE IF EXISTS NULL;
+CREATE TABLE NULL (c String) ENGINE = MergeTree ORDER BY c;
+SHOW INDEX FROM NULL;
+DROP TABLE NULL;
+
+DROP DATABASE IF EXISTS `'`;
+CREATE DATABASE `'`;
+CREATE TABLE `'`.`'` (c String) ENGINE = MergeTree ORDER BY c;
+SHOW INDEX FROM `'` FROM `'`;
+SHOW INDEX FROM `'`.`'`; -- abbreviated form
+DROP TABLE `'`.`'`;
+DROP DATABASE `'`;
+
 -- Create a table in a different database. Intentionally using the same table/column names as above so
 -- we notice if something is buggy in the implementation of SHOW INDEX.
 DROP DATABASE IF EXISTS database_123456789abcde;

From dd1bbf7c784019b9be87b2e0c374643bf9aaec80 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 5 May 2023 16:27:12 +0200
Subject: [PATCH 177/689] fix another issue with dependencies

---
 src/Databases/DDLDependencyVisitor.cpp        | 35 +++++++++++++++++--
 src/Databases/DDLLoadingDependencyVisitor.cpp |  2 +-
 src/Databases/DDLRenamingVisitor.cpp          |  2 +-
 .../getDictionaryConfigurationFromAST.cpp     |  8 +++--
 .../getDictionaryConfigurationFromAST.h       |  1 +
 .../test_dictionaries_dependency/test.py      |  4 +++
 6 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp
index 26f912f6482..31d636a22f8 100644
--- a/src/Databases/DDLDependencyVisitor.cpp
+++ b/src/Databases/DDLDependencyVisitor.cpp
@@ -12,6 +12,8 @@
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
 #include <Parsers/ASTTablesInSelectQuery.h>
+#include <Parsers/ParserSelectWithUnionQuery.h>
+#include <Parsers/parseQuery.h>
 #include <Common/KnownObjectNames.h>
 #include <Poco/String.h>
 
@@ -25,6 +27,7 @@ namespace
     /// Used to visits ASTCreateQuery and extracts the names of all tables explicitly referenced in the create query.
     class DDLDependencyVisitorData
     {
+        friend void tryVisitNestedSelect(const String & query, DDLDependencyVisitorData & data);
     public:
         DDLDependencyVisitorData(const ContextPtr & context_, const QualifiedTableName & table_name_, const ASTPtr & ast_)
             : create_query(ast_), table_name(table_name_), current_database(context_->getCurrentDatabase()), context(context_)
@@ -106,9 +109,17 @@ namespace
             if (!info || !info->is_local)
                 return;
 
-            if (info->table_name.database.empty())
-                info->table_name.database = current_database;
-            dependencies.emplace(std::move(info->table_name));
+            if (!info->table_name.table.empty())
+            {
+                if (info->table_name.database.empty())
+                    info->table_name.database = current_database;
+                dependencies.emplace(std::move(info->table_name));
+            }
+            else
+            {
+                /// We don't have a table name, we have a select query instead
+                tryVisitNestedSelect(info->query, *this);
+            }
         }
 
         /// ASTTableExpression represents a reference to a table in SELECT query.
@@ -424,6 +435,24 @@ namespace
         static bool needChildVisit(const ASTPtr &, const ASTPtr & child, const Data & data) { return data.needChildVisit(child); }
         static void visit(const ASTPtr & ast, Data & data) { data.visit(ast); }
     };
+
+    void tryVisitNestedSelect(const String & query, DDLDependencyVisitorData & data)
+    {
+        try
+        {
+            ParserSelectWithUnionQuery parser;
+            String description = fmt::format("Query for ClickHouse dictionary {}", data.table_name);
+            ASTPtr select = parseQuery(parser, query, description,
+                                       data.context->getSettingsRef().max_query_size, data.context->getSettingsRef().max_parser_depth);
+
+            DDLDependencyVisitor::Visitor visitor{data};
+            visitor.visit(select);
+        }
+        catch (...)
+        {
+            tryLogCurrentException("DDLDependencyVisitor");
+        }
+    }
 }
 
 
diff --git a/src/Databases/DDLLoadingDependencyVisitor.cpp b/src/Databases/DDLLoadingDependencyVisitor.cpp
index 22dc0b4af7b..99538fd801e 100644
--- a/src/Databases/DDLLoadingDependencyVisitor.cpp
+++ b/src/Databases/DDLLoadingDependencyVisitor.cpp
@@ -103,7 +103,7 @@ void DDLLoadingDependencyVisitor::visit(const ASTFunctionWithKeyValueArguments &
     auto config = getDictionaryConfigurationFromAST(data.create_query->as<ASTCreateQuery &>(), data.global_context);
     auto info = getInfoIfClickHouseDictionarySource(config, data.global_context);
 
-    if (!info || !info->is_local)
+    if (!info || !info->is_local || info->table_name.table.empty())
         return;
 
     if (info->table_name.database.empty())
diff --git a/src/Databases/DDLRenamingVisitor.cpp b/src/Databases/DDLRenamingVisitor.cpp
index 9651861fd4f..6cd414635a0 100644
--- a/src/Databases/DDLRenamingVisitor.cpp
+++ b/src/Databases/DDLRenamingVisitor.cpp
@@ -137,7 +137,7 @@ namespace
 
         auto config = getDictionaryConfigurationFromAST(data.create_query->as<ASTCreateQuery &>(), data.global_context);
         auto info = getInfoIfClickHouseDictionarySource(config, data.global_context);
-        if (!info || !info->is_local)
+        if (!info || !info->is_local || info->table_name.table.empty())
             return;
 
         auto * source_list = dictionary.source->elements->as<ASTExpressionList>();
diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp
index 0de8b843604..05065df5251 100644
--- a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp
+++ b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp
@@ -649,10 +649,12 @@ getInfoIfClickHouseDictionarySource(DictionaryConfigurationPtr & config, Context
     String database = config->getString("dictionary.source.clickhouse.db", "");
     String table = config->getString("dictionary.source.clickhouse.table", "");
 
-    if (table.empty())
-        return {};
+    info.query = config->getString("dictionary.source.clickhouse.query", "");
 
-    info.table_name = {database, table};
+    if (!table.empty())
+        info.table_name = {database, table};
+    else if (info.query.empty())
+        return {};
 
     try
     {
diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.h b/src/Dictionaries/getDictionaryConfigurationFromAST.h
index ec44b9815ff..aa42835d33d 100644
--- a/src/Dictionaries/getDictionaryConfigurationFromAST.h
+++ b/src/Dictionaries/getDictionaryConfigurationFromAST.h
@@ -18,6 +18,7 @@ getDictionaryConfigurationFromAST(const ASTCreateQuery & query, ContextPtr conte
 struct ClickHouseDictionarySourceInfo
 {
     QualifiedTableName table_name;
+    String query;
     bool is_local = false;
 };
 
diff --git a/tests/integration/test_dictionaries_dependency/test.py b/tests/integration/test_dictionaries_dependency/test.py
index 7e73974e8cf..d262738af60 100644
--- a/tests/integration/test_dictionaries_dependency/test.py
+++ b/tests/integration/test_dictionaries_dependency/test.py
@@ -210,4 +210,8 @@ def test_no_lazy_load():
 
     assert "42\n" == node2.query("select dictGet('no_lazy.dict', 'mm', 42)")
 
+    assert "some tables depend on it" in node2.query_and_get_error(
+        "drop table no_lazy.src", settings={"check_referential_table_dependencies": 1}
+    )
+
     node2.query("drop database no_lazy")

From 7bec75a59c1b7c7b4219680b7ee17744a91e98d5 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 5 May 2023 14:36:45 +0000
Subject: [PATCH 178/689] Use the same queries with '*' instead of specifying
 columns

---
 .../00597_push_down_predicate_long.reference  | 24 ++++++++++-------
 .../00597_push_down_predicate_long.sql        | 27 +++++++++++--------
 2 files changed, 31 insertions(+), 20 deletions(-)

diff --git a/tests/queries/0_stateless/00597_push_down_predicate_long.reference b/tests/queries/0_stateless/00597_push_down_predicate_long.reference
index de2f1dc570d..4ac88edd40e 100644
--- a/tests/queries/0_stateless/00597_push_down_predicate_long.reference
+++ b/tests/queries/0_stateless/00597_push_down_predicate_long.reference
@@ -370,9 +370,9 @@ SELECT
     id,
     name,
     value,
-    r.date,
-    r.name,
-    r.value
+    date,
+    name,
+    value
 FROM
 (
     SELECT
@@ -385,15 +385,13 @@ FROM
 )
 ANY LEFT JOIN
 (
-    SELECT
-        date,
-        id,
-        name,
-        value
+    SELECT id
     FROM test_00597
-) AS r USING (id)
+) USING (id)
 WHERE id = 1
 2000-01-01	1	test string 1	1	2000-01-01	test string 1	1
+------- allow_experimental_analyzer=1 -------
+1	2000-01-01	test string 1	1	2000-01-01	test string 1	1
 SELECT
     id,
     date,
@@ -456,6 +454,8 @@ FROM
 )
 WHERE id = 1
 2000-01-01	1	test string 1	1
+------- allow_experimental_analyzer=1 -------
+1	2000-01-01	test string 1	1	2000-01-01	test string 1	1
 SELECT
     date,
     id,
@@ -484,6 +484,8 @@ ANY LEFT JOIN
 ) AS b USING (id)
 WHERE b.id = 1
 2000-01-01	1	test string 1	1	2000-01-01	test string 1	1
+------- allow_experimental_analyzer=1 -------
+1	2000-01-01	test string 1	1	2000-01-01	test string 1	1
 SELECT
     id,
     date,
@@ -508,6 +510,8 @@ ANY LEFT JOIN
 ) AS b USING (date, id)
 WHERE b.date = toDate(\'2000-01-01\')
 1	2000-01-01	test string 1	1
+------- allow_experimental_analyzer=1 -------
+2000-01-01	1	test string 1	1
 SELECT
     date,
     id,
@@ -589,6 +593,8 @@ SEMI LEFT JOIN
 ) AS r USING (id)
 WHERE r.id = 1
 2000-01-01	1	test string 1	1	2000-01-01	test string 1	1
+------- allow_experimental_analyzer=1 -------
+1	2000-01-01	test string 1	1	2000-01-01	test string 1	1
 SELECT value + t1.value AS expr
 FROM
 (
diff --git a/tests/queries/0_stateless/00597_push_down_predicate_long.sql b/tests/queries/0_stateless/00597_push_down_predicate_long.sql
index 19437b105af..caf6edd7372 100644
--- a/tests/queries/0_stateless/00597_push_down_predicate_long.sql
+++ b/tests/queries/0_stateless/00597_push_down_predicate_long.sql
@@ -109,9 +109,10 @@ EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597 UNION ALL SELECT * FROM t
 SELECT * FROM (SELECT * FROM test_00597 UNION ALL SELECT * FROM test_00597) WHERE id = 1;
 
 -- Optimize predicate expression with join query
-EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) as r USING id WHERE id = 1;
--- column names are provided explicitly to make test work with/without new analyzer (#42648)
-SELECT date, id, name, value, r.date, r.name, r.value FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) as r USING id WHERE id = 1;
+EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id WHERE id = 1;
+SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id WHERE id = 1 SETTINGS allow_experimental_analyzer=0;
+SELECT '------- allow_experimental_analyzer=1 -------';
+SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id WHERE id = 1 SETTINGS allow_experimental_analyzer=1;
 
 EXPLAIN SYNTAX SELECT * FROM (SELECT toInt8(1) AS id) ANY LEFT JOIN test_00597 USING id WHERE value = 1;
 SELECT * FROM (SELECT toInt8(1) AS id) ANY LEFT JOIN test_00597 USING id WHERE value = 1;
@@ -122,26 +123,30 @@ SELECT b.value FROM (SELECT toInt8(1) AS id) ANY LEFT JOIN test_00597 AS b USING
 
 -- Optimize predicate expression with join and nested subquery
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1;
--- column names are provided explicitly to make test work with/without new analyzer, see #49551
-SELECT date, id, name, value FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1;
+SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1 SETTINGS allow_experimental_analyzer=0;
+SELECT '------- allow_experimental_analyzer=1 -------';
+SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1 SETTINGS allow_experimental_analyzer=1;
 
 -- Optimize predicate expression with join query and qualified
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1;
--- column names are provided explicitly to make test work with/without new analyzer, see #49551
-SELECT date, id, name, value, b.date, b.name, b.value FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1;
+SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1 SETTINGS allow_experimental_analyzer=0;
+SELECT '------- allow_experimental_analyzer=1 -------';
+SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1 SETTINGS allow_experimental_analyzer=1;
 
 -- Compatibility test
 EXPLAIN SYNTAX SELECT * FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01');
--- column names are provided explicitly to make test work with/without new analyzer, see #49551
-SELECT id, date, name, value FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01');
+SELECT * FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01') SETTINGS allow_experimental_analyzer=0;
+SELECT '------- allow_experimental_analyzer=1 -------';
+SELECT * FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01') SETTINGS allow_experimental_analyzer=1;
 
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) AS a ANY LEFT JOIN (SELECT * FROM test_00597) AS b  ON  a.id = b.id) WHERE id = 1;
 SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) AS a ANY LEFT JOIN (SELECT * FROM test_00597) AS b  ON  a.id = b.id) WHERE id = 1;
 
 -- Explain with join subquery
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1;
--- column names are provided explicitly to make test work with/without new analyzer, see #49551
-SELECT date, id, name, value, r.date, r.name, r.value FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1;
+SELECT * FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1 SETTINGS allow_experimental_analyzer=0;
+SELECT '------- allow_experimental_analyzer=1 -------';
+SELECT * FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1 SETTINGS allow_experimental_analyzer=1;
 
 -- issue 20497
 EXPLAIN SYNTAX SELECT value + t1.value AS expr FROM (SELECT t0.value, t1.value FROM test_00597 AS t0 FULL JOIN test_00597 AS t1 USING date) WHERE expr < 3;

From 951bef4fd219587dcabe1d7b01e9bfbf19a7c62f Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 5 May 2023 15:10:50 +0000
Subject: [PATCH 179/689] Fixmemory counting in TCPHandler

---
 src/Server/TCPHandler.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index e45a26d76af..4e41fbf4230 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -505,8 +505,8 @@ void TCPHandler::runImpl()
             /// the MemoryTracker will be wrong for possible deallocations.
             /// (i.e. deallocations from the Aggregator with two-level aggregation)
             state.reset();
+            last_sent_snapshots = {};
             query_scope.reset();
-            last_sent_snapshots.clear();
             thread_trace_context.reset();
         }
         catch (const Exception & e)

From b13e3abc4d77fcd2bcc3069d90bb4481e6fca0d1 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 5 May 2023 15:17:16 +0000
Subject: [PATCH 180/689] Improve CI check for Analyzer

---
 docker/test/util/process_functional_tests_result.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py
index 470eb61b3fa..71c0d57b762 100755
--- a/docker/test/util/process_functional_tests_result.py
+++ b/docker/test/util/process_functional_tests_result.py
@@ -59,8 +59,12 @@ def process_test_log(log_path, broken_tests):
 
                 total += 1
                 if TIMEOUT_SIGN in line:
-                    failed += 1
-                    test_results.append((test_name, "Timeout", test_time, []))
+                    if test_name in broken_tests:
+                        success += 1
+                        test_results.append((test_name, "OK", test_time, []))
+                    else:
+                        failed += 1
+                        test_results.append((test_name, "Timeout", test_time, []))
                 elif FAIL_SIGN in line:
                     if test_name in broken_tests:
                         success += 1
@@ -76,11 +80,11 @@ def process_test_log(log_path, broken_tests):
                     test_results.append((test_name, "SKIPPED", test_time, []))
                 else:
                     if OK_SIGN in line and test_name in broken_tests:
-                        failed += 1
+                        skipped += 1
                         test_results.append(
                             (
                                 test_name,
-                                "SKIPPED",
+                                "EXPECTED_TO_FAIL",
                                 test_time,
                                 ["This test passed. Update broken_tests.txt.\n"],
                             )

From 36b028bc6b2c1987b986ae58c5913cc29b20f250 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <104093494+CheSema@users.noreply.github.com>
Date: Fri, 5 May 2023 17:48:57 +0200
Subject: [PATCH 181/689] Update src/Storages/StorageReplicatedMergeTree.cpp

Co-authored-by: Alexander Tokmakov <tavplubix@gmail.com>
---
 src/Storages/StorageReplicatedMergeTree.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 70ebdf21973..269a288327e 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -8270,7 +8270,7 @@ StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, co
         /// The metadata files is about to delete now.
         /// However remote data might be shared and has to be unlocked in the keper before removing.
         /// Actually there is some cases when it is clear without keper:
-        /// When the part has been fetched then remote data has to be  preserved, part doesn't own it.
+        /// When the part has been fetched then remote data has to be preserved, part doesn't own it.
         /// When the part has been merged then remote data can to removed, part own it.
         /// In opposition, when the part has been mutated in generally it hardlinks the files.
         /// Therefore remote data is shared, it has to be unlocked in the keper.

From efcf7fbe4f41ef611c88d15685d1ec0b6f79341f Mon Sep 17 00:00:00 2001
From: Sema Checherinda <104093494+CheSema@users.noreply.github.com>
Date: Fri, 5 May 2023 17:49:12 +0200
Subject: [PATCH 182/689] Update src/Storages/StorageReplicatedMergeTree.cpp

Co-authored-by: Alexander Tokmakov <tavplubix@gmail.com>
---
 src/Storages/StorageReplicatedMergeTree.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 269a288327e..1c049991100 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -8271,7 +8271,7 @@ StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, co
         /// However remote data might be shared and has to be unlocked in the keper before removing.
         /// Actually there is some cases when it is clear without keper:
         /// When the part has been fetched then remote data has to be preserved, part doesn't own it.
-        /// When the part has been merged then remote data can to removed, part own it.
+        /// When the part has been merged then remote data can be removed, part owns it.
         /// In opposition, when the part has been mutated in generally it hardlinks the files.
         /// Therefore remote data is shared, it has to be unlocked in the keper.
 

From f361d96f21ac3176793032ad9da7958cf14fedff Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Fri, 5 May 2023 15:50:43 +0000
Subject: [PATCH 183/689] Forbid the change of data type of Object('json')
 column

---
 src/Storages/AlterCommands.cpp                    | 14 ++++++++++++++
 .../02725_object_column_alter.reference           |  0
 .../0_stateless/02725_object_column_alter.sql     | 15 +++++++++++++++
 3 files changed, 29 insertions(+)
 create mode 100644 tests/queries/0_stateless/02725_object_column_alter.reference
 create mode 100644 tests/queries/0_stateless/02725_object_column_alter.sql

diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp
index aff17465466..6cc427f9ee7 100644
--- a/src/Storages/AlterCommands.cpp
+++ b/src/Storages/AlterCommands.cpp
@@ -1117,6 +1117,20 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const
                 }
             }
 
+            /// The change of data type to/from Object is broken, so disable it for now
+            if (command.data_type)
+            {
+                GetColumnsOptions options(GetColumnsOptions::AllPhysical);
+                auto old_data_type = all_columns.getColumn(options, column_name).type;
+
+                if (WhichDataType(command.data_type).isObject()
+                    || WhichDataType(old_data_type).isObject())
+                    throw Exception(
+                        ErrorCodes::BAD_ARGUMENTS,
+                        "The change of data type {} of column {} to {} is not allowed",
+                        old_data_type->getFamilyName(), backQuote(column_name), command.data_type->getFamilyName());
+            }
+
             if (command.isRemovingProperty())
             {
                 if (!column_default && command.to_remove == AlterCommand::RemoveProperty::DEFAULT)
diff --git a/tests/queries/0_stateless/02725_object_column_alter.reference b/tests/queries/0_stateless/02725_object_column_alter.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/02725_object_column_alter.sql b/tests/queries/0_stateless/02725_object_column_alter.sql
new file mode 100644
index 00000000000..436640537fb
--- /dev/null
+++ b/tests/queries/0_stateless/02725_object_column_alter.sql
@@ -0,0 +1,15 @@
+-- Eventually this ALTER should be fixed
+
+SET allow_experimental_object_type=1;
+
+DROP TABLE IF EXISTS t_to;
+DROP TABLE IF EXISTS t_from;
+
+CREATE TABLE t_to (id UInt64, value Nullable(String)) ENGINE MergeTree() ORDER BY id;
+CREATE TABLE t_from (id UInt64, value Object('json')) ENGINE MergeTree() ORDER BY id;
+
+ALTER TABLE t_to MODIFY COLUMN value Object('json'); -- { serverError BAD_ARGUMENTS }
+ALTER TABLE t_from MODIFY COLUMN value Nullable(String); -- { serverError BAD_ARGUMENTS }
+
+DROP TABLE t_to;
+DROP TABLE t_from;
\ No newline at end of file

From 5422e3058fd4a8842bae1d20a326634a0eca8155 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 5 May 2023 16:19:01 +0000
Subject: [PATCH 184/689] Use more specific test status

---
 docker/test/util/process_functional_tests_result.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py
index 71c0d57b762..ad544136a9a 100755
--- a/docker/test/util/process_functional_tests_result.py
+++ b/docker/test/util/process_functional_tests_result.py
@@ -61,14 +61,14 @@ def process_test_log(log_path, broken_tests):
                 if TIMEOUT_SIGN in line:
                     if test_name in broken_tests:
                         success += 1
-                        test_results.append((test_name, "OK", test_time, []))
+                        test_results.append((test_name, "KNOWN_BROKEN", test_time, []))
                     else:
                         failed += 1
                         test_results.append((test_name, "Timeout", test_time, []))
                 elif FAIL_SIGN in line:
                     if test_name in broken_tests:
                         success += 1
-                        test_results.append((test_name, "OK", test_time, []))
+                        test_results.append((test_name, "KNOWN_BROKEN", test_time, []))
                     else:
                         failed += 1
                         test_results.append((test_name, "FAIL", test_time, []))

From 3fe0cb28c4c99ce51d2dacaefb33a44b85258a24 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 5 May 2023 16:21:49 +0000
Subject: [PATCH 185/689] Update broken_tests.txt

---
 tests/broken_tests.txt | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/broken_tests.txt b/tests/broken_tests.txt
index 96c4ea62f55..de0115b7ce1 100644
--- a/tests/broken_tests.txt
+++ b/tests/broken_tests.txt
@@ -134,3 +134,11 @@
 00261_storage_aliases_and_array_join
 01825_type_json_multiple_files
 01281_group_by_limit_memory_tracking
+02723_zookeeper_name
+00002_log_and_exception_messages_formatting
+01646_rewrite_sum_if_bug
+02725_agg_projection_resprect_PK
+01019_alter_materialized_view_consistent
+01600_parts_states_metrics_long
+01600_parts_types_metrics_long
+01287_max_execution_speed

From 945a5130cfd81b15e16f454d62554afc47cbd9c4 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 5 May 2023 16:04:12 +0000
Subject: [PATCH 186/689] Simplify simplify simplify

---
 src/Interpreters/Cache/QueryCache.cpp         | 63 +++++++++++--------
 src/Interpreters/Cache/QueryCache.h           | 15 +++--
 src/Interpreters/executeQuery.cpp             |  6 +-
 src/Processors/Sources/SourceFromChunks.cpp   | 58 +----------------
 src/Processors/Sources/SourceFromChunks.h     | 19 +-----
 .../StreamInQueryCacheTransform.cpp           |  6 +-
 .../Transforms/StreamInQueryCacheTransform.h  |  4 +-
 src/QueryPipeline/QueryPipeline.cpp           | 30 +++++++--
 src/QueryPipeline/QueryPipeline.h             |  6 +-
 ...2494_query_cache_sparse_columns.reference} |  0
 ...l => 02494_query_cache_sparse_columns.sql} |  0
 .../02494_query_cache_totals_extremes.sql     |  6 +-
 12 files changed, 97 insertions(+), 116 deletions(-)
 rename tests/queries/0_stateless/{02708_query_cache_sparse_columns.reference => 02494_query_cache_sparse_columns.reference} (100%)
 rename tests/queries/0_stateless/{02708_query_cache_sparse_columns.sql => 02494_query_cache_sparse_columns.sql} (100%)

diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp
index 512abc82529..796be10ccc8 100644
--- a/src/Interpreters/Cache/QueryCache.cpp
+++ b/src/Interpreters/Cache/QueryCache.cpp
@@ -6,7 +6,6 @@
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTSetQuery.h>
 #include <Parsers/IAST.h>
-#include <Processors/Sources/SourceFromChunks.h>
 #include <Common/logger_useful.h>
 #include <Common/ProfileEvents.h>
 #include <Common/SipHash.h>
@@ -204,27 +203,27 @@ QueryCache::Writer::Writer(const Writer & other)
 {
 }
 
-void QueryCache::Writer::buffer(Chunk && chunk, Type type)
+void QueryCache::Writer::buffer(Chunk && chunk, ChunkType chunk_type)
 {
     if (skip_insert)
         return;
 
     std::lock_guard lock(mutex);
 
-    switch (type)
+    switch (chunk_type)
     {
-        case Type::Result:
+        case ChunkType::Result:
         {
             /// Normal query result chunks are simply buffered. They are squashed and compressed later in finalizeWrite().
             query_result->chunks.emplace_back(std::move(chunk));
             break;
         }
-        case Type::Totals:
-        case Type::Extremes:
+        case ChunkType::Totals:
+        case ChunkType::Extremes:
         {
             /// For simplicity, totals and extremes chunks are immediately squashed (totals/extremes are obscure and even if enabled, few
             /// such chunks are expected).
-            auto & buffered_chunk = (type == Type::Totals) ? query_result->totals : query_result->extremes;
+            auto & buffered_chunk = (chunk_type == ChunkType::Totals) ? query_result->totals : query_result->extremes;
 
             convertToFullIfSparse(chunk);
 
@@ -349,19 +348,24 @@ void QueryCache::Writer::finalizeWrite()
     was_finalized = true;
 }
 
-/// Constructs a pipe with just one processor: SourceFromChunks which serves data from the query cache
-void QueryCache::Reader::buildPipe(Block header, Chunks && chunks, const std::optional<Chunk> & totals, const std::optional<Chunk> & extremes)
+/// Creates a source processor which serves result chunks stored in the query cache, and separate sources for optional totals/extremes.
+void QueryCache::Reader::buildSourceFromChunks(Block header, Chunks && chunks, const std::optional<Chunk> & totals, const std::optional<Chunk> & extremes)
 {
-    auto source_from_chunks = std::make_shared<SourceFromChunks>(
-            header, std::move(chunks),
-            totals.has_value() ? std::optional(totals->clone()) : std::nullopt,
-            extremes.has_value() ? std::optional(extremes->clone()) : std::nullopt);
+    source_from_chunks = std::make_unique<SourceFromChunks>(header, std::move(chunks));
 
-    OutputPort * out = &source_from_chunks->getPort();
-    OutputPort * out_totals = source_from_chunks->getTotalsPort();
-    OutputPort * out_extremes = source_from_chunks->getExtremesPort();
+    if (totals.has_value())
+    {
+        Chunks chunks_totals;
+        chunks_totals.emplace_back(totals->clone());
+        source_from_chunks_totals = std::make_unique<SourceFromChunks>(header, std::move(chunks_totals));
+    }
 
-    pipe = Pipe(source_from_chunks, out, out_totals, out_extremes);
+    if (extremes.has_value())
+    {
+        Chunks chunks_extremes;
+        chunks_extremes.emplace_back(extremes->clone());
+        source_from_chunks_extremes = std::make_unique<SourceFromChunks>(header, std::move(chunks_extremes));
+    }
 }
 
 QueryCache::Reader::Reader(Cache & cache_, const Key & key, const std::lock_guard<std::mutex> &)
@@ -398,7 +402,7 @@ QueryCache::Reader::Reader(Cache & cache_, const Key & key, const std::lock_guar
         for (const auto & chunk : entry->mapped->chunks)
             cloned_chunks.push_back(chunk.clone());
 
-        buildPipe(entry->key.header, std::move(cloned_chunks), entry->mapped->totals, entry->mapped->extremes);
+        buildSourceFromChunks(entry->key.header, std::move(cloned_chunks), entry->mapped->totals, entry->mapped->extremes);
     }
     else
     {
@@ -417,7 +421,7 @@ QueryCache::Reader::Reader(Cache & cache_, const Key & key, const std::lock_guar
             decompressed_chunks.push_back(std::move(decompressed_chunk));
         }
 
-        buildPipe(entry->key.header, std::move(decompressed_chunks), entry->mapped->totals, entry->mapped->extremes);
+        buildSourceFromChunks(entry->key.header, std::move(decompressed_chunks), entry->mapped->totals, entry->mapped->extremes);
     }
 
     LOG_TRACE(&Poco::Logger::get("QueryCache"), "Entry found for query {}", key.queryStringFromAst());
@@ -425,20 +429,29 @@ QueryCache::Reader::Reader(Cache & cache_, const Key & key, const std::lock_guar
 
 bool QueryCache::Reader::hasCacheEntryForKey() const
 {
-    bool res = !pipe.empty();
+    bool has_entry = (source_from_chunks != nullptr);
 
-    if (res)
+    if (has_entry)
         ProfileEvents::increment(ProfileEvents::QueryCacheHits);
     else
         ProfileEvents::increment(ProfileEvents::QueryCacheMisses);
 
-    return res;
+    return has_entry;
 }
 
-Pipe && QueryCache::Reader::getPipe()
+std::unique_ptr<SourceFromChunks> QueryCache::Reader::getSource()
 {
-    chassert(!pipe.empty()); // cf. hasCacheEntryForKey()
-    return std::move(pipe);
+    return std::move(source_from_chunks);
+}
+
+std::unique_ptr<SourceFromChunks> QueryCache::Reader::getSourceTotals()
+{
+    return std::move(source_from_chunks_totals);
+}
+
+std::unique_ptr<SourceFromChunks> QueryCache::Reader::getSourceExtremes()
+{
+    return std::move(source_from_chunks_extremes);
 }
 
 QueryCache::Reader QueryCache::createReader(const Key & key)
diff --git a/src/Interpreters/Cache/QueryCache.h b/src/Interpreters/Cache/QueryCache.h
index f614ca248d4..4736dc62fb1 100644
--- a/src/Interpreters/Cache/QueryCache.h
+++ b/src/Interpreters/Cache/QueryCache.h
@@ -3,6 +3,7 @@
 #include <Common/CacheBase.h>
 #include <Core/Block.h>
 #include <Parsers/IAST_fwd.h>
+#include <Processors/Sources/SourceFromChunks.h>
 #include <Poco/Util/LayeredConfiguration.h>
 #include <Processors/Chunk.h>
 #include <QueryPipeline/Pipe.h>
@@ -114,8 +115,8 @@ public:
 
         Writer(const Writer & other);
 
-        enum class Type {Result, Totals, Extremes};
-        void buffer(Chunk && chunk, Type type);
+        enum class ChunkType {Result, Totals, Extremes};
+        void buffer(Chunk && chunk, ChunkType chunk_type);
 
         void finalizeWrite();
     private:
@@ -146,11 +147,15 @@ public:
     {
     public:
         bool hasCacheEntryForKey() const;
-        Pipe && getPipe(); /// must be called only if hasCacheEntryForKey() returns true
+        std::unique_ptr<SourceFromChunks> getSource();
+        std::unique_ptr<SourceFromChunks> getSourceTotals();
+        std::unique_ptr<SourceFromChunks> getSourceExtremes();
     private:
         Reader(Cache & cache_, const Key & key, const std::lock_guard<std::mutex> &);
-        void buildPipe(Block header, Chunks && chunks, const std::optional<Chunk> & totals, const std::optional<Chunk> & extremes);
-        Pipe pipe;
+        void buildSourceFromChunks(Block header, Chunks && chunks, const std::optional<Chunk> & totals, const std::optional<Chunk> & extremes);
+        std::unique_ptr<SourceFromChunks> source_from_chunks;
+        std::unique_ptr<SourceFromChunks> source_from_chunks_totals;
+        std::unique_ptr<SourceFromChunks> source_from_chunks_extremes;
         friend class QueryCache; /// for createReader()
     };
 
diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index ffb01f812de..1461c4a9533 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -737,7 +737,9 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
                     QueryCache::Reader reader = query_cache->createReader(key);
                     if (reader.hasCacheEntryForKey())
                     {
-                        res.pipeline = QueryPipeline(reader.getPipe());
+                        QueryPipeline pipeline;
+                        pipeline.readFromQueryCache(reader.getSource(), reader.getSourceTotals(), reader.getSourceExtremes());
+                        res.pipeline = std::move(pipeline);
                         read_result_from_query_cache = true;
                     }
                 }
@@ -768,7 +770,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
                                          settings.max_block_size,
                                          settings.query_cache_max_size_in_bytes,
                                          settings.query_cache_max_entries));
-                        res.pipeline.streamIntoQueryCache(query_cache_writer);
+                        res.pipeline.writeResultIntoQueryCache(query_cache_writer);
                     }
                 }
 
diff --git a/src/Processors/Sources/SourceFromChunks.cpp b/src/Processors/Sources/SourceFromChunks.cpp
index 54a6e4b54e2..9fad508c23b 100644
--- a/src/Processors/Sources/SourceFromChunks.cpp
+++ b/src/Processors/Sources/SourceFromChunks.cpp
@@ -3,62 +3,11 @@
 namespace DB
 {
 
-SourceFromChunks::SourceFromChunks(Block header, Chunks && chunks_, std::optional<Chunk> && chunk_totals_, std::optional<Chunk> && chunk_extremes_)
-    : ISource(header)
+SourceFromChunks::SourceFromChunks(Block header, Chunks chunks_)
+    : ISource(std::move(header))
     , chunks(std::move(chunks_))
     , it(chunks.begin())
-{
-    if (chunk_totals_ != std::nullopt)
-    {
-        outputs.emplace_back(header, this);
-        chunk_totals = std::move(chunk_totals_);
-        output_totals = &outputs.back();
-    }
-
-    if (chunk_extremes_ != std::nullopt)
-    {
-        outputs.emplace_back(header, this);
-        chunk_extremes = std::move(chunk_extremes_);
-        output_extremes = &outputs.back();
-    }
-}
-
-SourceFromChunks::Status SourceFromChunks::prepare()
-{
-    if (!finished_chunks)
-    {
-        Status status = ISource::prepare();
-
-        if (status != Status::Finished)
-            return status;
-
-        finished_chunks = true;
-    }
-
-    if (getTotalsPort())
-    {
-        /// This logic force-pushes the data into the port instead of checking the port status first (isFinished(), canPush()). Seems to
-        /// work but should be improved (TODO).
-        chassert(chunk_totals.has_value());
-        output_totals->push(std::move(*chunk_totals));
-        output_totals->finish();
-    }
-
-    if (getExtremesPort())
-    {
-        chassert(chunk_extremes.has_value());
-        output_extremes->push(std::move(*chunk_extremes));
-        output_extremes->finish();
-    }
-
-    return Status::Finished;
-}
-
-void SourceFromChunks::work()
-{
-    if (!finished_chunks)
-        ISource::work();
-}
+{}
 
 String SourceFromChunks::getName() const
 {
@@ -78,4 +27,3 @@ Chunk SourceFromChunks::generate()
 }
 
 }
-
diff --git a/src/Processors/Sources/SourceFromChunks.h b/src/Processors/Sources/SourceFromChunks.h
index 58616a8058c..b508f206d0c 100644
--- a/src/Processors/Sources/SourceFromChunks.h
+++ b/src/Processors/Sources/SourceFromChunks.h
@@ -7,33 +7,20 @@
 namespace DB
 {
 
-/// The big brother of SourceFromSingleChunk. Supports multiple chunks and totals/extremes.
+/// The big brother of SourceFromSingleChunk.
 class SourceFromChunks : public ISource
 {
 public:
-    SourceFromChunks(Block header, Chunks && chunks_, std::optional<Chunk> && chunk_totals_, std::optional<Chunk> && chunk_extremes_);
+    SourceFromChunks(Block header, Chunks chunks_);
 
     String getName() const override;
 
-    Status prepare() override;
-    void work() override;
-
-    OutputPort * getTotalsPort() const { return output_totals; }
-    OutputPort * getExtremesPort() const { return output_extremes; }
-
+protected:
     Chunk generate() override;
 
 private:
     Chunks chunks;
     Chunks::iterator it;
-
-    std::optional<Chunk> chunk_totals = std::nullopt;
-    std::optional<Chunk> chunk_extremes = std::nullopt;
-
-    OutputPort * output_totals = nullptr;
-    OutputPort * output_extremes = nullptr;
-
-    bool finished_chunks = false;
 };
 
 }
diff --git a/src/Processors/Transforms/StreamInQueryCacheTransform.cpp b/src/Processors/Transforms/StreamInQueryCacheTransform.cpp
index c4210d7afa5..6bcae1d3053 100644
--- a/src/Processors/Transforms/StreamInQueryCacheTransform.cpp
+++ b/src/Processors/Transforms/StreamInQueryCacheTransform.cpp
@@ -6,16 +6,16 @@ namespace DB
 StreamInQueryCacheTransform::StreamInQueryCacheTransform(
     const Block & header_,
     std::shared_ptr<QueryCache::Writer> query_cache_writer_,
-    QueryCache::Writer::Type type_)
+    QueryCache::Writer::ChunkType chunk_type_)
     : ISimpleTransform(header_, header_, false)
     , query_cache_writer(query_cache_writer_)
-    , type(type_)
+    , chunk_type(chunk_type_)
 {
 }
 
 void StreamInQueryCacheTransform::transform(Chunk & chunk)
 {
-    query_cache_writer->buffer(chunk.clone(), type);
+    query_cache_writer->buffer(chunk.clone(), chunk_type);
 }
 
 void StreamInQueryCacheTransform::finalizeWriteInQueryCache()
diff --git a/src/Processors/Transforms/StreamInQueryCacheTransform.h b/src/Processors/Transforms/StreamInQueryCacheTransform.h
index e26f343290d..c2357049cc1 100644
--- a/src/Processors/Transforms/StreamInQueryCacheTransform.h
+++ b/src/Processors/Transforms/StreamInQueryCacheTransform.h
@@ -12,7 +12,7 @@ public:
     StreamInQueryCacheTransform(
         const Block & header_,
         std::shared_ptr<QueryCache::Writer> query_cache_writer,
-        QueryCache::Writer::Type type);
+        QueryCache::Writer::ChunkType chunk_type);
 
 protected:
     void transform(Chunk & chunk) override;
@@ -23,7 +23,7 @@ public:
 
 private:
     const std::shared_ptr<QueryCache::Writer> query_cache_writer;
-    const QueryCache::Writer::Type type;
+    const QueryCache::Writer::ChunkType chunk_type;
 };
 
 }
diff --git a/src/QueryPipeline/QueryPipeline.cpp b/src/QueryPipeline/QueryPipeline.cpp
index 1149cef0094..87a1408969e 100644
--- a/src/QueryPipeline/QueryPipeline.cpp
+++ b/src/QueryPipeline/QueryPipeline.cpp
@@ -14,6 +14,7 @@
 #include <Processors/Sinks/SinkToStorage.h>
 #include <Processors/Sources/NullSource.h>
 #include <Processors/Sources/RemoteSource.h>
+#include <Processors/Sources/SourceFromChunks.h>
 #include <Processors/ISource.h>
 #include <Processors/Transforms/CountingTransform.h>
 #include <Processors/Transforms/LimitsCheckingTransform.h>
@@ -577,7 +578,7 @@ bool QueryPipeline::tryGetResultRowsAndBytes(UInt64 & result_rows, UInt64 & resu
     return true;
 }
 
-void QueryPipeline::streamIntoQueryCache(std::shared_ptr<QueryCache::Writer> query_cache_writer)
+void QueryPipeline::writeResultIntoQueryCache(std::shared_ptr<QueryCache::Writer> query_cache_writer)
 {
     assert(pulling());
 
@@ -586,18 +587,18 @@ void QueryPipeline::streamIntoQueryCache(std::shared_ptr<QueryCache::Writer> que
     /// This ensures that all transforms write to the single same cache entry. The writer object synchronizes internally, the
     /// expensive stuff like cloning chunks happens outside lock scopes).
 
-    auto add_stream_in_query_cache_transform = [&](OutputPort *& out_port, QueryCache::Writer::Type type)
+    auto add_stream_in_query_cache_transform = [&](OutputPort *& out_port, QueryCache::Writer::ChunkType chunk_type)
     {
         if (!out_port)
             return;
 
-        auto transform = std::make_shared<StreamInQueryCacheTransform>(out_port->getHeader(), query_cache_writer, type);
+        auto transform = std::make_shared<StreamInQueryCacheTransform>(out_port->getHeader(), query_cache_writer, chunk_type);
         connect(*out_port, transform->getInputPort());
         out_port = &transform->getOutputPort();
         processors->emplace_back(std::move(transform));
     };
 
-    using enum QueryCache::Writer::Type;
+    using enum QueryCache::Writer::ChunkType;
 
     add_stream_in_query_cache_transform(output, Result);
     add_stream_in_query_cache_transform(totals, Totals);
@@ -616,6 +617,27 @@ void QueryPipeline::finalizeWriteInQueryCache()
         dynamic_cast<StreamInQueryCacheTransform &>(**it).finalizeWriteInQueryCache();
 }
 
+void QueryPipeline::readFromQueryCache(
+        std::unique_ptr<SourceFromChunks> source,
+        std::unique_ptr<SourceFromChunks> source_totals,
+        std::unique_ptr<SourceFromChunks> source_extremes)
+{
+    /// Construct the pipeline from the input source processors. The processors are provided by the query cache to produce chunks of a
+    /// previous query result.
+
+    auto add_stream_from_query_cache_source = [&](OutputPort *& out_port, std::unique_ptr<SourceFromChunks> source_)
+    {
+        if (!source_)
+            return;
+        out_port = &source_->getPort();
+        processors->emplace_back(std::shared_ptr<SourceFromChunks>(std::move(source_)));
+    };
+
+    add_stream_from_query_cache_source(output, std::move(source));
+    add_stream_from_query_cache_source(totals, std::move(source_totals));
+    add_stream_from_query_cache_source(extremes, std::move(source_extremes));
+}
+
 void QueryPipeline::addStorageHolder(StoragePtr storage)
 {
     resources.storage_holders.emplace_back(std::move(storage));
diff --git a/src/QueryPipeline/QueryPipeline.h b/src/QueryPipeline/QueryPipeline.h
index 297178715fd..0476b8e4bbf 100644
--- a/src/QueryPipeline/QueryPipeline.h
+++ b/src/QueryPipeline/QueryPipeline.h
@@ -105,8 +105,12 @@ public:
     void setLimitsAndQuota(const StreamLocalLimits & limits, std::shared_ptr<const EnabledQuota> quota_);
     bool tryGetResultRowsAndBytes(UInt64 & result_rows, UInt64 & result_bytes) const;
 
-    void streamIntoQueryCache(std::shared_ptr<QueryCache::Writer> query_cache_writer);
+    void writeResultIntoQueryCache(std::shared_ptr<QueryCache::Writer> query_cache_writer);
     void finalizeWriteInQueryCache();
+    void readFromQueryCache(
+        std::unique_ptr<SourceFromChunks> source,
+        std::unique_ptr<SourceFromChunks> source_totals,
+        std::unique_ptr<SourceFromChunks> source_extremes);
 
     void setQuota(std::shared_ptr<const EnabledQuota> quota_);
 
diff --git a/tests/queries/0_stateless/02708_query_cache_sparse_columns.reference b/tests/queries/0_stateless/02494_query_cache_sparse_columns.reference
similarity index 100%
rename from tests/queries/0_stateless/02708_query_cache_sparse_columns.reference
rename to tests/queries/0_stateless/02494_query_cache_sparse_columns.reference
diff --git a/tests/queries/0_stateless/02708_query_cache_sparse_columns.sql b/tests/queries/0_stateless/02494_query_cache_sparse_columns.sql
similarity index 100%
rename from tests/queries/0_stateless/02708_query_cache_sparse_columns.sql
rename to tests/queries/0_stateless/02494_query_cache_sparse_columns.sql
diff --git a/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql b/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql
index d49609885b1..0d05adca6c8 100644
--- a/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql
+++ b/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql
@@ -9,7 +9,7 @@ DROP TABLE IF EXISTS tbl;
 CREATE TABLE tbl (key UInt64, agg UInt64) ENGINE = MergeTree ORDER BY key;
 INSERT INTO tbl VALUES (1, 3), (2, 2), (1, 4), (1, 1);
 
--- A query with totals calculation. The result should written into / read from the query cache.
+-- A query with totals calculation. The result should be written into / read from the query cache.
 -- Check that both queries produce the same result and that a query cache entry exists.
 SELECT '1st run:';
 SELECT key, sum(agg) FROM tbl GROUP BY key WITH totals ORDER BY key SETTINGS use_query_cache = 1;
@@ -22,7 +22,7 @@ SELECT '---';
 
 SYSTEM DROP QUERY CACHE;
 
--- A query with extremes calculation. The result should written into / read from the query cache.
+-- A query with extremes calculation. The result should be written into / read from the query cache.
 -- Check that both queries produce the same result.
 SELECT '1st run:';
 SELECT key, sum(agg) FROM tbl GROUP BY key ORDER BY key SETTINGS use_query_cache = 1, extremes = 1;
@@ -35,7 +35,7 @@ SELECT '---';
 
 SYSTEM DROP QUERY CACHE;
 
--- A query with totals and extremes calculation. The result should written into / read from the query cache.
+-- A query with totals and extremes calculation. The result should be written into / read from the query cache.
 -- Check that both queries produce the same result.
 SELECT '1st run:';
 SELECT key, sum(agg) FROM tbl GROUP BY key WITH totals ORDER BY key SETTINGS use_query_cache = 1, extremes = 1;

From 20cd799e518c25e41de9cf6ce1bec1429fda2749 Mon Sep 17 00:00:00 2001
From: ltrk2 <107155950+ltrk2@users.noreply.github.com>
Date: Fri, 5 May 2023 16:26:53 +0000
Subject: [PATCH 187/689] Add CMake option for BOOST_USE_UCONTEXT

---
 base/base/defines.h                | 12 -----------
 contrib/boost-cmake/CMakeLists.txt | 33 +++++++++++++++++-------------
 src/CMakeLists.txt                 |  3 ++-
 3 files changed, 21 insertions(+), 27 deletions(-)

diff --git a/base/base/defines.h b/base/base/defines.h
index 91c35dc28b6..6abf8155b95 100644
--- a/base/base/defines.h
+++ b/base/base/defines.h
@@ -73,18 +73,6 @@
 #    endif
 #endif
 
-#if defined(ADDRESS_SANITIZER)
-#    define BOOST_USE_ASAN 1
-#    define BOOST_USE_UCONTEXT 1
-#endif
-
-#if defined(THREAD_SANITIZER)
-#    define BOOST_USE_TSAN 1
-#    define BOOST_USE_UCONTEXT 1
-#endif
-
-/// TODO: Strange enough, there is no way to detect UB sanitizer.
-
 /// Explicitly allow undefined behaviour for certain functions. Use it as a function attribute.
 /// It is useful in case when compiler cannot see (and exploit) it, but UBSan can.
 /// Example: multiplication of signed integers with possibility of overflow when both sides are from user input.
diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt
index 2a70c25ffe1..c9a759eab9c 100644
--- a/contrib/boost-cmake/CMakeLists.txt
+++ b/contrib/boost-cmake/CMakeLists.txt
@@ -92,6 +92,8 @@ add_library (boost::system ALIAS _boost_system)
 target_include_directories (_boost_system PRIVATE ${LIBRARY_DIR})
 
 # context
+option (BOOST_USE_UCONTEXT "Use ucontext_t for context switching of boost::fiber within boost::context" OFF)
+
 enable_language(ASM)
 SET(ASM_OPTIONS "-x assembler-with-cpp")
 
@@ -100,20 +102,6 @@ set (SRCS_CONTEXT
     "${LIBRARY_DIR}/libs/context/src/posix/stack_traits.cpp"
 )
 
-if (SANITIZE AND (SANITIZE STREQUAL "address" OR SANITIZE STREQUAL "thread"))
-    add_compile_definitions(BOOST_USE_UCONTEXT)
-
-    if (SANITIZE STREQUAL "address")
-        add_compile_definitions(BOOST_USE_ASAN)
-    elseif (SANITIZE STREQUAL "thread")
-        add_compile_definitions(BOOST_USE_TSAN)
-    endif()
-
-    set (SRCS_CONTEXT ${SRCS_CONTEXT}
-            "${LIBRARY_DIR}/libs/context/src/fiber.cpp"
-            "${LIBRARY_DIR}/libs/context/src/continuation.cpp"
-    )
-endif()
 if (ARCH_AARCH64)
     set (SRCS_CONTEXT ${SRCS_CONTEXT}
         "${LIBRARY_DIR}/libs/context/src/asm/jump_arm64_aapcs_elf_gas.S"
@@ -152,10 +140,27 @@ else()
     )
 endif()
 
+if (SANITIZE OR BOOST_USE_UCONTEXT)
+    list (APPEND SRCS_CONTEXT
+        "${LIBRARY_DIR}/libs/context/src/fiber.cpp"
+        "${LIBRARY_DIR}/libs/context/src/continuation.cpp"
+    )
+endif()
+
 add_library (_boost_context ${SRCS_CONTEXT})
 add_library (boost::context ALIAS _boost_context)
 target_include_directories (_boost_context PRIVATE ${LIBRARY_DIR})
 
+if (SANITIZE OR BOOST_USE_UCONTEXT)
+    target_compile_definitions(_boost_context PUBLIC BOOST_USE_UCONTEXT)
+endif()
+
+if (SANITIZE STREQUAL "address")
+    target_compile_definitions(_boost_context PUBLIC BOOST_USE_ASAN)
+elseif (SANITIZE STREQUAL "thread")
+    target_compile_definitions(_boost_context PUBLIC BOOST_USE_TSAN)
+endif()
+
 # coroutine
 
 set (SRCS_COROUTINE
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 9b1968c7d82..76a67ade99c 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -544,7 +544,8 @@ if (TARGET ch_contrib::qpl)
 dbms_target_link_libraries(PUBLIC ch_contrib::qpl)
 endif ()
 
-dbms_target_link_libraries(PRIVATE _boost_context)
+target_link_libraries(clickhouse_common_io PUBLIC boost::context)
+dbms_target_link_libraries(PUBLIC boost::context)
 
 if (ENABLE_NLP)
     dbms_target_link_libraries (PUBLIC ch_contrib::stemmer)

From 7b7050b91fd6f95039c655fb5087737f81030581 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 5 May 2023 16:34:07 +0000
Subject: [PATCH 188/689] Fix
 01655_plan_optimizations_optimize_read_in_window_order for analyzer

---
 ...tions_optimize_read_in_window_order.reference | 10 ++++++++++
 ...ptimizations_optimize_read_in_window_order.sh | 16 ++++++++++++----
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.reference b/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.reference
index 00eb03bd5f0..f23cf03913b 100644
--- a/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.reference
+++ b/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.reference
@@ -1,15 +1,25 @@
 Partial sorting plan
   optimize_read_in_window_order=0
     Sort description: n ASC, x ASC
+  optimize_read_in_window_order=0, allow_experimental_analyzer=1
+    Sort description: n ASC, x ASC
   optimize_read_in_window_order=1
     Prefix sort description: n ASC
     Result sort description: n ASC, x ASC
+  optimize_read_in_window_order=1, allow_experimental_analyzer=1
+    Prefix sort description: default.test_01655_plan_optimizations_optimize_read_in_window_order_n.n_0 ASC
+    Result sort description: default.test_01655_plan_optimizations_optimize_read_in_window_order_n.n_0 ASC, default.test_01655_plan_optimizations_optimize_read_in_window_order_n.x_1 ASC
 No sorting plan
   optimize_read_in_window_order=0
     Sort description: n ASC, x ASC
+  optimize_read_in_window_order=0, allow_experimental_analyzer=1
+    Sort description: default.test_01655_plan_optimizations_optimize_read_in_window_order_n_x.n_0 ASC, default.test_01655_plan_optimizations_optimize_read_in_window_order_n_x.x_1 ASC
   optimize_read_in_window_order=1
     Prefix sort description: n ASC, x ASC
     Result sort description: n ASC, x ASC
+  optimize_read_in_window_order=1, allow_experimental_analyzer=1
+    Prefix sort description: default.test_01655_plan_optimizations_optimize_read_in_window_order_n_x.n_0 ASC, default.test_01655_plan_optimizations_optimize_read_in_window_order_n_x.x_1 ASC
+    Result sort description: default.test_01655_plan_optimizations_optimize_read_in_window_order_n_x.n_0 ASC, default.test_01655_plan_optimizations_optimize_read_in_window_order_n_x.x_1 ASC
 Complex ORDER BY
   optimize_read_in_window_order=0
 3	3	1
diff --git a/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.sh b/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.sh
index a606f1a2f9e..24c8cf5052e 100755
--- a/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.sh
+++ b/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.sh
@@ -19,17 +19,25 @@ $CLICKHOUSE_CLIENT -q "optimize table ${name}_n_x final"
 
 echo 'Partial sorting plan'
 echo '  optimize_read_in_window_order=0'
-$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0" | grep -i "sort description"
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0,allow_experimental_analyzer=0" | grep -i "sort description"
+echo '  optimize_read_in_window_order=0, allow_experimental_analyzer=1'
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0,allow_experimental_analyzer=0" | grep -i "sort description"
 
 echo '  optimize_read_in_window_order=1'
-$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=1" | grep -i "sort description"
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=1,allow_experimental_analyzer=0" | grep -i "sort description"
+echo '  optimize_read_in_window_order=1, allow_experimental_analyzer=1'
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=1,allow_experimental_analyzer=1" | grep -i "sort description"
 
 echo 'No sorting plan'
 echo '  optimize_read_in_window_order=0'
-$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0" | grep -i "sort description"
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0,allow_experimental_analyzer=0" | grep -i "sort description"
+echo '  optimize_read_in_window_order=0, allow_experimental_analyzer=1'
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0,allow_experimental_analyzer=1" | grep -i "sort description"
 
 echo '  optimize_read_in_window_order=1'
-$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=1" | grep -i "sort description"
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=1,allow_experimental_analyzer=0" | grep -i "sort description"
+echo '  optimize_read_in_window_order=1, allow_experimental_analyzer=1'
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=1,allow_experimental_analyzer=1" | grep -i "sort description"
 
 echo 'Complex ORDER BY'
 $CLICKHOUSE_CLIENT -q "CREATE TABLE ${name}_complex (unique1 Int32, unique2 Int32, ten Int32) ENGINE=MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192"

From 40bd46ca3a8fc5a1ce59b4f730078f46b03ade3f Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 5 May 2023 16:44:55 +0000
Subject: [PATCH 189/689] Update broken_tests file

---
 tests/broken_tests.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/broken_tests.txt b/tests/broken_tests.txt
index f052faa6515..ef9dc467a7b 100644
--- a/tests/broken_tests.txt
+++ b/tests/broken_tests.txt
@@ -61,7 +61,6 @@
 01624_soft_constraints
 01651_bugs_from_15889
 01655_plan_optimizations
-01655_plan_optimizations_optimize_read_in_window_order
 01656_test_query_log_factories_info
 01681_bloom_filter_nullable_column
 01700_system_zookeeper_path_in
@@ -141,4 +140,4 @@
 00261_storage_aliases_and_array_join
 02701_non_parametric_function
 01825_type_json_multiple_files
-01281_group_by_limit_memory_tracking
\ No newline at end of file
+01281_group_by_limit_memory_tracking

From dfd8826254e64de100e9c3f9f283dde0ae5ffee0 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Fri, 5 May 2023 18:59:58 +0200
Subject: [PATCH 190/689] Update 02731_zero_objects_in_metadata.sh

---
 tests/queries/0_stateless/02731_zero_objects_in_metadata.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh
index 71c5617795e..690cf977d08 100755
--- a/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh
+++ b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh
@@ -9,7 +9,7 @@ ${CLICKHOUSE_CLIENT} -n --query "
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (id Int32, empty Array(Int32))
     ENGINE=MergeTree ORDER BY id
-    SETTINGS min_rows_for_wide_part=0, min_bytes_for_wide_part=0, storage_policy='s3_disk';
+    SETTINGS min_rows_for_wide_part=0, min_bytes_for_wide_part=0, disk='s3_disk';
 
 INSERT INTO test (id) VALUES (1);
 SELECT * FROM test;

From edf658010cdab38171ba7f17762e45d637f5b5bd Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio2368@users.noreply.github.com>
Date: Fri, 5 May 2023 19:09:32 +0200
Subject: [PATCH 191/689] Add comments for enums

---
 src/IO/S3/ProviderType.h | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/IO/S3/ProviderType.h b/src/IO/S3/ProviderType.h
index eab36da1d30..e1b79597a3e 100644
--- a/src/IO/S3/ProviderType.h
+++ b/src/IO/S3/ProviderType.h
@@ -10,6 +10,11 @@
 namespace DB::S3
 {
 
+/// Provider type defines the platform containing the object
+/// we are trying to access
+/// This information is useful for determining general support for
+/// some feature like multipart copy which is currently supported by AWS
+/// but not by GCS
 enum class ProviderType : uint8_t
 {
     AWS,
@@ -19,6 +24,13 @@ enum class ProviderType : uint8_t
 
 std::string_view toString(ProviderType provider_type);
 
+/// Mode in which we can use the XML API
+/// This value can be same as the provider type but there can be a difference
+/// For example, GCS can work in both
+/// AWS compatible mode (accept headers starting with x-amz)
+/// and GCS mode (accept only headers starting with x-goog)
+/// Because GCS mode is enforced when some features are used we
+/// need to have support for both.
 enum class ApiMode : uint8_t
 {
     AWS,

From 02520e93584de6d0acb55faa6d87e3561438bed2 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 5 May 2023 17:37:52 +0000
Subject: [PATCH 192/689] Make EXPRECTED_TO_FAIL red

---
 tests/ci/report.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/report.py b/tests/ci/report.py
index cdef8409e7e..2d7090f878b 100644
--- a/tests/ci/report.py
+++ b/tests/ci/report.py
@@ -281,7 +281,7 @@ def _format_header(
 
 def _get_status_style(status: str, colortheme: Optional[ColorTheme] = None) -> str:
     ok_statuses = ("OK", "success", "PASSED")
-    fail_statuses = ("FAIL", "failure", "error", "FAILED", "Timeout")
+    fail_statuses = ("FAIL", "failure", "error", "FAILED", "Timeout", "EXPECTED_TO_FAIL")
 
     if colortheme is None:
         colortheme = ReportColorTheme.default

From 0baaaaba4210b71c237a8d2cb7f98fab0fd6faf5 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 5 May 2023 17:43:54 +0000
Subject: [PATCH 193/689] KNOWN_BROKEN -> BROKEN

---
 docker/test/util/process_functional_tests_result.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py
index ad544136a9a..3e59fc1d310 100755
--- a/docker/test/util/process_functional_tests_result.py
+++ b/docker/test/util/process_functional_tests_result.py
@@ -61,14 +61,14 @@ def process_test_log(log_path, broken_tests):
                 if TIMEOUT_SIGN in line:
                     if test_name in broken_tests:
                         success += 1
-                        test_results.append((test_name, "KNOWN_BROKEN", test_time, []))
+                        test_results.append((test_name, "BROKEN", test_time, []))
                     else:
                         failed += 1
                         test_results.append((test_name, "Timeout", test_time, []))
                 elif FAIL_SIGN in line:
                     if test_name in broken_tests:
                         success += 1
-                        test_results.append((test_name, "KNOWN_BROKEN", test_time, []))
+                        test_results.append((test_name, "BROKEN", test_time, []))
                     else:
                         failed += 1
                         test_results.append((test_name, "FAIL", test_time, []))

From a3b3b78eb0a7d10b32f68d8e6012511e43d66f3c Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 5 May 2023 17:53:15 +0000
Subject: [PATCH 194/689] EXPECTED_TO_FAIL -> NOT_FAILED

---
 docker/test/util/process_functional_tests_result.py | 2 +-
 tests/ci/report.py                                  | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py
index 3e59fc1d310..c75a3500831 100755
--- a/docker/test/util/process_functional_tests_result.py
+++ b/docker/test/util/process_functional_tests_result.py
@@ -84,7 +84,7 @@ def process_test_log(log_path, broken_tests):
                         test_results.append(
                             (
                                 test_name,
-                                "EXPECTED_TO_FAIL",
+                                "NOT_FAILED",
                                 test_time,
                                 ["This test passed. Update broken_tests.txt.\n"],
                             )
diff --git a/tests/ci/report.py b/tests/ci/report.py
index 2d7090f878b..89422298083 100644
--- a/tests/ci/report.py
+++ b/tests/ci/report.py
@@ -281,7 +281,7 @@ def _format_header(
 
 def _get_status_style(status: str, colortheme: Optional[ColorTheme] = None) -> str:
     ok_statuses = ("OK", "success", "PASSED")
-    fail_statuses = ("FAIL", "failure", "error", "FAILED", "Timeout", "EXPECTED_TO_FAIL")
+    fail_statuses = ("FAIL", "failure", "error", "FAILED", "Timeout", "NOT_FAILED")
 
     if colortheme is None:
         colortheme = ReportColorTheme.default
@@ -348,8 +348,8 @@ def create_test_html_report(
                 has_log_urls = True
 
             row = "<tr>"
-            is_fail = test_result.status in ("FAIL", "FLAKY")
-            if is_fail and test_result.raw_logs is not None:
+            has_error = test_result.status in ("FAIL", "FLAKY", "NOT_FAILED")
+            if has_error and test_result.raw_logs is not None:
                 row = '<tr class="failed">'
             row += "<td>" + test_result.name + "</td>"
             colspan += 1
@@ -357,7 +357,7 @@ def create_test_html_report(
 
             # Allow to quickly scroll to the first failure.
             fail_id = ""
-            if is_fail:
+            if has_error:
                 num_fails = num_fails + 1
                 fail_id = f'id="fail{num_fails}" '
 

From bba85c9c255f060f0b80d5594a5fd2651dfafde9 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 5 May 2023 18:11:33 +0000
Subject: [PATCH 195/689] Improve docs

---
 src/Interpreters/Cache/QueryCache.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Interpreters/Cache/QueryCache.h b/src/Interpreters/Cache/QueryCache.h
index 4736dc62fb1..bafa78f13d5 100644
--- a/src/Interpreters/Cache/QueryCache.h
+++ b/src/Interpreters/Cache/QueryCache.h
@@ -142,11 +142,13 @@ public:
         friend class QueryCache; /// for createWriter()
     };
 
-    /// Looks up a query result for a key in the cache and (if found) constructs a pipe with the query result chunks as source.
+    /// Reader's constructor looks up a query result for a key in the cache. If found, it constructs source processors (that generate the
+    /// cached result) for use in a pipe or query pipeline.
     class Reader
     {
     public:
         bool hasCacheEntryForKey() const;
+        /// getSource*() moves source processors out of the Reader. Call each of these method just once.
         std::unique_ptr<SourceFromChunks> getSource();
         std::unique_ptr<SourceFromChunks> getSourceTotals();
         std::unique_ptr<SourceFromChunks> getSourceExtremes();

From 2536dfad3a5ac3e99976e4fb00d8ed62e22da946 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 5 May 2023 18:21:25 +0000
Subject: [PATCH 196/689] Fix build

---
 src/Server/TCPHandler.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index 4e41fbf4230..0522b6d8a48 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -505,7 +505,7 @@ void TCPHandler::runImpl()
             /// the MemoryTracker will be wrong for possible deallocations.
             /// (i.e. deallocations from the Aggregator with two-level aggregation)
             state.reset();
-            last_sent_snapshots = {};
+            last_sent_snapshots = ProfileEvents::ThreadIdToCountersSnapshot{};
             query_scope.reset();
             thread_trace_context.reset();
         }

From 5e2221bc5df10b2115367e42077f9107b677c70b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 5 May 2023 21:36:10 +0300
Subject: [PATCH 197/689] Update play.html

---
 programs/server/play.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/programs/server/play.html b/programs/server/play.html
index 193d824d594..3ee133f0e01 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -536,6 +536,7 @@
     const pass_from_url = current_url.searchParams.get('password');
     if (pass_from_url) {
         document.getElementById('password').value = pass_from_url;
+	/// Browsers don't allow manipulating history for the 'file:' protocol.
         if (!opened_locally) {
             let replaced_pass = current_url.searchParams;
             replaced_pass.delete('password');

From d29193da512c5a899a31295ada6520e93848ad33 Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Fri, 5 May 2023 18:37:49 +0000
Subject: [PATCH 198/689] Also prohibit complex types with Object('json')

---
 src/Storages/AlterCommands.cpp                         | 10 +++++-----
 .../queries/0_stateless/02725_object_column_alter.sql  |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp
index 6cc427f9ee7..0f5fbb1e4fe 100644
--- a/src/Storages/AlterCommands.cpp
+++ b/src/Storages/AlterCommands.cpp
@@ -1120,15 +1120,15 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const
             /// The change of data type to/from Object is broken, so disable it for now
             if (command.data_type)
             {
-                GetColumnsOptions options(GetColumnsOptions::AllPhysical);
-                auto old_data_type = all_columns.getColumn(options, column_name).type;
+                const GetColumnsOptions options(GetColumnsOptions::AllPhysical);
+                const auto old_data_type = all_columns.getColumn(options, column_name).type;
 
-                if (WhichDataType(command.data_type).isObject()
-                    || WhichDataType(old_data_type).isObject())
+                if (command.data_type->getName().contains("Object")
+                    || old_data_type->getName().contains("Object"))
                     throw Exception(
                         ErrorCodes::BAD_ARGUMENTS,
                         "The change of data type {} of column {} to {} is not allowed",
-                        old_data_type->getFamilyName(), backQuote(column_name), command.data_type->getFamilyName());
+                        old_data_type->getName(), backQuote(column_name), command.data_type->getName());
             }
 
             if (command.isRemovingProperty())
diff --git a/tests/queries/0_stateless/02725_object_column_alter.sql b/tests/queries/0_stateless/02725_object_column_alter.sql
index 436640537fb..0e41070742a 100644
--- a/tests/queries/0_stateless/02725_object_column_alter.sql
+++ b/tests/queries/0_stateless/02725_object_column_alter.sql
@@ -12,4 +12,4 @@ ALTER TABLE t_to MODIFY COLUMN value Object('json'); -- { serverError BAD_ARGUME
 ALTER TABLE t_from MODIFY COLUMN value Nullable(String); -- { serverError BAD_ARGUMENTS }
 
 DROP TABLE t_to;
-DROP TABLE t_from;
\ No newline at end of file
+DROP TABLE t_from;

From 846abe95e902361b82336f64762f2b2720e8e7b6 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 5 May 2023 20:50:13 +0200
Subject: [PATCH 199/689] fix

---
 src/Databases/DDLDependencyVisitor.cpp    |  4 ++-
 src/Dictionaries/ExternalQueryBuilder.cpp | 33 ++++++++++++-----------
 src/Dictionaries/ExternalQueryBuilder.h   |  1 +
 3 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp
index 31d636a22f8..fc0cda03484 100644
--- a/src/Databases/DDLDependencyVisitor.cpp
+++ b/src/Databases/DDLDependencyVisitor.cpp
@@ -1,5 +1,6 @@
 #include <Databases/DDLDependencyVisitor.h>
 #include <Dictionaries/getDictionaryConfigurationFromAST.h>
+#include <Dictionaries/ExternalQueryBuilder.h>
 #include <Interpreters/Cluster.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/misc.h>
@@ -442,7 +443,8 @@ namespace
         {
             ParserSelectWithUnionQuery parser;
             String description = fmt::format("Query for ClickHouse dictionary {}", data.table_name);
-            ASTPtr select = parseQuery(parser, query, description,
+            String fixed_query = removeWhereConditionPlaceholder(query);
+            ASTPtr select = parseQuery(parser, fixed_query, description,
                                        data.context->getSettingsRef().max_query_size, data.context->getSettingsRef().max_parser_depth);
 
             DDLDependencyVisitor::Visitor visitor{data};
diff --git a/src/Dictionaries/ExternalQueryBuilder.cpp b/src/Dictionaries/ExternalQueryBuilder.cpp
index 19dabe92d64..05eaf1576b9 100644
--- a/src/Dictionaries/ExternalQueryBuilder.cpp
+++ b/src/Dictionaries/ExternalQueryBuilder.cpp
@@ -82,23 +82,26 @@ std::string ExternalQueryBuilder::composeLoadAllQuery() const
         writeChar(';', out);
         return out.str();
     }
-    else
-    {
-        /** In case UPDATE_FIELD is specified in {condition} for dictionary that must load all data.
-          * Replace {condition} with true_condition for initial dictionary load.
-          * For next dictionary loads {condition} will be updated with UPDATE_FIELD.
-          */
-        static constexpr auto true_condition = "(1 = 1)";
-        auto condition_position = query.find(CONDITION_PLACEHOLDER_TO_REPLACE_VALUE);
-        if (condition_position != std::string::npos)
-        {
-            auto query_copy = query;
-            query_copy.replace(condition_position, CONDITION_PLACEHOLDER_TO_REPLACE_VALUE.size(), true_condition);
-            return query_copy;
-        }
 
-        return query;
+    return removeWhereConditionPlaceholder(query);
+}
+
+String removeWhereConditionPlaceholder(const String  & query)
+{
+    /** In case UPDATE_FIELD is specified in {condition} for dictionary that must load all data.
+      * Replace {condition} with true_condition for initial dictionary load.
+      * For next dictionary loads {condition} will be updated with UPDATE_FIELD.
+      */
+    static constexpr auto true_condition = "(1 = 1)";
+    auto condition_position = query.find(CONDITION_PLACEHOLDER_TO_REPLACE_VALUE);
+    if (condition_position != std::string::npos)
+    {
+        auto query_copy = query;
+        query_copy.replace(condition_position, CONDITION_PLACEHOLDER_TO_REPLACE_VALUE.size(), true_condition);
+        return query_copy;
     }
+
+    return query;
 }
 
 void ExternalQueryBuilder::composeLoadAllQuery(WriteBuffer & out) const
diff --git a/src/Dictionaries/ExternalQueryBuilder.h b/src/Dictionaries/ExternalQueryBuilder.h
index 353367b12b6..9521019d51e 100644
--- a/src/Dictionaries/ExternalQueryBuilder.h
+++ b/src/Dictionaries/ExternalQueryBuilder.h
@@ -12,6 +12,7 @@ namespace DB
 struct DictionaryStructure;
 class WriteBuffer;
 
+String removeWhereConditionPlaceholder(const String  & query);
 
 /** Builds a query to load data from external database.
   */

From 223adc1725c6788a932a8956ef06b29d78591b07 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Fri, 5 May 2023 19:13:50 +0000
Subject: [PATCH 200/689] fix ThreadPool::wait

---
 src/Common/ThreadPool.cpp                     |  3 +++
 .../gtest_thread_pool_schedule_exception.cpp  | 25 +++++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp
index 5d27f71d8c4..63eef72dffb 100644
--- a/src/Common/ThreadPool.cpp
+++ b/src/Common/ThreadPool.cpp
@@ -397,7 +397,10 @@ void ThreadPoolImpl<Thread>::worker(typename std::list<Thread>::iterator thread_
 
             /// We don't run jobs after `shutdown` is set, but we have to properly dequeue all jobs and finish them.
             if (shutdown)
+            {
+                job_is_done = true;
                 continue;
+            }
         }
 
         ALLOW_ALLOCATIONS_IN_SCOPE;
diff --git a/src/Common/tests/gtest_thread_pool_schedule_exception.cpp b/src/Common/tests/gtest_thread_pool_schedule_exception.cpp
index 176e469d5ef..fd77a95546b 100644
--- a/src/Common/tests/gtest_thread_pool_schedule_exception.cpp
+++ b/src/Common/tests/gtest_thread_pool_schedule_exception.cpp
@@ -51,3 +51,28 @@ TEST(ThreadPool, ExceptionFromSchedule)
 {
     EXPECT_TRUE(check());
 }
+
+static bool check2()
+{
+    ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 2);
+
+    pool.scheduleOrThrowOnError([&]{ throw std::runtime_error("Hello, world!"); });
+    pool.scheduleOrThrowOnError([]{});
+
+    try
+    {
+        pool.wait();
+    }
+    catch (const std::runtime_error &)
+    {
+        return true;
+    }
+
+    return false;
+}
+
+TEST(ThreadPool, ExceptionFromWait)
+{
+    for (size_t i = 0; i < 1000; ++i)
+        EXPECT_TRUE(check2());
+}

From 9cd81400f325b379fbb304c7e0da51c7e86e1143 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 5 May 2023 19:19:00 +0000
Subject: [PATCH 201/689] Query cache: disable for internal queries

---
 src/Interpreters/executeQuery.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index 7852f4cefa8..238c72e3e17 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -644,7 +644,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
         }
 
         bool can_use_query_cache =
-            settings.allow_experimental_query_cache && settings.use_query_cache
+            settings.allow_experimental_query_cache && settings.use_query_cache && !internal
             && !ast->as<ASTExplainQuery>();
 
         if (!async_insert)

From 9436aef8931cce03e9f2eacf495f8bdec0a6781e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 5 May 2023 22:44:07 +0200
Subject: [PATCH 202/689] Remove test_merge_tree_s3_restore

---
 .../test_merge_tree_s3_restore/__init__.py    |   0
 .../config.d/bg_processing_pool_conf.xml      |   5 -
 .../configs/config.d/clusters.xml             |  22 -
 .../configs/config.d/storage_conf.xml         |  34 -
 .../config.d/storage_conf_another_bucket.xml  |  34 -
 .../storage_conf_another_bucket_path.xml      |  34 -
 .../config.d/storage_conf_not_restorable.xml  |  35 -
 .../configs/config.d/users.xml                |   5 -
 .../configs/config.xml                        |  18 -
 .../test_merge_tree_s3_restore/test.py        | 696 ------------------
 10 files changed, 883 deletions(-)
 delete mode 100644 tests/integration/test_merge_tree_s3_restore/__init__.py
 delete mode 100644 tests/integration/test_merge_tree_s3_restore/configs/config.d/bg_processing_pool_conf.xml
 delete mode 100644 tests/integration/test_merge_tree_s3_restore/configs/config.d/clusters.xml
 delete mode 100644 tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf.xml
 delete mode 100644 tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml
 delete mode 100644 tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket_path.xml
 delete mode 100644 tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf_not_restorable.xml
 delete mode 100644 tests/integration/test_merge_tree_s3_restore/configs/config.d/users.xml
 delete mode 100644 tests/integration/test_merge_tree_s3_restore/configs/config.xml
 delete mode 100644 tests/integration/test_merge_tree_s3_restore/test.py

diff --git a/tests/integration/test_merge_tree_s3_restore/__init__.py b/tests/integration/test_merge_tree_s3_restore/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_merge_tree_s3_restore/configs/config.d/bg_processing_pool_conf.xml b/tests/integration/test_merge_tree_s3_restore/configs/config.d/bg_processing_pool_conf.xml
deleted file mode 100644
index e45b647fd24..00000000000
--- a/tests/integration/test_merge_tree_s3_restore/configs/config.d/bg_processing_pool_conf.xml
+++ /dev/null
@@ -1,5 +0,0 @@
-<clickhouse>
-    <background_processing_pool_thread_sleep_seconds>0.5</background_processing_pool_thread_sleep_seconds>
-    <background_processing_pool_task_sleep_seconds_when_no_work_min>0.5</background_processing_pool_task_sleep_seconds_when_no_work_min>
-    <background_processing_pool_task_sleep_seconds_when_no_work_max>0.5</background_processing_pool_task_sleep_seconds_when_no_work_max>
-</clickhouse>
diff --git a/tests/integration/test_merge_tree_s3_restore/configs/config.d/clusters.xml b/tests/integration/test_merge_tree_s3_restore/configs/config.d/clusters.xml
deleted file mode 100644
index d02d5ab0bf5..00000000000
--- a/tests/integration/test_merge_tree_s3_restore/configs/config.d/clusters.xml
+++ /dev/null
@@ -1,22 +0,0 @@
-<clickhouse>
-    <remote_servers>
-        <node>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>node</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </node>
-        <node_another_bucket>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>node_another_bucket</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </node_another_bucket>
-    </remote_servers>
-</clickhouse>
diff --git a/tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf.xml
deleted file mode 100644
index 9dfcf1e2f9b..00000000000
--- a/tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<clickhouse>
-    <storage_configuration>
-        <disks>
-            <s3>
-                <type>s3</type>
-                <endpoint>http://minio1:9001/root/data/</endpoint>
-                <access_key_id>minio</access_key_id>
-                <secret_access_key>minio123</secret_access_key>
-                <send_metadata>true</send_metadata>
-                <list_object_keys_size>1</list_object_keys_size> <!-- To effectively test restore parallelism -->
-            </s3>
-            <hdd>
-                <type>local</type>
-                <path>/</path>
-            </hdd>
-        </disks>
-        <policies>
-            <s3>
-                <volumes>
-                    <main>
-                        <disk>s3</disk>
-                    </main>
-                    <external>
-                        <disk>hdd</disk>
-                    </external>
-                </volumes>
-            </s3>
-        </policies>
-    </storage_configuration>
-
-    <merge_tree>
-        <min_bytes_for_wide_part>0</min_bytes_for_wide_part>
-    </merge_tree>
-</clickhouse>
diff --git a/tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml b/tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml
deleted file mode 100644
index 920db2c9edd..00000000000
--- a/tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<clickhouse>
-    <storage_configuration>
-        <disks>
-            <s3>
-                <type>s3</type>
-                <endpoint>http://minio1:9001/root2/data/</endpoint>
-                <access_key_id>minio</access_key_id>
-                <secret_access_key>minio123</secret_access_key>
-                <send_metadata>true</send_metadata>
-                <list_object_keys_size>1</list_object_keys_size> <!-- To effectively test restore parallelism -->
-            </s3>
-            <hdd>
-                <type>local</type>
-                <path>/</path>
-            </hdd>
-        </disks>
-        <policies>
-            <s3>
-                <volumes>
-                    <main>
-                        <disk>s3</disk>
-                    </main>
-                    <external>
-                        <disk>hdd</disk>
-                    </external>
-                </volumes>
-            </s3>
-        </policies>
-    </storage_configuration>
-
-    <merge_tree>
-        <min_bytes_for_wide_part>0</min_bytes_for_wide_part>
-    </merge_tree>
-</clickhouse>
diff --git a/tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket_path.xml b/tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket_path.xml
deleted file mode 100644
index 40cdf0f3402..00000000000
--- a/tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket_path.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<clickhouse>
-    <storage_configuration>
-        <disks>
-            <s3>
-                <type>s3</type>
-                <endpoint>http://minio1:9001/root2/another_data/</endpoint>
-                <access_key_id>minio</access_key_id>
-                <secret_access_key>minio123</secret_access_key>
-                <send_metadata>true</send_metadata>
-                <list_object_keys_size>1</list_object_keys_size> <!-- To effectively test restore parallelism -->
-            </s3>
-            <hdd>
-                <type>local</type>
-                <path>/</path>
-            </hdd>
-        </disks>
-        <policies>
-            <s3>
-                <volumes>
-                    <main>
-                        <disk>s3</disk>
-                    </main>
-                    <external>
-                        <disk>hdd</disk>
-                    </external>
-                </volumes>
-            </s3>
-        </policies>
-    </storage_configuration>
-
-    <merge_tree>
-        <min_bytes_for_wide_part>0</min_bytes_for_wide_part>
-    </merge_tree>
-</clickhouse>
diff --git a/tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf_not_restorable.xml b/tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf_not_restorable.xml
deleted file mode 100644
index 4f21fa93381..00000000000
--- a/tests/integration/test_merge_tree_s3_restore/configs/config.d/storage_conf_not_restorable.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<clickhouse>
-    <storage_configuration>
-        <disks>
-            <s3>
-                <type>s3</type>
-                <endpoint>http://minio1:9001/root/another_data/</endpoint>
-                <access_key_id>minio</access_key_id>
-                <secret_access_key>minio123</secret_access_key>
-                <send_metadata>false</send_metadata>
-                <list_object_keys_size>1</list_object_keys_size> <!-- To effectively test restore parallelism -->
-                <retry_attempts>0</retry_attempts>
-            </s3>
-            <hdd>
-                <type>local</type>
-                <path>/</path>
-            </hdd>
-        </disks>
-        <policies>
-            <s3>
-                <volumes>
-                    <main>
-                        <disk>s3</disk>
-                    </main>
-                    <external>
-                        <disk>hdd</disk>
-                    </external>
-                </volumes>
-            </s3>
-        </policies>
-    </storage_configuration>
-
-    <merge_tree>
-        <min_bytes_for_wide_part>0</min_bytes_for_wide_part>
-    </merge_tree>
-</clickhouse>
diff --git a/tests/integration/test_merge_tree_s3_restore/configs/config.d/users.xml b/tests/integration/test_merge_tree_s3_restore/configs/config.d/users.xml
deleted file mode 100644
index 0011583a68c..00000000000
--- a/tests/integration/test_merge_tree_s3_restore/configs/config.d/users.xml
+++ /dev/null
@@ -1,5 +0,0 @@
-<clickhouse>
-    <profiles>
-        <default/>
-    </profiles>
-</clickhouse>
diff --git a/tests/integration/test_merge_tree_s3_restore/configs/config.xml b/tests/integration/test_merge_tree_s3_restore/configs/config.xml
deleted file mode 100644
index feb537ebbce..00000000000
--- a/tests/integration/test_merge_tree_s3_restore/configs/config.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-<clickhouse>
-    <tcp_port>9000</tcp_port>
-    <listen_host>127.0.0.1</listen_host>
-
-    <openSSL>
-        <client>
-            <cacheSessions>true</cacheSessions>
-            <verificationMode>none</verificationMode>
-            <invalidCertificateHandler>
-                <name>AcceptCertificateHandler</name>
-            </invalidCertificateHandler>
-        </client>
-    </openSSL>
-
-    <max_concurrent_queries>500</max_concurrent_queries>
-    <path>./clickhouse/</path>
-    <users_config>users.xml</users_config>
-</clickhouse>
diff --git a/tests/integration/test_merge_tree_s3_restore/test.py b/tests/integration/test_merge_tree_s3_restore/test.py
deleted file mode 100644
index ffb0d9477cf..00000000000
--- a/tests/integration/test_merge_tree_s3_restore/test.py
+++ /dev/null
@@ -1,696 +0,0 @@
-import os
-import logging
-import random
-import string
-import time
-
-import pytest
-from helpers.cluster import ClickHouseCluster
-from helpers.wait_for_helpers import wait_for_delete_empty_parts
-from helpers.wait_for_helpers import wait_for_delete_inactive_parts
-
-
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-COMMON_CONFIGS = [
-    "configs/config.d/bg_processing_pool_conf.xml",
-    "configs/config.d/clusters.xml",
-]
-
-
-def replace_config(path, old, new):
-    config = open(path, "r")
-    config_lines = config.readlines()
-    config.close()
-    config_lines = [line.replace(old, new) for line in config_lines]
-    config = open(path, "w")
-    config.writelines(config_lines)
-    config.close()
-
-
-@pytest.fixture(scope="module")
-def cluster():
-    try:
-        cluster = ClickHouseCluster(__file__)
-
-        cluster.add_instance(
-            "node",
-            main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf.xml"],
-            macros={"cluster": "node", "replica": "0"},
-            with_minio=True,
-            with_zookeeper=True,
-            stay_alive=True,
-        )
-        cluster.add_instance(
-            "node_another_bucket",
-            main_configs=COMMON_CONFIGS
-            + ["configs/config.d/storage_conf_another_bucket.xml"],
-            macros={"cluster": "node_another_bucket", "replica": "0"},
-            with_zookeeper=True,
-            stay_alive=True,
-        )
-        cluster.add_instance(
-            "node_another_bucket_path",
-            main_configs=COMMON_CONFIGS
-            + ["configs/config.d/storage_conf_another_bucket_path.xml"],
-            stay_alive=True,
-        )
-        cluster.add_instance(
-            "node_not_restorable",
-            main_configs=COMMON_CONFIGS
-            + ["configs/config.d/storage_conf_not_restorable.xml"],
-            stay_alive=True,
-        )
-
-        logging.info("Starting cluster...")
-        cluster.start()
-        logging.info("Cluster started")
-
-        yield cluster
-    finally:
-        cluster.shutdown()
-
-
-def random_string(length):
-    letters = string.ascii_letters
-    return "".join(random.choice(letters) for i in range(length))
-
-
-def generate_values(date_str, count, sign=1):
-    data = [[date_str, sign * (i + 1), random_string(10)] for i in range(count)]
-    data.sort(key=lambda tup: tup[1])
-    return ",".join(["('{}',{},'{}',{})".format(x, y, z, 0) for x, y, z in data])
-
-
-def create_table(
-    node, table_name, attach=False, replicated=False, db_atomic=False, uuid=""
-):
-    node.query("DROP DATABASE IF EXISTS s3")
-
-    node.query(
-        "CREATE DATABASE IF NOT EXISTS s3 ENGINE = {engine}".format(
-            engine="Atomic" if db_atomic else "Ordinary"
-        ),
-        settings={"allow_deprecated_database_ordinary": 1},
-    )
-
-    create_table_statement = """
-        {create} TABLE s3.{table_name} {uuid} {on_cluster} (
-            dt Date,
-            id Int64,
-            data String,
-            counter Int64,
-            INDEX min_max (id) TYPE minmax GRANULARITY 3
-        ) ENGINE={engine}
-        PARTITION BY dt
-        ORDER BY (dt, id)
-        SETTINGS
-            storage_policy='s3',
-            index_granularity=512,
-            old_parts_lifetime=1
-        """.format(
-        create="ATTACH" if attach else "CREATE",
-        table_name=table_name,
-        uuid="UUID '{uuid}'".format(uuid=uuid) if db_atomic and uuid else "",
-        on_cluster="ON CLUSTER '{}'".format(node.name) if replicated else "",
-        engine="ReplicatedMergeTree('/clickhouse/tables/{cluster}/test', '{replica}')"
-        if replicated
-        else "MergeTree()",
-    )
-
-    node.query(create_table_statement)
-
-
-def purge_s3(cluster, bucket):
-    minio = cluster.minio_client
-    for obj in list(minio.list_objects(bucket, recursive=True)):
-        if str(obj.object_name).find(".SCHEMA_VERSION") != -1:
-            continue
-        minio.remove_object(bucket, obj.object_name)
-
-
-def drop_s3_metadata(node):
-    node.exec_in_container(
-        ["bash", "-c", "rm -rf /var/lib/clickhouse/disks/s3/*"], user="root"
-    )
-
-
-def drop_shadow_information(node):
-    node.exec_in_container(
-        ["bash", "-c", "rm -rf /var/lib/clickhouse/shadow/*"], user="root"
-    )
-
-
-def create_restore_file(node, revision=None, bucket=None, path=None, detached=None):
-    node.exec_in_container(
-        ["bash", "-c", "mkdir -p /var/lib/clickhouse/disks/s3/"], user="root"
-    )
-
-    node.exec_in_container(
-        ["bash", "-c", "touch /var/lib/clickhouse/disks/s3/restore"], user="root"
-    )
-
-    add_restore_option = 'echo -en "{}={}\n" >> /var/lib/clickhouse/disks/s3/restore'
-    if revision:
-        node.exec_in_container(
-            ["bash", "-c", add_restore_option.format("revision", revision)], user="root"
-        )
-    if bucket:
-        node.exec_in_container(
-            ["bash", "-c", add_restore_option.format("source_bucket", bucket)],
-            user="root",
-        )
-    if path:
-        node.exec_in_container(
-            ["bash", "-c", add_restore_option.format("source_path", path)], user="root"
-        )
-    if detached:
-        node.exec_in_container(
-            ["bash", "-c", add_restore_option.format("detached", "true")], user="root"
-        )
-
-
-def get_revision_counter(node, backup_number):
-    return int(
-        node.exec_in_container(
-            [
-                "bash",
-                "-c",
-                "cat /var/lib/clickhouse/disks/s3/shadow/{}/revision.txt".format(
-                    backup_number
-                ),
-            ],
-            user="root",
-        )
-    )
-
-
-def get_table_uuid(node, db_atomic, table):
-    uuid = ""
-    if db_atomic:
-        uuid = node.query(
-            "SELECT uuid FROM system.tables WHERE database='s3' AND table='{}' FORMAT TabSeparated".format(
-                table
-            )
-        ).strip()
-    return uuid
-
-
-@pytest.fixture(autouse=True)
-def drop_table(cluster):
-    yield
-
-    node_names = [
-        "node",
-        "node_another_bucket",
-        "node_another_bucket_path",
-        "node_not_restorable",
-    ]
-
-    for node_name in node_names:
-        node = cluster.instances[node_name]
-        node.query("DROP TABLE IF EXISTS s3.test SYNC")
-        node.query("DROP DATABASE IF EXISTS s3 SYNC")
-
-        drop_s3_metadata(node)
-        drop_shadow_information(node)
-
-    buckets = [cluster.minio_bucket, cluster.minio_bucket_2]
-    for bucket in buckets:
-        purge_s3(cluster, bucket)
-
-
-@pytest.mark.parametrize("replicated", [False, True])
-@pytest.mark.parametrize("db_atomic", [False, True])
-def test_full_restore(cluster, replicated, db_atomic):
-    node = cluster.instances["node"]
-
-    create_table(node, "test", attach=False, replicated=replicated, db_atomic=db_atomic)
-
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-04", 4096, -1))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096, -1))
-    )
-
-    node.query("DETACH TABLE s3.test")
-    drop_s3_metadata(node)
-    create_restore_file(node)
-    node.restart_clickhouse()
-
-    assert node.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(
-        4096 * 4
-    )
-    assert node.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0)
-
-
-@pytest.mark.parametrize("db_atomic", [False, True])
-def test_restore_another_bucket_path(cluster, db_atomic):
-    node = cluster.instances["node"]
-
-    create_table(node, "test", db_atomic=db_atomic)
-    uuid = get_table_uuid(node, db_atomic, "test")
-
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-04", 4096, -1))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096, -1))
-    )
-
-    # To ensure parts have merged
-    node.query("OPTIMIZE TABLE s3.test")
-    wait_for_delete_inactive_parts(node, "s3.test", retry_count=120)
-
-    assert node.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(
-        4096 * 4
-    )
-    assert node.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0)
-
-    node_another_bucket = cluster.instances["node_another_bucket"]
-
-    create_restore_file(node_another_bucket, bucket="root")
-    node_another_bucket.restart_clickhouse()
-    create_table(
-        node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid
-    )
-
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 4)
-    assert node_another_bucket.query(
-        "SELECT sum(id) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-
-    node_another_bucket_path = cluster.instances["node_another_bucket_path"]
-
-    create_restore_file(node_another_bucket_path, bucket="root2", path="data")
-    node_another_bucket_path.restart_clickhouse()
-    create_table(
-        node_another_bucket_path, "test", attach=True, db_atomic=db_atomic, uuid=uuid
-    )
-
-    assert node_another_bucket_path.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 4)
-    assert node_another_bucket_path.query(
-        "SELECT sum(id) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-
-
-@pytest.mark.parametrize("db_atomic", [False, True])
-def test_restore_different_revisions(cluster, db_atomic):
-    node = cluster.instances["node"]
-
-    create_table(node, "test", db_atomic=db_atomic)
-    uuid = get_table_uuid(node, db_atomic, "test")
-
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-04", 4096, -1))
-    )
-
-    node.query("ALTER TABLE s3.test FREEZE")
-    revision1 = get_revision_counter(node, 1)
-
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096, -1))
-    )
-
-    node.query("ALTER TABLE s3.test FREEZE")
-    revision2 = get_revision_counter(node, 2)
-
-    # To ensure parts have merged
-    node.query("OPTIMIZE TABLE s3.test")
-    wait_for_delete_inactive_parts(node, "s3.test", retry_count=120)
-
-    assert node.query("SELECT count(*) from system.parts where table = 'test'") == "3\n"
-
-    node.query("ALTER TABLE s3.test FREEZE")
-    revision3 = get_revision_counter(node, 3)
-
-    assert node.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(
-        4096 * 4
-    )
-    assert node.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0)
-    assert node.query("SELECT count(*) from system.parts where table = 'test'") == "3\n"
-
-    node_another_bucket = cluster.instances["node_another_bucket"]
-
-    # Restore to revision 1 (2 parts).
-    create_restore_file(node_another_bucket, revision=revision1, bucket="root")
-    node_another_bucket.restart_clickhouse()
-    create_table(
-        node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid
-    )
-
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 2)
-    assert node_another_bucket.query(
-        "SELECT sum(id) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-    assert (
-        node_another_bucket.query(
-            "SELECT count(*) from system.parts where table = 'test'"
-        )
-        == "2\n"
-    )
-
-    # Restore to revision 2 (4 parts).
-    node_another_bucket.query("DETACH TABLE s3.test")
-    create_restore_file(node_another_bucket, revision=revision2, bucket="root")
-    node_another_bucket.restart_clickhouse()
-
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 4)
-    assert node_another_bucket.query(
-        "SELECT sum(id) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-    assert (
-        node_another_bucket.query(
-            "SELECT count(*) from system.parts where table = 'test'"
-        )
-        == "4\n"
-    )
-
-    # Restore to revision 3 (4 parts + 1 merged).
-    node_another_bucket.query("DETACH TABLE s3.test")
-    create_restore_file(node_another_bucket, revision=revision3, bucket="root")
-    node_another_bucket.restart_clickhouse()
-
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 4)
-    assert node_another_bucket.query(
-        "SELECT sum(id) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-    assert (
-        node_another_bucket.query(
-            "SELECT count(*) from system.parts where table = 'test'"
-        )
-        == "3\n"
-    )
-
-
-@pytest.mark.parametrize("db_atomic", [False, True])
-def test_restore_mutations(cluster, db_atomic):
-    node = cluster.instances["node"]
-
-    create_table(node, "test", db_atomic=db_atomic)
-    uuid = get_table_uuid(node, db_atomic, "test")
-
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096, -1))
-    )
-
-    node.query("ALTER TABLE s3.test FREEZE")
-    revision_before_mutation = get_revision_counter(node, 1)
-
-    node.query(
-        "ALTER TABLE s3.test UPDATE counter = 1 WHERE 1", settings={"mutations_sync": 2}
-    )
-
-    node.query("ALTER TABLE s3.test FREEZE")
-    revision_after_mutation = get_revision_counter(node, 2)
-
-    node_another_bucket = cluster.instances["node_another_bucket"]
-
-    # Restore to revision before mutation.
-    create_restore_file(
-        node_another_bucket, revision=revision_before_mutation, bucket="root"
-    )
-    node_another_bucket.restart_clickhouse()
-    create_table(
-        node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid
-    )
-
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 2)
-    assert node_another_bucket.query(
-        "SELECT sum(id) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-    assert node_another_bucket.query(
-        "SELECT sum(counter) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-
-    # Restore to revision after mutation.
-    node_another_bucket.query("DETACH TABLE s3.test")
-    create_restore_file(
-        node_another_bucket, revision=revision_after_mutation, bucket="root"
-    )
-    node_another_bucket.restart_clickhouse()
-
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 2)
-    assert node_another_bucket.query(
-        "SELECT sum(id) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-    assert node_another_bucket.query(
-        "SELECT sum(counter) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 2)
-    assert node_another_bucket.query(
-        "SELECT sum(counter) FROM s3.test WHERE id > 0 FORMAT Values"
-    ) == "({})".format(4096)
-
-    # Restore to revision in the middle of mutation.
-    # Unfinished mutation should be completed after table startup.
-    node_another_bucket.query("DETACH TABLE s3.test")
-    revision = (revision_before_mutation + revision_after_mutation) // 2
-    create_restore_file(node_another_bucket, revision=revision, bucket="root")
-    node_another_bucket.restart_clickhouse()
-
-    # Wait for unfinished mutation completion.
-    time.sleep(3)
-
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 2)
-    assert node_another_bucket.query(
-        "SELECT sum(id) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-    assert node_another_bucket.query(
-        "SELECT sum(counter) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 2)
-    assert node_another_bucket.query(
-        "SELECT sum(counter) FROM s3.test WHERE id > 0 FORMAT Values"
-    ) == "({})".format(4096)
-
-
-def test_migrate_to_restorable_schema(cluster):
-    db_atomic = True
-    node = cluster.instances["node_not_restorable"]
-    config_path = os.path.join(
-        SCRIPT_DIR,
-        "./{}/node_not_restorable/configs/config.d/storage_conf_not_restorable.xml".format(
-            cluster.instances_dir_name
-        ),
-    )
-
-    create_table(node, "test", db_atomic=db_atomic)
-    uuid = get_table_uuid(node, db_atomic, "test")
-
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-04", 4096, -1))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096, -1))
-    )
-
-    replace_config(
-        config_path,
-        "<send_metadata>false</send_metadata>",
-        "<send_metadata>true</send_metadata>",
-    )
-    node.restart_clickhouse()
-
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-06", 4096))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-06", 4096, -1))
-    )
-
-    node.query("ALTER TABLE s3.test FREEZE")
-    revision = get_revision_counter(node, 1)
-
-    assert revision != 0
-
-    node_another_bucket = cluster.instances["node_another_bucket"]
-
-    # Restore to revision before mutation.
-    create_restore_file(
-        node_another_bucket, revision=revision, bucket="root", path="another_data"
-    )
-    node_another_bucket.restart_clickhouse()
-    create_table(
-        node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid
-    )
-
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 6)
-    assert node_another_bucket.query(
-        "SELECT sum(id) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-
-
-@pytest.mark.parametrize("replicated", [False, True])
-@pytest.mark.parametrize("db_atomic", [False, True])
-def test_restore_to_detached(cluster, replicated, db_atomic):
-    node = cluster.instances["node"]
-
-    create_table(node, "test", attach=False, replicated=replicated, db_atomic=db_atomic)
-    uuid = get_table_uuid(node, db_atomic, "test")
-
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-04", 4096, -1))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-06", 4096, -1))
-    )
-    node.query(
-        "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-07", 4096, 0))
-    )
-
-    # Add some mutation.
-    node.query(
-        "ALTER TABLE s3.test UPDATE counter = 1 WHERE 1", settings={"mutations_sync": 2}
-    )
-
-    # Detach some partition.
-    node.query("ALTER TABLE s3.test DETACH PARTITION '2020-01-07'")
-    wait_for_delete_empty_parts(node, "s3.test", retry_count=120)
-    wait_for_delete_inactive_parts(node, "s3.test", retry_count=120)
-
-    node.query("ALTER TABLE s3.test FREEZE")
-    revision = get_revision_counter(node, 1)
-
-    node_another_bucket = cluster.instances["node_another_bucket"]
-
-    create_restore_file(
-        node_another_bucket,
-        revision=revision,
-        bucket="root",
-        path="data",
-        detached=True,
-    )
-    node_another_bucket.restart_clickhouse()
-    create_table(
-        node_another_bucket,
-        "test",
-        replicated=replicated,
-        db_atomic=db_atomic,
-        uuid=uuid,
-    )
-
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-
-    node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-03'")
-    node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-04'")
-    node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-05'")
-    node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-06'")
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 4)
-
-    assert node_another_bucket.query(
-        "SELECT sum(id) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-    assert node_another_bucket.query(
-        "SELECT sum(counter) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 4)
-
-    # Attach partition that was already detached before backup-restore.
-    node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-07'")
-
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 5)
-    assert node_another_bucket.query(
-        "SELECT sum(id) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-    assert node_another_bucket.query(
-        "SELECT sum(counter) FROM s3.test FORMAT Values"
-    ) == "({})".format(4096 * 5)
-
-
-@pytest.mark.parametrize("replicated", [False, True])
-@pytest.mark.parametrize("db_atomic", [False, True])
-def test_restore_without_detached(cluster, replicated, db_atomic):
-    node = cluster.instances["node"]
-
-    create_table(node, "test", attach=False, replicated=replicated, db_atomic=db_atomic)
-    uuid = get_table_uuid(node, db_atomic, "test")
-
-    node.query("INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 1)))
-
-    assert node.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(1)
-
-    node.query("ALTER TABLE s3.test FREEZE")
-    revision = get_revision_counter(node, 1)
-
-    node_another_bucket = cluster.instances["node_another_bucket"]
-
-    create_restore_file(
-        node_another_bucket,
-        revision=revision,
-        bucket="root",
-        path="data",
-        detached=True,
-    )
-    node_another_bucket.restart_clickhouse()
-    create_table(
-        node_another_bucket,
-        "test",
-        replicated=replicated,
-        db_atomic=db_atomic,
-        uuid=uuid,
-    )
-
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(0)
-
-    node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-03'")
-
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(1)

From ed6e1a7289f2e42855576555ad63de5d99988805 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 5 May 2023 23:14:25 +0200
Subject: [PATCH 203/689] Update tests

---
 tests/queries/0_stateless/00098_k_union_all.sql               | 1 +
 tests/queries/0_stateless/00405_pretty_formats.sql            | 2 ++
 tests/queries/0_stateless/00730_unicode_terminal_format.sql   | 2 ++
 .../0_stateless/01065_window_view_event_hop_watch_bounded.py  | 4 ++--
 tests/queries/0_stateless/01082_window_view_watch_limit.py    | 4 ++--
 .../0_stateless/01509_output_format_pretty_row_numbers.sql    | 2 ++
 tests/queries/0_stateless/02375_pretty_formats.sql.j2         | 1 +
 7 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/00098_k_union_all.sql b/tests/queries/0_stateless/00098_k_union_all.sql
index 311e5bb19c4..8f396d3479a 100644
--- a/tests/queries/0_stateless/00098_k_union_all.sql
+++ b/tests/queries/0_stateless/00098_k_union_all.sql
@@ -1,3 +1,4 @@
+SET output_format_pretty_squash_ms = 0;
 SELECT 1 FORMAT PrettySpace;
 SELECT 1 UNION ALL SELECT 1 FORMAT PrettySpace;
 SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 FORMAT PrettySpace;
diff --git a/tests/queries/0_stateless/00405_pretty_formats.sql b/tests/queries/0_stateless/00405_pretty_formats.sql
index 3c8af776278..a7dc5aa0600 100644
--- a/tests/queries/0_stateless/00405_pretty_formats.sql
+++ b/tests/queries/0_stateless/00405_pretty_formats.sql
@@ -1,3 +1,5 @@
+SET output_format_pretty_squash_ms = 0;
+
 SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT Pretty;
 SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompact;
 SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettySpace;
diff --git a/tests/queries/0_stateless/00730_unicode_terminal_format.sql b/tests/queries/0_stateless/00730_unicode_terminal_format.sql
index b7a8084c406..6327b41a05a 100644
--- a/tests/queries/0_stateless/00730_unicode_terminal_format.sql
+++ b/tests/queries/0_stateless/00730_unicode_terminal_format.sql
@@ -1,5 +1,7 @@
 DROP TABLE IF EXISTS unicode;
 
+SET output_format_pretty_squash_ms = 0;
+
 CREATE TABLE unicode(c1 String, c2 String) ENGINE = Memory;
 INSERT INTO unicode VALUES ('Здравствуйте', 'Этот код можно отредактировать и запустить!');
 INSERT INTO unicode VALUES ('你好', '这段代码是可以编辑并且能够运行的！');
diff --git a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py
index 7f00130b184..42efbc8071b 100755
--- a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py
+++ b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py
@@ -19,13 +19,13 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.expect(prompt)
 
-    client1.send("SET allow_experimental_analyzer = 0")
+    client1.send("SET allow_experimental_analyzer = 0, output_format_pretty_squash_ms = 0;")
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
     client1.send("SET window_view_heartbeat_interval = 1")
     client1.expect(prompt)
-    client2.send("SET allow_experimental_window_view = 1")
+    client2.send("SET allow_experimental_window_view = 1, output_format_pretty_squash_ms = 0;")
     client2.expect(prompt)
 
     client1.send(
diff --git a/tests/queries/0_stateless/01082_window_view_watch_limit.py b/tests/queries/0_stateless/01082_window_view_watch_limit.py
index 12c8d295591..1d43555bddb 100755
--- a/tests/queries/0_stateless/01082_window_view_watch_limit.py
+++ b/tests/queries/0_stateless/01082_window_view_watch_limit.py
@@ -19,13 +19,13 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.expect(prompt)
 
-    client1.send("SET allow_experimental_analyzer = 0")
+    client1.send("SET allow_experimental_analyzer = 0, output_format_pretty_squash_ms = 0;")
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
     client1.send("SET window_view_heartbeat_interval = 1")
     client1.expect(prompt)
-    client2.send("SET allow_experimental_window_view = 1")
+    client2.send("SET allow_experimental_window_view = 1, output_format_pretty_squash_ms = 0;")
     client2.expect(prompt)
 
     client1.send("CREATE DATABASE IF NOT EXISTS 01082_window_view_watch_limit")
diff --git a/tests/queries/0_stateless/01509_output_format_pretty_row_numbers.sql b/tests/queries/0_stateless/01509_output_format_pretty_row_numbers.sql
index f8ec0be74d7..fb4c743d54b 100644
--- a/tests/queries/0_stateless/01509_output_format_pretty_row_numbers.sql
+++ b/tests/queries/0_stateless/01509_output_format_pretty_row_numbers.sql
@@ -1,3 +1,5 @@
+SET output_format_pretty_squash_ms = 0;
+
 SELECT * FROM numbers(10) FORMAT Pretty;
 SELECT * FROM numbers(10) FORMAT PrettyCompact;
 SELECT * FROM numbers(10) FORMAT PrettyCompactMonoBlock;
diff --git a/tests/queries/0_stateless/02375_pretty_formats.sql.j2 b/tests/queries/0_stateless/02375_pretty_formats.sql.j2
index cc61346d267..92e0a357138 100644
--- a/tests/queries/0_stateless/02375_pretty_formats.sql.j2
+++ b/tests/queries/0_stateless/02375_pretty_formats.sql.j2
@@ -3,6 +3,7 @@
                   'PrettySpaceNoEscapesMonoBlock'] -%}
 
 select '{{ format }}';
+SET output_format_pretty_squash_ms = 0;
 select number as x, number + 1 as y from numbers(4) settings max_block_size=2 format {{ format }};
 
 {% endfor -%}

From 8a548170f850d2bece6f047271833683767a6c03 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 5 May 2023 23:25:47 +0200
Subject: [PATCH 204/689] Remove obsolete test

---
 .../test_detach_part_wrong_partition_id.py    | 46 -------------------
 1 file changed, 46 deletions(-)
 delete mode 100644 tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py

diff --git a/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py b/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py
deleted file mode 100644
index a6f7a8653da..00000000000
--- a/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import pytest
-
-from helpers.cluster import ClickHouseCluster
-
-cluster = ClickHouseCluster(__file__)
-# Version 21.6.3.14 has incompatible partition id for tables with UUID in partition key.
-node_21_6 = cluster.add_instance(
-    "node_21_6",
-    image="yandex/clickhouse-server",
-    tag="21.6.3.14",
-    stay_alive=True,
-    with_installed_binary=True,
-)
-
-
-@pytest.fixture(scope="module")
-def start_cluster():
-    try:
-        cluster.start()
-        yield cluster
-
-    finally:
-        cluster.shutdown()
-
-
-def test_detach_part_wrong_partition_id(start_cluster):
-    # Here we create table with partition by UUID.
-    node_21_6.query(
-        "create table tab (id UUID, value UInt32) engine = MergeTree PARTITION BY (id) order by tuple()"
-    )
-    node_21_6.query(
-        "insert into tab values ('61f0c404-5cb3-11e7-907b-a6006ad3dba0', 2)"
-    )
-
-    # After restart, partition id will be different.
-    # There is a single 0-level part, which will become broken.
-    # We expect that it will not be removed (as usual for 0-level broken parts),
-    # but moved to /detached
-    node_21_6.restart_with_latest_version()
-
-    num_detached = node_21_6.query("select count() from  system.detached_parts")
-    assert num_detached == "1\n"
-
-    node_21_6.restart_with_original_version()
-
-    node_21_6.query("drop table tab SYNC")

From 53afdd51eada51a966cf67ec148d808f688264d3 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Fri, 5 May 2023 21:27:03 +0000
Subject: [PATCH 205/689] Automatic style fix

---
 .../01065_window_view_event_hop_watch_bounded.py          | 8 ++++++--
 .../queries/0_stateless/01082_window_view_watch_limit.py  | 8 ++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py
index 42efbc8071b..52d0c2c87d4 100755
--- a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py
+++ b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py
@@ -19,13 +19,17 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.expect(prompt)
 
-    client1.send("SET allow_experimental_analyzer = 0, output_format_pretty_squash_ms = 0;")
+    client1.send(
+        "SET allow_experimental_analyzer = 0, output_format_pretty_squash_ms = 0;"
+    )
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
     client1.send("SET window_view_heartbeat_interval = 1")
     client1.expect(prompt)
-    client2.send("SET allow_experimental_window_view = 1, output_format_pretty_squash_ms = 0;")
+    client2.send(
+        "SET allow_experimental_window_view = 1, output_format_pretty_squash_ms = 0;"
+    )
     client2.expect(prompt)
 
     client1.send(
diff --git a/tests/queries/0_stateless/01082_window_view_watch_limit.py b/tests/queries/0_stateless/01082_window_view_watch_limit.py
index 1d43555bddb..07dbc232a63 100755
--- a/tests/queries/0_stateless/01082_window_view_watch_limit.py
+++ b/tests/queries/0_stateless/01082_window_view_watch_limit.py
@@ -19,13 +19,17 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.expect(prompt)
 
-    client1.send("SET allow_experimental_analyzer = 0, output_format_pretty_squash_ms = 0;")
+    client1.send(
+        "SET allow_experimental_analyzer = 0, output_format_pretty_squash_ms = 0;"
+    )
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
     client1.send("SET window_view_heartbeat_interval = 1")
     client1.expect(prompt)
-    client2.send("SET allow_experimental_window_view = 1, output_format_pretty_squash_ms = 0;")
+    client2.send(
+        "SET allow_experimental_window_view = 1, output_format_pretty_squash_ms = 0;"
+    )
     client2.expect(prompt)
 
     client1.send("CREATE DATABASE IF NOT EXISTS 01082_window_view_watch_limit")

From 290bf0ae58f3cbb78d3bb93882fe736f76cca221 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 5 May 2023 23:29:48 +0200
Subject: [PATCH 206/689] Remove a test for in-memory parts

---
 .../test_polymorphic_parts/test.py            | 181 ------------------
 1 file changed, 181 deletions(-)

diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py
index 006662ff4bf..fb1f363b825 100644
--- a/tests/integration/test_polymorphic_parts/test.py
+++ b/tests/integration/test_polymorphic_parts/test.py
@@ -489,187 +489,6 @@ def test_polymorphic_parts_non_adaptive(start_cluster):
     )
 
 
-def test_in_memory(start_cluster):
-    node9.query("SYSTEM STOP MERGES")
-    node10.query("SYSTEM STOP MERGES")
-
-    for size in [200, 200, 300, 600]:
-        insert_random_data("in_memory_table", node9, size)
-    node10.query("SYSTEM SYNC REPLICA in_memory_table", timeout=20)
-
-    assert node9.query("SELECT count() FROM in_memory_table") == "1300\n"
-    assert node10.query("SELECT count() FROM in_memory_table") == "1300\n"
-
-    expected = "Compact\t1\nInMemory\t2\nWide\t1\n"
-
-    assert TSV(
-        node9.query(
-            "SELECT part_type, count() FROM system.parts "
-            "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type"
-        )
-    ) == TSV(expected)
-    assert TSV(
-        node10.query(
-            "SELECT part_type, count() FROM system.parts "
-            "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type"
-        )
-    ) == TSV(expected)
-
-    node9.query("SYSTEM START MERGES")
-    node10.query("SYSTEM START MERGES")
-
-    assert_eq_with_retry(
-        node9,
-        "OPTIMIZE TABLE in_memory_table FINAL SETTINGS optimize_throw_if_noop = 1",
-        "",
-    )
-    node10.query("SYSTEM SYNC REPLICA in_memory_table", timeout=20)
-
-    assert node9.query("SELECT count() FROM in_memory_table") == "1300\n"
-    assert node10.query("SELECT count() FROM in_memory_table") == "1300\n"
-
-    assert TSV(
-        node9.query(
-            "SELECT part_type, count() FROM system.parts "
-            "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type"
-        )
-    ) == TSV("Wide\t1\n")
-    assert TSV(
-        node10.query(
-            "SELECT part_type, count() FROM system.parts "
-            "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type"
-        )
-    ) == TSV("Wide\t1\n")
-
-
-def test_in_memory_wal_rotate(start_cluster):
-    # Write every part to single wal
-    node11.query(
-        "ALTER TABLE restore_table MODIFY SETTING write_ahead_log_max_bytes = 10"
-    )
-    for i in range(5):
-        insert_random_data("restore_table", node11, 50)
-
-    for i in range(5):
-        # Check file exists
-        node11.exec_in_container(
-            [
-                "bash",
-                "-c",
-                "test -f /var/lib/clickhouse/data/default/restore_table/wal_{0}_{0}.bin".format(
-                    i
-                ),
-            ]
-        )
-
-    for node in [node11, node12]:
-        node.query(
-            "ALTER TABLE restore_table MODIFY SETTING number_of_free_entries_in_pool_to_lower_max_size_of_merge = 0"
-        )
-        node.query(
-            "ALTER TABLE restore_table MODIFY SETTING max_bytes_to_merge_at_max_space_in_pool = 10000000"
-        )
-
-    assert_eq_with_retry(
-        node11,
-        "OPTIMIZE TABLE restore_table FINAL SETTINGS optimize_throw_if_noop = 1",
-        "",
-    )
-    # Restart to be sure, that clearing stale logs task was ran
-    node11.restart_clickhouse(kill=True)
-
-    for i in range(5):
-        # check file doesn't exist
-        node11.exec_in_container(
-            [
-                "bash",
-                "-c",
-                "test ! -e /var/lib/clickhouse/data/default/restore_table/wal_{0}_{0}.bin".format(
-                    i
-                ),
-            ]
-        )
-
-    # New wal file was created and ready to write part to it
-    # Check file exists
-    node11.exec_in_container(
-        ["bash", "-c", "test -f /var/lib/clickhouse/data/default/restore_table/wal.bin"]
-    )
-    # Chech file empty
-    node11.exec_in_container(
-        [
-            "bash",
-            "-c",
-            "test ! -s /var/lib/clickhouse/data/default/restore_table/wal.bin",
-        ]
-    )
-
-
-def test_in_memory_deduplication(start_cluster):
-    for i in range(3):
-        # table can be in readonly node
-        exec_query_with_retry(
-            node9,
-            "INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')",
-        )
-        exec_query_with_retry(
-            node10,
-            "INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')",
-        )
-
-    node9.query("SYSTEM SYNC REPLICA deduplication_table", timeout=20)
-    node10.query("SYSTEM SYNC REPLICA deduplication_table", timeout=20)
-
-    assert (
-        node9.query("SELECT date, id, s FROM deduplication_table")
-        == "2020-03-03\t1\tfoo\n"
-    )
-    assert (
-        node10.query("SELECT date, id, s FROM deduplication_table")
-        == "2020-03-03\t1\tfoo\n"
-    )
-
-
-# Checks that restoring from WAL works after table schema changed
-def test_in_memory_alters(start_cluster):
-    def check_parts_type(parts_num):
-        assert (
-            node9.query(
-                "SELECT part_type, count() FROM system.parts WHERE table = 'alters_table' \
-             AND active GROUP BY part_type"
-            )
-            == "InMemory\t{}\n".format(parts_num)
-        )
-
-    node9.query(
-        "INSERT INTO alters_table (date, id, s) VALUES (toDate('2020-10-10'), 1, 'ab'), (toDate('2020-10-10'), 2, 'cd')"
-    )
-    node9.query("ALTER TABLE alters_table ADD COLUMN col1 UInt32")
-    node9.restart_clickhouse(kill=True)
-
-    expected = "1\tab\t0\n2\tcd\t0\n"
-    assert node9.query("SELECT id, s, col1 FROM alters_table ORDER BY id") == expected
-    check_parts_type(1)
-    node9.query(
-        "INSERT INTO alters_table (date, id, col1) VALUES (toDate('2020-10-10'), 3, 100)"
-    )
-    node9.query("ALTER TABLE alters_table MODIFY COLUMN col1 String")
-    node9.query("ALTER TABLE alters_table DROP COLUMN s")
-    node9.restart_clickhouse(kill=True)
-
-    check_parts_type(2)
-    with pytest.raises(Exception):
-        node9.query("SELECT id, s, col1 FROM alters_table")
-
-    # Values of col1 was not materialized as integers, so they have
-    # default string values after alter
-    expected = "1\t_foo\n2\t_foo\n3\t100_foo\n"
-    assert (
-        node9.query("SELECT id, col1 || '_foo' FROM alters_table ORDER BY id")
-        == expected
-    )
-
-
 def test_polymorphic_parts_index(start_cluster):
     node1.query(
         "CREATE DATABASE test_index ENGINE=Ordinary",

From 45c28e1221fa9f3903318bc85e19f5cbdd93c448 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 28 Apr 2023 10:10:42 +0000
Subject: [PATCH 207/689] Introduce more fields for in-source function
 documentation

---
 src/Common/Documentation.cpp                  |  30 -----
 src/Common/FunctionDocumentation.cpp          |  44 +++++++
 ...ocumentation.h => FunctionDocumentation.h} |  48 ++++----
 src/Functions/FunctionFactory.cpp             |   4 +-
 src/Functions/FunctionFactory.h               |  12 +-
 src/Functions/FunctionToDecimalString.cpp     |   8 +-
 src/Functions/FunctionsCodingULID.cpp         |  12 +-
 .../FunctionsExternalDictionaries.cpp         |  80 ++++++------
 src/Functions/FunctionsHashingMisc.cpp        |  34 +++---
 src/Functions/JSONArrayLength.cpp             |   4 +-
 .../URL/cutToFirstSignificantSubdomain.cpp    |  38 +++---
 .../cutToFirstSignificantSubdomainCustom.cpp  |  34 +++---
 src/Functions/URL/domain.cpp                  |  16 +--
 src/Functions/URL/domainWithoutWWW.cpp        |  16 +--
 .../URL/firstSignificantSubdomain.cpp         |  16 +--
 src/Functions/URL/port.cpp                    |  14 +--
 src/Functions/URL/topLevelDomain.cpp          |  16 +--
 src/Functions/UTCTimestamp.cpp                |  10 +-
 .../UniqTheta/FunctionsUniqTheta.cpp          |  27 ++---
 .../array/FunctionsMapMiscellaneous.cpp       | 114 +++++++++---------
 src/Functions/array/arrayShuffle.cpp          |  33 ++---
 src/Functions/array/arraySort.cpp             |  37 +++---
 src/Functions/array/length.cpp                |  18 +--
 src/Functions/ascii.cpp                       |   8 +-
 src/Functions/asin.cpp                        |  14 +--
 src/Functions/canonicalRand.cpp               |   8 +-
 src/Functions/concatWithSeparator.cpp         |  16 +--
 src/Functions/dateDiff.cpp                    |   8 +-
 src/Functions/divideDecimal.cpp               |  63 ++++------
 src/Functions/factorial.cpp                   |   8 +-
 src/Functions/formatReadableDecimalSize.cpp   |  10 +-
 src/Functions/generateULID.cpp                |  12 +-
 src/Functions/getSubcolumn.cpp                |   8 +-
 src/Functions/hasToken.cpp                    |  19 +--
 src/Functions/hasTokenCaseInsensitive.cpp     |  16 +--
 .../keyvaluepair/extractKeyValuePairs.cpp     |  12 +-
 src/Functions/modulo.cpp                      |   8 +-
 src/Functions/mortonDecode.cpp                |  20 +--
 src/Functions/mortonEncode.cpp                |  20 +--
 src/Functions/multiplyDecimal.cpp             |   6 +-
 src/Functions/nested.cpp                      |   8 +-
 src/Functions/randDistribution.cpp            | 110 ++++++++---------
 src/Functions/regexpExtract.cpp               |   4 +-
 src/Functions/reverseDNSQuery.cpp             |  50 ++------
 src/Functions/serverConstants.cpp             |   8 +-
 src/Functions/soundex.cpp                     |   2 +-
 src/Functions/tryDecrypt.cpp                  |   6 +-
 src/Functions/vectorFunctions.cpp             |  48 ++++----
 src/Functions/widthBucket.cpp                 |  10 +-
 .../System/StorageSystemFunctions.cpp         |   6 +-
 src/TableFunctions/ITableFunction.h           |   4 +-
 src/TableFunctions/TableFunctionDeltaLake.cpp |   7 +-
 src/TableFunctions/TableFunctionExplain.cpp   |  15 +--
 src/TableFunctions/TableFunctionFormat.cpp    |  12 +-
 src/TableFunctions/TableFunctionHudi.cpp      |   6 +-
 src/TableFunctions/TableFunctionIceberg.cpp   |   6 +-
 src/TableFunctions/TableFunctionS3.cpp        |  12 +-
 src/TableFunctions/TableFunctionZeros.cpp     |  33 +----
 58 files changed, 597 insertions(+), 671 deletions(-)
 delete mode 100644 src/Common/Documentation.cpp
 create mode 100644 src/Common/FunctionDocumentation.cpp
 rename src/Common/{Documentation.h => FunctionDocumentation.h} (63%)

diff --git a/src/Common/Documentation.cpp b/src/Common/Documentation.cpp
deleted file mode 100644
index 862ecb6054c..00000000000
--- a/src/Common/Documentation.cpp
+++ /dev/null
@@ -1,30 +0,0 @@
-#include <Common/Documentation.h>
-
-namespace DB
-{
-
-std::string Documentation::examplesAsString() const
-{
-    std::string res;
-    for (const auto & [example_name, example_query] : examples)
-    {
-        res += example_name + ":\n\n";
-        res += "```sql\n";
-        res += example_query + "\n";
-        res += "```\n";
-    }
-    return res;
-}
-
-std::string Documentation::categoriesAsString() const
-{
-    if (categories.empty())
-        return "";
-
-    std::string res = categories[0];
-    for (size_t i = 1; i < categories.size(); ++i)
-        res += ", " + categories[i];
-    return res;
-}
-
-}
diff --git a/src/Common/FunctionDocumentation.cpp b/src/Common/FunctionDocumentation.cpp
new file mode 100644
index 00000000000..2aad23b90b7
--- /dev/null
+++ b/src/Common/FunctionDocumentation.cpp
@@ -0,0 +1,44 @@
+#include <Common/FunctionDocumentation.h>
+
+namespace DB
+{
+
+std::string FunctionDocumentation::argumentsAsString() const
+{
+    std::string res;
+    for (const auto & [name, desc] : arguments)
+    {
+        res += "- " + name + ":" + desc + "\n";
+    }
+    return res;
+}
+
+std::string FunctionDocumentation::examplesAsString() const
+{
+    std::string res;
+    for (const auto & [name, query, result] : examples)
+    {
+        res += name + ":\n\n";
+        res += "``` sql\n";
+        res += query + "\n";
+        res += "```\n\n";
+        res += "``` text\n";
+        res += result + "\n";
+        res += "```\n";
+    }
+    return res;
+}
+
+std::string FunctionDocumentation::categoriesAsString() const
+{
+    if (categories.empty())
+        return "";
+
+    auto it = categories.begin();
+    std::string res = *it;
+    for (; it != categories.end(); ++it)
+        res += ", " + *it;
+    return res;
+}
+
+}
diff --git a/src/Common/Documentation.h b/src/Common/FunctionDocumentation.h
similarity index 63%
rename from src/Common/Documentation.h
rename to src/Common/FunctionDocumentation.h
index c71aa8772ed..52b8aed2639 100644
--- a/src/Common/Documentation.h
+++ b/src/Common/FunctionDocumentation.h
@@ -1,15 +1,14 @@
 #pragma once
 
+#include <set>
 #include <string>
 #include <vector>
-#include <map>
 
 
 namespace DB
 {
 
-/** Embedded reference documentation for high-level server components,
-  * such as SQL functions, table functions, data types, table engines, etc.
+/** Embedded reference documentation for functions.
   *
   * The advantages of embedded documentation are:
   * - it is easy to write and update with code;
@@ -34,50 +33,49 @@ namespace DB
   * - examples (queries that can be referenced from the text by names);
   * - categories - one or a few text strings like {"Mathematical", "Array Processing"};
   *
-  * Only the description is mandatory.
-  *
   * The description should be represented in Markdown (or just plaintext).
   * Some extensions for Markdown are added:
   * - [example:name] will reference to an example with the corresponding name.
   *
   * Documentation does not support multiple languages.
   * The only available language is English.
-  *
-  * TODO: Allow to specify Syntax, Argument(s) and a Returned Value.
-  * TODO: Organize Examples as a struct of ExampleName, ExampleQuery and ExampleResult.
   */
-struct Documentation
+struct FunctionDocumentation
 {
     using Description = std::string;
 
     using Syntax = std::string;
 
-    using Argument = std::string;
+    struct Argument
+    {
+        std::string name;
+        std::string description;
+    };
     using Arguments = std::vector<Argument>;
 
     using ReturnedValue = std::string;
 
-    using ExampleName = std::string;
-    using ExampleQuery = std::string;
-    using Examples = std::map<ExampleName, ExampleQuery>;
+    struct Example
+    {
+        std::string name;
+        std::string query;
+        std::string result;
+    };
+    using Examples = std::vector<Example>;
 
     using Category = std::string;
-    using Categories = std::vector<Category>;
+    using Categories = std::set<Category>;
 
     using Related = std::string;
 
-    Description description;
-    Examples examples;
-    Categories categories;
-
-    Documentation(Description description_) : description(std::move(description_)) {} /// NOLINT
-    Documentation(Description description_, Examples examples_) : description(std::move(description_)), examples(std::move(examples_)) {}
-    Documentation(Description description_, Examples examples_, Categories categories_)
-        : description(std::move(description_)), examples(std::move(examples_)), categories(std::move(categories_)) {}
-
-    /// TODO: Please remove this constructor. Documentation should always be non-empty.
-    Documentation() = default;
+    Description description;        /// E.g. "Returns the position (in bytes, starting at 1) of a substring needle in a string haystack."
+    Syntax syntax;                  /// E.g. "position(haystack, needle)"
+    Arguments arguments;            /// E.g. ["haystack — String in which the search is performed. String.", "needle — Substring to be searched. String."]
+    ReturnedValue returned_value;   /// E.g. "Starting position in bytes and counting from 1, if the substring was found."
+    Examples examples;              ///
+    Categories categories;          /// E.g. {"String Search"}
 
+    std::string argumentsAsString() const;
     std::string examplesAsString() const;
     std::string categoriesAsString() const;
 };
diff --git a/src/Functions/FunctionFactory.cpp b/src/Functions/FunctionFactory.cpp
index 361ac6be496..6a7274376b9 100644
--- a/src/Functions/FunctionFactory.cpp
+++ b/src/Functions/FunctionFactory.cpp
@@ -29,7 +29,7 @@ const String & getFunctionCanonicalNameIfAny(const String & name)
 void FunctionFactory::registerFunction(
     const std::string & name,
     FunctionCreator creator,
-    Documentation doc,
+    FunctionDocumentation doc,
     CaseSensitiveness case_sensitiveness)
 {
     if (!functions.emplace(name, FunctionFactoryData{creator, doc}).second)
@@ -141,7 +141,7 @@ FunctionFactory & FunctionFactory::instance()
     return ret;
 }
 
-Documentation FunctionFactory::getDocumentation(const std::string & name) const
+FunctionDocumentation FunctionFactory::getDocumentation(const std::string & name) const
 {
     auto it = functions.find(name);
     if (it == functions.end())
diff --git a/src/Functions/FunctionFactory.h b/src/Functions/FunctionFactory.h
index b9063e2c29a..deea41e6677 100644
--- a/src/Functions/FunctionFactory.h
+++ b/src/Functions/FunctionFactory.h
@@ -3,7 +3,7 @@
 #include <Interpreters/Context_fwd.h>
 #include <Common/register_objects.h>
 #include <Common/IFactoryWithAliases.h>
-#include <Common/Documentation.h>
+#include <Common/FunctionDocumentation.h>
 #include <Functions/IFunction.h>
 #include <Functions/IFunctionAdaptors.h>
 
@@ -17,7 +17,7 @@ namespace DB
 {
 
 using FunctionCreator = std::function<FunctionOverloadResolverPtr(ContextPtr)>;
-using FunctionFactoryData = std::pair<FunctionCreator, Documentation>;
+using FunctionFactoryData = std::pair<FunctionCreator, FunctionDocumentation>;
 
 /** Creates function by name.
   * Function could use for initialization (take ownership of shared_ptr, for example)
@@ -29,13 +29,13 @@ public:
     static FunctionFactory & instance();
 
     template <typename Function>
-    void registerFunction(Documentation doc = {}, CaseSensitiveness case_sensitiveness = CaseSensitive)
+    void registerFunction(FunctionDocumentation doc = {}, CaseSensitiveness case_sensitiveness = CaseSensitive)
     {
         registerFunction<Function>(Function::name, std::move(doc), case_sensitiveness);
     }
 
     template <typename Function>
-    void registerFunction(const std::string & name, Documentation doc = {}, CaseSensitiveness case_sensitiveness = CaseSensitive)
+    void registerFunction(const std::string & name, FunctionDocumentation doc = {}, CaseSensitiveness case_sensitiveness = CaseSensitive)
     {
         if constexpr (std::is_base_of_v<IFunction, Function>)
             registerFunction(name, &adaptFunctionToOverloadResolver<Function>, std::move(doc), case_sensitiveness);
@@ -63,10 +63,10 @@ public:
     void registerFunction(
         const std::string & name,
         FunctionCreator creator,
-        Documentation doc = {},
+        FunctionDocumentation doc = {},
         CaseSensitiveness case_sensitiveness = CaseSensitive);
 
-    Documentation getDocumentation(const std::string & name) const;
+    FunctionDocumentation getDocumentation(const std::string & name) const;
 
 private:
     using Functions = std::unordered_map<std::string, Value>;
diff --git a/src/Functions/FunctionToDecimalString.cpp b/src/Functions/FunctionToDecimalString.cpp
index fb076d03c6b..fe417b19137 100644
--- a/src/Functions/FunctionToDecimalString.cpp
+++ b/src/Functions/FunctionToDecimalString.cpp
@@ -8,14 +8,14 @@ namespace DB
 REGISTER_FUNCTION(ToDecimalString)
 {
     factory.registerFunction<FunctionToDecimalString>(
-        {
-            R"(
+        FunctionDocumentation{
+            .description=R"(
 Returns string representation of a number. First argument is the number of any numeric type,
 second argument is the desired number of digits in fractional part. Returns String.
 
         )",
-            Documentation::Examples{{"toDecimalString", "SELECT toDecimalString(2.1456,2)"}},
-            Documentation::Categories{"String"}
+            .examples{{"toDecimalString", "SELECT toDecimalString(2.1456,2)", ""}},
+            .categories{"String"}
         }, FunctionFactory::CaseInsensitive);
 }
 
diff --git a/src/Functions/FunctionsCodingULID.cpp b/src/Functions/FunctionsCodingULID.cpp
index 90e72abb7c1..1d426ee464a 100644
--- a/src/Functions/FunctionsCodingULID.cpp
+++ b/src/Functions/FunctionsCodingULID.cpp
@@ -169,17 +169,17 @@ public:
 
 REGISTER_FUNCTION(ULIDStringToDateTime)
 {
-    factory.registerFunction<FunctionULIDStringToDateTime>(
+    factory.registerFunction<FunctionULIDStringToDateTime>(FunctionDocumentation
         {
-            R"(
+            .description=R"(
 This function extracts the timestamp from a ULID and returns it as a DateTime64(3) typed value.
 The function expects the ULID to be provided as the first argument, which can be either a String or a FixedString(26) data type.
 An optional second argument can be passed to specify a timezone for the timestamp.
 )",
-            Documentation::Examples{
-                {"ulid", "SELECT ULIDStringToDateTime(generateULID())"},
-                {"timezone", "SELECT ULIDStringToDateTime(generateULID(), 'Asia/Istanbul')"}},
-            Documentation::Categories{"ULID"}
+            .examples{
+                {"ulid", "SELECT ULIDStringToDateTime(generateULID())", ""},
+                {"timezone", "SELECT ULIDStringToDateTime(generateULID(), 'Asia/Istanbul')", ""}},
+            .categories{"ULID"}
         },
         FunctionFactory::CaseSensitive);
 }
diff --git a/src/Functions/FunctionsExternalDictionaries.cpp b/src/Functions/FunctionsExternalDictionaries.cpp
index babc133603f..70b1e3cc861 100644
--- a/src/Functions/FunctionsExternalDictionaries.cpp
+++ b/src/Functions/FunctionsExternalDictionaries.cpp
@@ -47,69 +47,69 @@ Returned value: value of the dictionary attribute parsed in the attribute’s da
 Throws an exception if cannot parse the value of the attribute or the value does not match the attribute data type.
 )" };
 
-    factory.registerFunction<FunctionDictGetNoType<DictionaryGetFunctionType::get>>(Documentation{ fmt::format(dict_get_description, "attribute’s data type") });
-    factory.registerFunction<FunctionDictGetNoType<DictionaryGetFunctionType::getOrDefault>>(Documentation{ fmt::format(dict_get_or_default_description, "attribute’s data type") });
-    factory.registerFunction<FunctionDictGetOrNull>(Documentation{ dict_get_or_null_description });
+    factory.registerFunction<FunctionDictGetNoType<DictionaryGetFunctionType::get>>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "attribute’s data type") });
+    factory.registerFunction<FunctionDictGetNoType<DictionaryGetFunctionType::getOrDefault>>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "attribute’s data type") });
+    factory.registerFunction<FunctionDictGetOrNull>(FunctionDocumentation{ .description=dict_get_or_null_description });
 
-    factory.registerFunction<FunctionDictGetUInt8>(Documentation{ fmt::format(dict_get_description, "UInt8") });
-    factory.registerFunction<FunctionDictGetUInt16>(Documentation{ fmt::format(dict_get_description, "UInt16") });
-    factory.registerFunction<FunctionDictGetUInt32>(Documentation{ fmt::format(dict_get_description, "UInt32") });
-    factory.registerFunction<FunctionDictGetUInt64>(Documentation{ fmt::format(dict_get_description, "UInt64") });
-    factory.registerFunction<FunctionDictGetInt8>(Documentation{ fmt::format(dict_get_description, "Int8") });
-    factory.registerFunction<FunctionDictGetInt16>(Documentation{ fmt::format(dict_get_description, "Int16") });
-    factory.registerFunction<FunctionDictGetInt32>(Documentation{ fmt::format(dict_get_description, "Int32") });
-    factory.registerFunction<FunctionDictGetInt64>(Documentation{ fmt::format(dict_get_description, "Int64") });
-    factory.registerFunction<FunctionDictGetFloat32>(Documentation{ fmt::format(dict_get_description, "Float32") });
-    factory.registerFunction<FunctionDictGetFloat64>(Documentation{ fmt::format(dict_get_description, "Float64") });
-    factory.registerFunction<FunctionDictGetDate>(Documentation{ fmt::format(dict_get_description, "Date") });
-    factory.registerFunction<FunctionDictGetDateTime>(Documentation{ fmt::format(dict_get_description, "DateTime") });
-    factory.registerFunction<FunctionDictGetUUID>(Documentation{ fmt::format(dict_get_description, "UUID") });
-    factory.registerFunction<FunctionDictGetIPv4>(Documentation{ fmt::format(dict_get_description, "IPv4") });
-    factory.registerFunction<FunctionDictGetIPv6>(Documentation{ fmt::format(dict_get_description, "IPv6") });
-    factory.registerFunction<FunctionDictGetString>(Documentation{ fmt::format(dict_get_description, "String") });
+    factory.registerFunction<FunctionDictGetUInt8>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "UInt8") });
+    factory.registerFunction<FunctionDictGetUInt16>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "UInt16") });
+    factory.registerFunction<FunctionDictGetUInt32>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "UInt32") });
+    factory.registerFunction<FunctionDictGetUInt64>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "UInt64") });
+    factory.registerFunction<FunctionDictGetInt8>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "Int8") });
+    factory.registerFunction<FunctionDictGetInt16>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "Int16") });
+    factory.registerFunction<FunctionDictGetInt32>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "Int32") });
+    factory.registerFunction<FunctionDictGetInt64>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "Int64") });
+    factory.registerFunction<FunctionDictGetFloat32>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "Float32") });
+    factory.registerFunction<FunctionDictGetFloat64>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "Float64") });
+    factory.registerFunction<FunctionDictGetDate>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "Date") });
+    factory.registerFunction<FunctionDictGetDateTime>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "DateTime") });
+    factory.registerFunction<FunctionDictGetUUID>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "UUID") });
+    factory.registerFunction<FunctionDictGetIPv4>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "IPv4") });
+    factory.registerFunction<FunctionDictGetIPv6>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "IPv6") });
+    factory.registerFunction<FunctionDictGetString>(FunctionDocumentation{ .description=fmt::format(dict_get_description, "String") });
 
-    factory.registerFunction<FunctionDictGetUInt8OrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "UInt8") });
-    factory.registerFunction<FunctionDictGetUInt16OrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "UInt16") });
-    factory.registerFunction<FunctionDictGetUInt32OrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "UInt32") });
-    factory.registerFunction<FunctionDictGetUInt64OrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "UInt64") });
-    factory.registerFunction<FunctionDictGetInt8OrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "Int8") });
-    factory.registerFunction<FunctionDictGetInt16OrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "Int16") });
-    factory.registerFunction<FunctionDictGetInt32OrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "Int32") });
-    factory.registerFunction<FunctionDictGetInt64OrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "Int64") });
-    factory.registerFunction<FunctionDictGetFloat32OrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "Float32") });
-    factory.registerFunction<FunctionDictGetFloat64OrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "Float64") });
-    factory.registerFunction<FunctionDictGetDateOrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "Date") });
-    factory.registerFunction<FunctionDictGetDateTimeOrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "DateTime") });
-    factory.registerFunction<FunctionDictGetUUIDOrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "UUID") });
-    factory.registerFunction<FunctionDictGetIPv4OrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "IPv4") });
-    factory.registerFunction<FunctionDictGetIPv6OrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "IPv6") });
-    factory.registerFunction<FunctionDictGetStringOrDefault>(Documentation{ fmt::format(dict_get_or_default_description, "String") });
+    factory.registerFunction<FunctionDictGetUInt8OrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "UInt8") });
+    factory.registerFunction<FunctionDictGetUInt16OrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "UInt16") });
+    factory.registerFunction<FunctionDictGetUInt32OrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "UInt32") });
+    factory.registerFunction<FunctionDictGetUInt64OrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "UInt64") });
+    factory.registerFunction<FunctionDictGetInt8OrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "Int8") });
+    factory.registerFunction<FunctionDictGetInt16OrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "Int16") });
+    factory.registerFunction<FunctionDictGetInt32OrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "Int32") });
+    factory.registerFunction<FunctionDictGetInt64OrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "Int64") });
+    factory.registerFunction<FunctionDictGetFloat32OrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "Float32") });
+    factory.registerFunction<FunctionDictGetFloat64OrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "Float64") });
+    factory.registerFunction<FunctionDictGetDateOrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "Date") });
+    factory.registerFunction<FunctionDictGetDateTimeOrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "DateTime") });
+    factory.registerFunction<FunctionDictGetUUIDOrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "UUID") });
+    factory.registerFunction<FunctionDictGetIPv4OrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "IPv4") });
+    factory.registerFunction<FunctionDictGetIPv6OrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "IPv6") });
+    factory.registerFunction<FunctionDictGetStringOrDefault>(FunctionDocumentation{ .description=fmt::format(dict_get_or_default_description, "String") });
 
-    factory.registerFunction<FunctionDictHas>(Documentation{ R"(
+    factory.registerFunction<FunctionDictHas>(FunctionDocumentation{ .description=R"(
 Checks whether a key is present in a dictionary.
 Accepts 2 parameters: name of the dictionary, key value - expression returning dictionary key-type value or tuple-type value - depending on the dictionary configuration.
 Returned value: 0 if there is no key, 1 if there is a key, type of UInt8
 )"});
 
-    factory.registerFunction<FunctionDictGetHierarchy>(Documentation{ R"(
+    factory.registerFunction<FunctionDictGetHierarchy>(FunctionDocumentation{ .description=R"(
 Creates an array, containing all the parents of a key in the hierarchical dictionary.
 Accepts 2 parameters: name of the dictionary, key value - expression returning a UInt64-type value.
 Returned value: parents for the key, type of Array(UInt64)
 )"});
 
-    factory.registerFunction<FunctionDictIsIn>(Documentation{ R"(
+    factory.registerFunction<FunctionDictIsIn>(FunctionDocumentation{ .description=R"(
 Checks the ancestor of a key through the whole hierarchical chain in the dictionary.
 Accepts 3 parameters: name of the dictionary, key to be checked - expression returning a UInt64-type value, alleged ancestor of the key - expression returning a UInt64-type.
 Returned value: 0 if key is not a child of the ancestor, 1 if key is a child of the ancestor or if key is the ancestor, type of UInt8
 )"});
 
-    factory.registerFunction<FunctionDictGetChildrenOverloadResolver>(Documentation{ R"(
+    factory.registerFunction<FunctionDictGetChildrenOverloadResolver>(FunctionDocumentation{ .description=R"(
 Returns first-level children as an array of indexes. It is the inverse transformation for dictGetHierarchy.
 Accepts 2 parameters: name of the dictionary, key value - expression returning a UInt64-type value.
 Returned value: first-level descendants for the key, type of Array(UInt64)
 )"});
 
-    factory.registerFunction<FunctionDictGetDescendantsOverloadResolver>(Documentation{ R"(
+    factory.registerFunction<FunctionDictGetDescendantsOverloadResolver>(FunctionDocumentation{ .description=R"(
 Returns all descendants as if dictGetChildren function was applied level times recursively.
 Accepts 3 parameters: name of the dictionary, key value - expression returning a UInt64-type value, level — hierarchy level - If level = 0 returns all descendants to the end - UInt8
 Returned value: descendants for the key, type of Array(UInt64)
diff --git a/src/Functions/FunctionsHashingMisc.cpp b/src/Functions/FunctionsHashingMisc.cpp
index c1c3df1d53e..127c87ba940 100644
--- a/src/Functions/FunctionsHashingMisc.cpp
+++ b/src/Functions/FunctionsHashingMisc.cpp
@@ -15,15 +15,15 @@ REGISTER_FUNCTION(Hashing)
     factory.registerFunction<FunctionSipHash64Keyed>();
     factory.registerFunction<FunctionSipHash128>();
     factory.registerFunction<FunctionSipHash128Keyed>();
-    factory.registerFunction<FunctionSipHash128Reference>({
-        "Like [sipHash128](#hash_functions-siphash128) but implements the 128-bit algorithm from the original authors of SipHash.",
-        Documentation::Examples{{"hash", "SELECT hex(sipHash128Reference('foo', '\\x01', 3))"}},
-        Documentation::Categories{"Hash"}
+    factory.registerFunction<FunctionSipHash128Reference>(FunctionDocumentation{
+        .description="Like [sipHash128](#hash_functions-siphash128) but implements the 128-bit algorithm from the original authors of SipHash.",
+        .examples{{"hash", "SELECT hex(sipHash128Reference('foo', '\\x01', 3))", ""}},
+        .categories{"Hash"}
     });
-    factory.registerFunction<FunctionSipHash128ReferenceKeyed>({
-        "Same as [sipHash128Reference](#hash_functions-siphash128reference) but additionally takes an explicit key argument instead of using a fixed key.",
-        Documentation::Examples{{"hash", "SELECT hex(sipHash128ReferenceKeyed((506097522914230528, 1084818905618843912),'foo', '\\x01', 3));"}},
-        Documentation::Categories{"Hash"}
+    factory.registerFunction<FunctionSipHash128ReferenceKeyed>(FunctionDocumentation{
+        .description="Same as [sipHash128Reference](#hash_functions-siphash128reference) but additionally takes an explicit key argument instead of using a fixed key.",
+        .examples{{"hash", "SELECT hex(sipHash128ReferenceKeyed((506097522914230528, 1084818905618843912),'foo', '\\x01', 3));", ""}},
+        .categories{"Hash"}
     });
     factory.registerFunction<FunctionCityHash64>();
     factory.registerFunction<FunctionFarmFingerprint64>();
@@ -37,10 +37,10 @@ REGISTER_FUNCTION(Hashing)
     factory.registerFunction<FunctionXxHash32>();
     factory.registerFunction<FunctionXxHash64>();
     factory.registerFunction<FunctionXXH3>(
-        {
-            "Calculates value of XXH3 64-bit hash function. Refer to https://github.com/Cyan4973/xxHash for detailed documentation.",
-            Documentation::Examples{{"hash", "SELECT xxh3('ClickHouse')"}},
-            Documentation::Categories{"Hash"}
+        FunctionDocumentation{
+            .description="Calculates value of XXH3 64-bit hash function. Refer to https://github.com/Cyan4973/xxHash for detailed documentation.",
+            .examples{{"hash", "SELECT xxh3('ClickHouse')", ""}},
+            .categories{"Hash"}
         },
         FunctionFactory::CaseSensitive);
 
@@ -48,16 +48,16 @@ REGISTER_FUNCTION(Hashing)
 
 
     factory.registerFunction<FunctionBLAKE3>(
-    {
-        R"(
+    FunctionDocumentation{
+        .description=R"(
 Calculates BLAKE3 hash string and returns the resulting set of bytes as FixedString.
 This cryptographic hash-function is integrated into ClickHouse with BLAKE3 Rust library.
 The function is rather fast and shows approximately two times faster performance compared to SHA-2, while generating hashes of the same length as SHA-256.
 It returns a BLAKE3 hash as a byte array with type FixedString(32).
 )",
-        Documentation::Examples{
-            {"hash", "SELECT hex(BLAKE3('ABC'))"}},
-        Documentation::Categories{"Hash"}
+        .examples{
+            {"hash", "SELECT hex(BLAKE3('ABC'))", ""}},
+        .categories{"Hash"}
     },
     FunctionFactory::CaseSensitive);
 }
diff --git a/src/Functions/JSONArrayLength.cpp b/src/Functions/JSONArrayLength.cpp
index 4f784ce140d..a82c50360f9 100644
--- a/src/Functions/JSONArrayLength.cpp
+++ b/src/Functions/JSONArrayLength.cpp
@@ -100,8 +100,8 @@ namespace
 
 REGISTER_FUNCTION(JSONArrayLength)
 {
-    factory.registerFunction<FunctionJSONArrayLength>(Documentation{
-        "Returns the number of elements in the outermost JSON array. The function returns NULL if input JSON string is invalid."});
+    factory.registerFunction<FunctionJSONArrayLength>(FunctionDocumentation{
+        .description="Returns the number of elements in the outermost JSON array. The function returns NULL if input JSON string is invalid."});
 
     /// For Spark compatibility.
     factory.registerAlias("JSON_ARRAY_LENGTH", "JSONArrayLength", FunctionFactory::CaseInsensitive);
diff --git a/src/Functions/URL/cutToFirstSignificantSubdomain.cpp b/src/Functions/URL/cutToFirstSignificantSubdomain.cpp
index 7bf09d1eb00..6e64b0b6ab8 100644
--- a/src/Functions/URL/cutToFirstSignificantSubdomain.cpp
+++ b/src/Functions/URL/cutToFirstSignificantSubdomain.cpp
@@ -44,32 +44,32 @@ using FunctionCutToFirstSignificantSubdomainWithWWWRFC = FunctionStringToString<
 REGISTER_FUNCTION(CutToFirstSignificantSubdomain)
 {
     factory.registerFunction<FunctionCutToFirstSignificantSubdomain>(
-        {
-        R"(Returns the part of the domain that includes top-level subdomains up to the "first significant subdomain" (see documentation of the `firstSignificantSubdomain`).)",
-        Documentation::Examples{
-            {"cutToFirstSignificantSubdomain1", "SELECT cutToFirstSignificantSubdomain('https://news.clickhouse.com.tr/')"},
-            {"cutToFirstSignificantSubdomain2", "SELECT cutToFirstSignificantSubdomain('www.tr')"},
-            {"cutToFirstSignificantSubdomain3", "SELECT cutToFirstSignificantSubdomain('tr')"},
+        FunctionDocumentation{
+        .description=R"(Returns the part of the domain that includes top-level subdomains up to the "first significant subdomain" (see documentation of the `firstSignificantSubdomain`).)",
+        .examples{
+            {"cutToFirstSignificantSubdomain1", "SELECT cutToFirstSignificantSubdomain('https://news.clickhouse.com.tr/')", ""},
+            {"cutToFirstSignificantSubdomain2", "SELECT cutToFirstSignificantSubdomain('www.tr')", ""},
+            {"cutToFirstSignificantSubdomain3", "SELECT cutToFirstSignificantSubdomain('tr')", ""},
         },
-        Documentation::Categories{"URL"}
+        .categories{"URL"}
         });
     factory.registerFunction<FunctionCutToFirstSignificantSubdomainWithWWW>(
-        {
-            R"(Returns the part of the domain that includes top-level subdomains up to the "first significant subdomain", without stripping "www".)",
-            Documentation::Examples{},
-            Documentation::Categories{"URL"}
+        FunctionDocumentation{
+            .description=R"(Returns the part of the domain that includes top-level subdomains up to the "first significant subdomain", without stripping "www".)",
+            .examples{},
+            .categories{"URL"}
         });
     factory.registerFunction<FunctionCutToFirstSignificantSubdomainRFC>(
-        {
-            R"(Similar to `cutToFirstSignificantSubdomain` but follows stricter rules to be compatible with RFC 3986 and less performant.)",
-            Documentation::Examples{},
-            Documentation::Categories{"URL"}
+        FunctionDocumentation{
+            .description=R"(Similar to `cutToFirstSignificantSubdomain` but follows stricter rules to be compatible with RFC 3986 and less performant.)",
+            .examples{},
+            .categories{"URL"}
         });
     factory.registerFunction<FunctionCutToFirstSignificantSubdomainWithWWWRFC>(
-        {
-            R"(Similar to `cutToFirstSignificantSubdomainWithWWW` but follows stricter rules to be compatible with RFC 3986 and less performant.)",
-            Documentation::Examples{},
-            Documentation::Categories{"URL"}
+        FunctionDocumentation{
+            .description=R"(Similar to `cutToFirstSignificantSubdomainWithWWW` but follows stricter rules to be compatible with RFC 3986 and less performant.)",
+            .examples{},
+            .categories{"URL"}
         });
 }
 
diff --git a/src/Functions/URL/cutToFirstSignificantSubdomainCustom.cpp b/src/Functions/URL/cutToFirstSignificantSubdomainCustom.cpp
index e81921d69ff..77f40e465a6 100644
--- a/src/Functions/URL/cutToFirstSignificantSubdomainCustom.cpp
+++ b/src/Functions/URL/cutToFirstSignificantSubdomainCustom.cpp
@@ -43,39 +43,39 @@ using FunctionCutToFirstSignificantSubdomainCustomWithWWWRFC = FunctionCutToFirs
 REGISTER_FUNCTION(CutToFirstSignificantSubdomainCustom)
 {
     factory.registerFunction<FunctionCutToFirstSignificantSubdomainCustom>(
-        {
-        R"(
+        FunctionDocumentation{
+        .description=R"(
 Returns the part of the domain that includes top-level subdomains up to the first significant subdomain. Accepts custom TLD list name.
 
 Can be useful if you need fresh TLD list or you have custom.
         )",
-        Documentation::Examples{
-            {"cutToFirstSignificantSubdomainCustom", "SELECT cutToFirstSignificantSubdomainCustom('bar.foo.there-is-no-such-domain', 'public_suffix_list');"},
+        .examples{
+            {"cutToFirstSignificantSubdomainCustom", "SELECT cutToFirstSignificantSubdomainCustom('bar.foo.there-is-no-such-domain', 'public_suffix_list');", ""},
         },
-        Documentation::Categories{"URL"}
+        .categories{"URL"}
         });
     factory.registerFunction<FunctionCutToFirstSignificantSubdomainCustomWithWWW>(
-        {
-        R"(
+        FunctionDocumentation{
+        .description=R"(
 Returns the part of the domain that includes top-level subdomains up to the first significant subdomain without stripping `www`.
 Accepts custom TLD list name from config.
 
 Can be useful if you need fresh TLD list or you have custom.
         )",
-        Documentation::Examples{{"cutToFirstSignificantSubdomainCustomWithWWW", "SELECT cutToFirstSignificantSubdomainCustomWithWWW('www.foo', 'public_suffix_list')"}},
-        Documentation::Categories{"URL"}
+        .examples{{"cutToFirstSignificantSubdomainCustomWithWWW", "SELECT cutToFirstSignificantSubdomainCustomWithWWW('www.foo', 'public_suffix_list')", ""}},
+        .categories{"URL"}
         });
     factory.registerFunction<FunctionCutToFirstSignificantSubdomainCustomRFC>(
-        {
-        R"(Similar to `cutToFirstSignificantSubdomainCustom` but follows stricter rules according to RFC 3986.)",
-        Documentation::Examples{},
-        Documentation::Categories{"URL"}
+        FunctionDocumentation{
+        .description=R"(Similar to `cutToFirstSignificantSubdomainCustom` but follows stricter rules according to RFC 3986.)",
+        .examples{},
+        .categories{"URL"}
         });
     factory.registerFunction<FunctionCutToFirstSignificantSubdomainCustomWithWWWRFC>(
-        {
-        R"(Similar to `cutToFirstSignificantSubdomainCustomWithWWW` but follows stricter rules according to RFC 3986.)",
-        Documentation::Examples{},
-        Documentation::Categories{"URL"}
+        FunctionDocumentation{
+        .description=R"(Similar to `cutToFirstSignificantSubdomainCustomWithWWW` but follows stricter rules according to RFC 3986.)",
+        .examples{},
+        .categories{"URL"}
         });
 }
 
diff --git a/src/Functions/URL/domain.cpp b/src/Functions/URL/domain.cpp
index fce7cea4693..443a3323075 100644
--- a/src/Functions/URL/domain.cpp
+++ b/src/Functions/URL/domain.cpp
@@ -14,23 +14,23 @@ using FunctionDomainRFC = FunctionStringToString<ExtractSubstringImpl<ExtractDom
 
 REGISTER_FUNCTION(Domain)
 {
-    factory.registerFunction<FunctionDomain>(
+    factory.registerFunction<FunctionDomain>(FunctionDocumentation
         {
-        R"(
+        .description=R"(
 Extracts the hostname from a URL.
 
 The URL can be specified with or without a scheme.
 If the argument can't be parsed as URL, the function returns an empty string.
         )",
-        Documentation::Examples{{"domain", "SELECT domain('svn+ssh://some.svn-hosting.com:80/repo/trunk')"}},
-        Documentation::Categories{"URL"}
+        .examples{{"domain", "SELECT domain('svn+ssh://some.svn-hosting.com:80/repo/trunk')", ""}},
+        .categories{"URL"}
         });
 
-    factory.registerFunction<FunctionDomainRFC>(
+    factory.registerFunction<FunctionDomainRFC>(FunctionDocumentation
         {
-        R"(Similar to `domain` but follows stricter rules to be compatible with RFC 3986 and less performant.)",
-        Documentation::Examples{},
-        Documentation::Categories{"URL"}
+        .description=R"(Similar to `domain` but follows stricter rules to be compatible with RFC 3986 and less performant.)",
+        .examples{},
+        .categories{"URL"}
         });
 }
 
diff --git a/src/Functions/URL/domainWithoutWWW.cpp b/src/Functions/URL/domainWithoutWWW.cpp
index 48401e5e6e5..f6c8b5c84fc 100644
--- a/src/Functions/URL/domainWithoutWWW.cpp
+++ b/src/Functions/URL/domainWithoutWWW.cpp
@@ -15,21 +15,21 @@ using FunctionDomainWithoutWWWRFC = FunctionStringToString<ExtractSubstringImpl<
 REGISTER_FUNCTION(DomainWithoutWWW)
 {
     factory.registerFunction<FunctionDomainWithoutWWW>(
-    {
-        R"(
+    FunctionDocumentation{
+        .description=R"(
 Extracts the hostname from a URL, removing the leading "www." if present.
 
 The URL can be specified with or without a scheme.
 If the argument can't be parsed as URL, the function returns an empty string.
         )",
-        Documentation::Examples{{"domainWithoutWWW", "SELECT domainWithoutWWW('https://www.clickhouse.com')"}},
-        Documentation::Categories{"URL"}
+        .examples{{"domainWithoutWWW", "SELECT domainWithoutWWW('https://www.clickhouse.com')", ""}},
+        .categories{"URL"}
     });
     factory.registerFunction<FunctionDomainWithoutWWWRFC>(
-    {
-        R"(Similar to `domainWithoutWWW` but follows stricter rules to be compatible with RFC 3986 and less performant.)",
-        Documentation::Examples{},
-        Documentation::Categories{"URL"}
+    FunctionDocumentation{
+        .description=R"(Similar to `domainWithoutWWW` but follows stricter rules to be compatible with RFC 3986 and less performant.)",
+        .examples{},
+        .categories{"URL"}
     });
 }
 
diff --git a/src/Functions/URL/firstSignificantSubdomain.cpp b/src/Functions/URL/firstSignificantSubdomain.cpp
index 62307ef816c..b04f6d882ef 100644
--- a/src/Functions/URL/firstSignificantSubdomain.cpp
+++ b/src/Functions/URL/firstSignificantSubdomain.cpp
@@ -15,8 +15,8 @@ using FunctionFirstSignificantSubdomainRFC = FunctionStringToString<ExtractSubst
 REGISTER_FUNCTION(FirstSignificantSubdomain)
 {
     factory.registerFunction<FunctionFirstSignificantSubdomain>(
-        {
-        R"(
+        FunctionDocumentation{
+        .description=R"(
 Returns the "first significant subdomain".
 
 The first significant subdomain is a second-level domain if it is 'com', 'net', 'org', or 'co'.
@@ -26,15 +26,15 @@ For example, firstSignificantSubdomain('https://news.clickhouse.com/') = 'clickh
 
 The list of "insignificant" second-level domains and other implementation details may change in the future.
         )",
-        Documentation::Examples{{"firstSignificantSubdomain", "SELECT firstSignificantSubdomain('https://news.clickhouse.com/')"}},
-        Documentation::Categories{"URL"}
+        .examples{{"firstSignificantSubdomain", "SELECT firstSignificantSubdomain('https://news.clickhouse.com/')", ""}},
+        .categories{"URL"}
         });
 
     factory.registerFunction<FunctionFirstSignificantSubdomainRFC>(
-        {
-        R"(Returns the "first significant subdomain" according to RFC 1034.)",
-        Documentation::Examples{},
-        Documentation::Categories{"URL"}
+        FunctionDocumentation{
+        .description=R"(Returns the "first significant subdomain" according to RFC 1034.)",
+        .examples{},
+        .categories{"URL"}
         });
 }
 
diff --git a/src/Functions/URL/port.cpp b/src/Functions/URL/port.cpp
index 65cebae15f1..942f6b702fd 100644
--- a/src/Functions/URL/port.cpp
+++ b/src/Functions/URL/port.cpp
@@ -138,17 +138,15 @@ struct FunctionPortRFC : public FunctionPortImpl<true>
 
 REGISTER_FUNCTION(Port)
 {
-    factory.registerFunction<FunctionPort>(
+    factory.registerFunction<FunctionPort>(FunctionDocumentation
     {
-        R"(Returns the port or `default_port` if there is no port in the URL (or in case of validation error).)",
-        Documentation::Examples{},
-        Documentation::Categories{"URL"}
+        .description=R"(Returns the port or `default_port` if there is no port in the URL (or in case of validation error).)",
+        .categories{"URL"}
     });
-    factory.registerFunction<FunctionPortRFC>(
+    factory.registerFunction<FunctionPortRFC>(FunctionDocumentation
     {
-        R"(Similar to `port`, but conforms to RFC 3986.)",
-        Documentation::Examples{},
-        Documentation::Categories{"URL"}
+        .description=R"(Similar to `port`, but conforms to RFC 3986.)",
+        .categories{"URL"}
     });
 }
 
diff --git a/src/Functions/URL/topLevelDomain.cpp b/src/Functions/URL/topLevelDomain.cpp
index ed9b40d4b73..25e9f383f60 100644
--- a/src/Functions/URL/topLevelDomain.cpp
+++ b/src/Functions/URL/topLevelDomain.cpp
@@ -53,22 +53,22 @@ using FunctionTopLevelDomainRFC = FunctionStringToString<ExtractSubstringImpl<Ex
 
 REGISTER_FUNCTION(TopLevelDomain)
 {
-    factory.registerFunction<FunctionTopLevelDomain>(
+    factory.registerFunction<FunctionTopLevelDomain>(FunctionDocumentation
     {
-        R"(
+        .description=R"(
 Extracts the the top-level domain from a URL.
 
 Returns an empty string if the argument cannot be parsed as a URL or does not contain a top-level domain.
         )",
-        Documentation::Examples{{"topLevelDomain", "SELECT topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk')"}},
-        Documentation::Categories{"URL"}
+        .examples{{"topLevelDomain", "SELECT topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk')", ""}},
+        .categories{"URL"}
     });
 
-    factory.registerFunction<FunctionTopLevelDomainRFC>(
+    factory.registerFunction<FunctionTopLevelDomainRFC>(FunctionDocumentation
     {
-        R"(Similar to topLevelDomain, but conforms to RFC 3986.)",
-        Documentation::Examples{},
-        Documentation::Categories{"URL"}
+        .description=R"(Similar to topLevelDomain, but conforms to RFC 3986.)",
+        .examples{},
+        .categories{"URL"}
     });
 }
 
diff --git a/src/Functions/UTCTimestamp.cpp b/src/Functions/UTCTimestamp.cpp
index 258a5fa5d94..acc34b0a974 100644
--- a/src/Functions/UTCTimestamp.cpp
+++ b/src/Functions/UTCTimestamp.cpp
@@ -107,17 +107,17 @@ public:
 /// UTC_timestamp for MySQL interface support
 REGISTER_FUNCTION(UTCTimestamp)
 {
-    factory.registerFunction<UTCTimestampOverloadResolver>({
-        R"(
+    factory.registerFunction<UTCTimestampOverloadResolver>(FunctionDocumentation{
+        .description=R"(
 Returns the current date and time at the moment of query analysis. The function is a constant expression.
 Same as `now('UTC')`. Was added only for MySQL support. `now` is preferred.
 
 Example:
 [example:typical]
 )",
-    Documentation::Examples{
-        {"typical", "SELECT UTCTimestamp();"}},
-    Documentation::Categories{"Dates and Times"}}, FunctionFactory::CaseInsensitive);
+    .examples{
+        {"typical", "SELECT UTCTimestamp();", ""}},
+    .categories{"Dates and Times"}}, FunctionFactory::CaseInsensitive);
     factory.registerAlias("UTC_timestamp", UTCTimestampOverloadResolver::name, FunctionFactory::CaseInsensitive);
 }
 
diff --git a/src/Functions/UniqTheta/FunctionsUniqTheta.cpp b/src/Functions/UniqTheta/FunctionsUniqTheta.cpp
index aa280c0818e..f371f4b33e8 100644
--- a/src/Functions/UniqTheta/FunctionsUniqTheta.cpp
+++ b/src/Functions/UniqTheta/FunctionsUniqTheta.cpp
@@ -10,8 +10,7 @@ namespace DB
 REGISTER_FUNCTION(UniqTheta)
 {
     factory.registerFunction<FunctionUniqThetaIntersect>(
-            {
-              R"(
+            FunctionDocumentation{.description = R"(
 Two uniqThetaSketch objects to do intersect calculation(set operation ∩), the result is a new uniqThetaSketch.
 
 A uniqThetaSketch object is to be constructed by aggregation function uniqTheta with -State.
@@ -22,14 +21,13 @@ For more information on RoaringBitmap, see: [Theta Sketch Framework](https://dat
 Typical usage:
 [example:typical]
 )",
-                    Documentation::Examples{
-                            {"typical", "select finalizeAggregation(uniqThetaIntersect(arrayReduce('uniqThetaState',[1,2]), arrayReduce('uniqThetaState',[2,3,4])));"}},
-                    Documentation::Categories{"uniqTheta"}
+                    .examples{
+                            {"typical", "select finalizeAggregation(uniqThetaIntersect(arrayReduce('uniqThetaState',[1,2]), arrayReduce('uniqThetaState',[2,3,4])));", ""}},
+                    .categories{"uniqTheta"}
             });
 
     factory.registerFunction<FunctionUniqThetaUnion>(
-            {
-              R"(
+            FunctionDocumentation{.description = R"(
 Two uniqThetaSketch objects to do union calculation(set operation ∪), the result is a new uniqThetaSketch.
 
 A uniqThetaSketch object is to be constructed by aggregation function uniqTheta with -State.
@@ -40,13 +38,12 @@ For more information on RoaringBitmap, see: [Theta Sketch Framework](https://dat
 Typical usage:
 [example:typical]
 )",
-                    Documentation::Examples{
-                            {"typical", "select finalizeAggregation(uniqThetaUnion(arrayReduce('uniqThetaState',[1,2]), arrayReduce('uniqThetaState',[2,3,4])));"}},
-                    Documentation::Categories{"uniqTheta"}
+                    .examples{
+                            {"typical", "select finalizeAggregation(uniqThetaUnion(arrayReduce('uniqThetaState',[1,2]), arrayReduce('uniqThetaState',[2,3,4])));", ""}},
+                    .categories{"uniqTheta"}
             });
     factory.registerFunction<FunctionUniqThetaNot>(
-            {
-              R"(
+            FunctionDocumentation{.description = R"(
 Two uniqThetaSketch objects to do a_not_b calculation(set operation ×), the result is a new uniqThetaSketch.
 
 A uniqThetaSketch object is to be constructed by aggregation function uniqTheta with -State.
@@ -57,9 +54,9 @@ For more information on RoaringBitmap, see: [Theta Sketch Framework](https://dat
 Typical usage:
 [example:typical]
 )",
-                    Documentation::Examples{
-                            {"typical", "select finalizeAggregation(uniqThetaNot(arrayReduce('uniqThetaState',[1,2]), arrayReduce('uniqThetaState',[2,3,4])));"}},
-                    Documentation::Categories{"uniqTheta"}
+                    .examples{
+                            {"typical", "select finalizeAggregation(uniqThetaNot(arrayReduce('uniqThetaState',[1,2]), arrayReduce('uniqThetaState',[2,3,4])));", ""}},
+                    .categories{"uniqTheta"}
             });
 }
 
diff --git a/src/Functions/array/FunctionsMapMiscellaneous.cpp b/src/Functions/array/FunctionsMapMiscellaneous.cpp
index 853156fcafc..157f2fa8a26 100644
--- a/src/Functions/array/FunctionsMapMiscellaneous.cpp
+++ b/src/Functions/array/FunctionsMapMiscellaneous.cpp
@@ -363,101 +363,101 @@ using FunctionMapPartialReverseSort = FunctionMapToArrayAdapter<FunctionArrayPar
 REGISTER_FUNCTION(MapMiscellaneous)
 {
     factory.registerFunction<FunctionMapConcat>(
-    {
-        "The same as arrayConcat.",
-        Documentation::Examples{{"mapConcat", "SELECT mapConcat(map('k1', 'v1'), map('k2', 'v2'))"}},
-        Documentation::Categories{"Map"},
+    FunctionDocumentation{
+        .description="The same as arrayConcat.",
+        .examples{{"mapConcat", "SELECT mapConcat(map('k1', 'v1'), map('k2', 'v2'))", ""}},
+        .categories{"Map"},
     });
 
     factory.registerFunction<FunctionMapKeys>(
-    {
-        "Returns an array with the keys of map.",
-        Documentation::Examples{{"mapKeys", "SELECT mapKeys(map('k1', 'v1', 'k2', 'v2'))"}},
-        Documentation::Categories{"Map"},
+    FunctionDocumentation{
+        .description="Returns an array with the keys of map.",
+        .examples{{"mapKeys", "SELECT mapKeys(map('k1', 'v1', 'k2', 'v2'))", ""}},
+        .categories{"Map"},
     });
 
     factory.registerFunction<FunctionMapValues>(
-    {
-        "Returns an array with the values of map.",
-        Documentation::Examples{{"mapValues", "SELECT mapValues(map('k1', 'v1', 'k2', 'v2'))"}},
-        Documentation::Categories{"Map"},
+    FunctionDocumentation{
+        .description="Returns an array with the values of map.",
+        .examples{{"mapValues", "SELECT mapValues(map('k1', 'v1', 'k2', 'v2'))", ""}},
+        .categories{"Map"},
     });
 
     factory.registerFunction<FunctionMapContains>(
-    {
-        "Checks whether the map has the specified key.",
-        Documentation::Examples{{"mapContains", "SELECT mapContains(map('k1', 'v1', 'k2', 'v2'), 'k1')"}},
-        Documentation::Categories{"Map"},
+    FunctionDocumentation{
+        .description="Checks whether the map has the specified key.",
+        .examples{{"mapContains", "SELECT mapContains(map('k1', 'v1', 'k2', 'v2'), 'k1')", ""}},
+        .categories{"Map"},
     });
 
     factory.registerFunction<FunctionMapFilter>(
-    {
-        "The same as arrayFilter.",
-        Documentation::Examples{{"mapFilter", "SELECT mapFilter((k, v) -> v > 1, map('k1', 1, 'k2', 2))"}},
-        Documentation::Categories{"Map"},
+    FunctionDocumentation{
+        .description="The same as arrayFilter.",
+        .examples{{"mapFilter", "SELECT mapFilter((k, v) -> v > 1, map('k1', 1, 'k2', 2))", ""}},
+        .categories{"Map"},
     });
 
     factory.registerFunction<FunctionMapApply>(
-    {
-        "The same as arrayMap.",
-        Documentation::Examples{{"mapApply", "SELECT mapApply((k, v) -> (k, v * 2), map('k1', 1, 'k2', 2))"}},
-        Documentation::Categories{"Map"},
+    FunctionDocumentation{
+        .description="The same as arrayMap.",
+        .examples{{"mapApply", "SELECT mapApply((k, v) -> (k, v * 2), map('k1', 1, 'k2', 2))", ""}},
+        .categories{"Map"},
     });
 
     factory.registerFunction<FunctionMapExists>(
-    {
-        "The same as arrayExists.",
-        Documentation::Examples{{"mapExists", "SELECT mapExists((k, v) -> v = 1, map('k1', 1, 'k2', 2))"}},
-        Documentation::Categories{"Map"},
+    FunctionDocumentation{
+        .description="The same as arrayExists.",
+        .examples{{"mapExists", "SELECT mapExists((k, v) -> v = 1, map('k1', 1, 'k2', 2))", ""}},
+        .categories{"Map"},
     });
 
-     factory.registerFunction<FunctionMapAll>(
-    {
-        "The same as arrayAll.",
-        Documentation::Examples{{"mapAll", "SELECT mapAll((k, v) -> v = 1, map('k1', 1, 'k2', 2))"}},
-        Documentation::Categories{"Map"},
+    factory.registerFunction<FunctionMapAll>(
+    FunctionDocumentation{
+        .description="The same as arrayAll.",
+        .examples{{"mapAll", "SELECT mapAll((k, v) -> v = 1, map('k1', 1, 'k2', 2))", ""}},
+        .categories{"Map"},
     });
 
     factory.registerFunction<FunctionMapSort>(
-    {
-        "The same as arraySort.",
-        Documentation::Examples{{"mapSort", "SELECT mapSort((k, v) -> v, map('k1', 3, 'k2', 1, 'k3', 2))"}},
-        Documentation::Categories{"Map"},
+    FunctionDocumentation{
+        .description="The same as arraySort.",
+        .examples{{"mapSort", "SELECT mapSort((k, v) -> v, map('k1', 3, 'k2', 1, 'k3', 2))", ""}},
+        .categories{"Map"},
     });
 
     factory.registerFunction<FunctionMapReverseSort>(
-    {
-        "The same as arrayReverseSort.",
-        Documentation::Examples{{"mapReverseSort", "SELECT mapReverseSort((k, v) -> v, map('k1', 3, 'k2', 1, 'k3', 2))"}},
-        Documentation::Categories{"Map"},
+    FunctionDocumentation{
+        .description="The same as arrayReverseSort.",
+        .examples{{"mapReverseSort", "SELECT mapReverseSort((k, v) -> v, map('k1', 3, 'k2', 1, 'k3', 2))", ""}},
+        .categories{"Map"},
     });
 
     factory.registerFunction<FunctionMapPartialSort>(
-    {
-        "The same as arrayReverseSort.",
-        Documentation::Examples{{"mapPartialSort", "SELECT mapPartialSort((k, v) -> v, 2, map('k1', 3, 'k2', 1, 'k3', 2))"}},
-        Documentation::Categories{"Map"},
+    FunctionDocumentation{
+        .description="The same as arrayReverseSort.",
+        .examples{{"mapPartialSort", "SELECT mapPartialSort((k, v) -> v, 2, map('k1', 3, 'k2', 1, 'k3', 2))", ""}},
+        .categories{"Map"},
     });
 
     factory.registerFunction<FunctionMapPartialReverseSort>(
-    {
-        "The same as arrayPartialReverseSort.",
-        Documentation::Examples{{"mapPartialReverseSort", "SELECT mapPartialReverseSort((k, v) -> v, 2, map('k1', 3, 'k2', 1, 'k3', 2))"}},
-        Documentation::Categories{"Map"},
+    FunctionDocumentation{
+        .description="The same as arrayPartialReverseSort.",
+        .examples{{"mapPartialReverseSort", "SELECT mapPartialReverseSort((k, v) -> v, 2, map('k1', 3, 'k2', 1, 'k3', 2))", ""}},
+        .categories{"Map"},
     });
 
     factory.registerFunction<FunctionMapContainsKeyLike>(
-    {
-        "Checks whether map contains key LIKE specified pattern.",
-        Documentation::Examples{{"mapContainsKeyLike", "SELECT mapContainsKeyLike(map('k1-1', 1, 'k2-1', 2), 'k1%')"}},
-        Documentation::Categories{"Map"},
+    FunctionDocumentation{
+        .description="Checks whether map contains key LIKE specified pattern.",
+        .examples{{"mapContainsKeyLike", "SELECT mapContainsKeyLike(map('k1-1', 1, 'k2-1', 2), 'k1%')", ""}},
+        .categories{"Map"},
     });
 
     factory.registerFunction<FunctionMapExtractKeyLike>(
-    {
-        "Returns a map with elements which key matches the specified pattern.",
-        Documentation::Examples{{"mapExtractKeyLike", "SELECT mapExtractKeyLike(map('k1-1', 1, 'k2-1', 2), 'k1%')"}},
-        Documentation::Categories{"Map"},
+    FunctionDocumentation{
+        .description="Returns a map with elements which key matches the specified pattern.",
+        .examples{{"mapExtractKeyLike", "SELECT mapExtractKeyLike(map('k1-1', 1, 'k2-1', 2), 'k1%')", ""}},
+        .categories{"Map"},
     });
 }
 
diff --git a/src/Functions/array/arrayShuffle.cpp b/src/Functions/array/arrayShuffle.cpp
index 9cf3ac8f3fe..7c20e195098 100644
--- a/src/Functions/array/arrayShuffle.cpp
+++ b/src/Functions/array/arrayShuffle.cpp
@@ -175,8 +175,8 @@ ColumnPtr FunctionArrayShuffleImpl<Traits>::executeGeneric(const ColumnArray & a
 REGISTER_FUNCTION(ArrayShuffle)
 {
     factory.registerFunction<FunctionArrayShuffleImpl<FunctionArrayShuffleTraits>>(
-        {
-            R"(
+        FunctionDocumentation{
+            .description=R"(
 Returns an array of the same size as the original array containing the elements in shuffled order.
 Elements are being reordered in such a way that each possible permutation of those elements has equal probability of appearance.
 
@@ -189,15 +189,16 @@ If no seed is provided a random one will be used:
 It is possible to override the seed to produce stable results:
 [example:explicit_seed]
 )",
-            Documentation::Examples{
-                {"random_seed", "SELECT arrayShuffle([1, 2, 3, 4])"},
-                {"explicit_seed", "SELECT arrayShuffle([1, 2, 3, 4], 41)"},
-                {"materialize", "SELECT arrayShuffle(materialize([1, 2, 3]), 42), arrayShuffle([1, 2, 3], 42) FROM numbers(10)"}},
-            Documentation::Categories{"Array"}},
+            .examples{
+                {"random_seed", "SELECT arrayShuffle([1, 2, 3, 4])", ""},
+                {"explicit_seed", "SELECT arrayShuffle([1, 2, 3, 4], 41)", ""},
+                {"materialize", "SELECT arrayShuffle(materialize([1, 2, 3]), 42), arrayShuffle([1, 2, 3], 42) FROM numbers(10)", ""}},
+            .categories{"Array"}},
         FunctionFactory::CaseInsensitive);
+
     factory.registerFunction<FunctionArrayShuffleImpl<FunctionArrayPartialShuffleTraits>>(
-        {
-            R"(
+        FunctionDocumentation{
+            .description=R"(
 Returns an array of the same size as the original array where elements in range [1..limit] are a random
 subset of the original array. Remaining (limit..n] shall contain the elements not in [1..limit] range in undefined order.
 Value of limit shall be in range [1..n]. Values outside of that range are equivalent to performing full arrayShuffle:
@@ -213,14 +214,14 @@ If no seed is provided a random one will be used:
 It is possible to override the seed to produce stable results:
 [example:explicit_seed]
 )",
-            Documentation::Examples{
-                {"no_limit1", "SELECT arrayPartialShuffle([1, 2, 3, 4], 0)"},
-                {"no_limit2", "SELECT arrayPartialShuffle([1, 2, 3, 4])"},
-                {"random_seed", "SELECT arrayPartialShuffle([1, 2, 3, 4], 2)"},
-                {"explicit_seed", "SELECT arrayPartialShuffle([1, 2, 3, 4], 2, 41)"},
+            .examples{
+                {"no_limit1", "SELECT arrayPartialShuffle([1, 2, 3, 4], 0)", ""},
+                {"no_limit2", "SELECT arrayPartialShuffle([1, 2, 3, 4])", ""},
+                {"random_seed", "SELECT arrayPartialShuffle([1, 2, 3, 4], 2)", ""},
+                {"explicit_seed", "SELECT arrayPartialShuffle([1, 2, 3, 4], 2, 41)", ""},
                 {"materialize",
-                 "SELECT arrayPartialShuffle(materialize([1, 2, 3, 4]), 2, 42), arrayPartialShuffle([1, 2, 3], 2, 42) FROM numbers(10)"}},
-            Documentation::Categories{"Array"}},
+                 "SELECT arrayPartialShuffle(materialize([1, 2, 3, 4]), 2, 42), arrayPartialShuffle([1, 2, 3], 2, 42) FROM numbers(10)", ""}},
+            .categories{"Array"}},
         FunctionFactory::CaseInsensitive);
 }
 
diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp
index b4593ec910c..a853289e8cc 100644
--- a/src/Functions/array/arraySort.cpp
+++ b/src/Functions/array/arraySort.cpp
@@ -86,8 +86,8 @@ REGISTER_FUNCTION(ArraySort)
     factory.registerFunction<FunctionArraySort>();
     factory.registerFunction<FunctionArrayReverseSort>();
 
-    factory.registerFunction<FunctionArrayPartialSort>({
-        R"(
+    factory.registerFunction<FunctionArrayPartialSort>(FunctionDocumentation{
+        .description=R"(
 Returns an array of the same size as the original array where elements in range `[1..limit]`
 are sorted in ascending order. Remaining elements `(limit..N]` shall contain elements in unspecified order.
 [example:simple_int]
@@ -106,15 +106,16 @@ that the arguments of `func` will correspond to.
 
 For more details see documentation of `arraySort`.
 )",
-        Documentation::Examples{
-            {"simple_int", "SELECT arrayPartialSort(2, [5, 9, 1, 3])"},
-            {"simple_string", "SELECT arrayPartialSort(2, ['expenses','lasso','embolism','gladly'])"},
-            {"retain_sorted", "SELECT arrayResize(arrayPartialSort(2, [5, 9, 1, 3]), 2)"},
-            {"lambda_simple", "SELECT arrayPartialSort((x) -> -x, 2, [5, 9, 1, 3])"},
-            {"lambda_complex", "SELECT arrayPartialSort((x, y) -> -y, 1, [0, 1, 2], [1, 2, 3]) as res"}},
-        Documentation::Categories{"Array"}});
-    factory.registerFunction<FunctionArrayPartialReverseSort>({
-        R"(
+        .examples{
+            {"simple_int", "SELECT arrayPartialSort(2, [5, 9, 1, 3])", ""},
+            {"simple_string", "SELECT arrayPartialSort(2, ['expenses','lasso','embolism','gladly'])", ""},
+            {"retain_sorted", "SELECT arrayResize(arrayPartialSort(2, [5, 9, 1, 3]), 2)", ""},
+            {"lambda_simple", "SELECT arrayPartialSort((x) -> -x, 2, [5, 9, 1, 3])", ""},
+            {"lambda_complex", "SELECT arrayPartialSort((x, y) -> -y, 1, [0, 1, 2], [1, 2, 3]) as res", ""}},
+        .categories{"Array"}});
+
+    factory.registerFunction<FunctionArrayPartialReverseSort>(FunctionDocumentation{
+        .description=R"(
 Returns an array of the same size as the original array where elements in range `[1..limit]`
 are sorted in descending order. Remaining elements `(limit..N]` shall contain elements in unspecified order.
 [example:simple_int]
@@ -133,13 +134,13 @@ that the arguments of `func` will correspond to.
 
 For more details see documentation of `arraySort`.
 )",
-        Documentation::Examples{
-            {"simple_int", "SELECT arrayPartialReverseSort(2, [5, 9, 1, 3])"},
-            {"simple_string", "SELECT arrayPartialReverseSort(2, ['expenses','lasso','embolism','gladly'])"},
-            {"retain_sorted", "SELECT arrayResize(arrayPartialReverseSort(2, [5, 9, 1, 3]), 2)"},
-            {"lambda_simple", "SELECT arrayPartialReverseSort((x) -> -x, 2, [5, 9, 1, 3])"},
-            {"lambda_complex", "SELECT arrayPartialReverseSort((x, y) -> -y, 1, [0, 1, 2], [1, 2, 3]) as res"}},
-        Documentation::Categories{"Array"}});
+        .examples{
+            {"simple_int", "SELECT arrayPartialReverseSort(2, [5, 9, 1, 3])", ""},
+            {"simple_string", "SELECT arrayPartialReverseSort(2, ['expenses','lasso','embolism','gladly'])", ""},
+            {"retain_sorted", "SELECT arrayResize(arrayPartialReverseSort(2, [5, 9, 1, 3]), 2)", ""},
+            {"lambda_simple", "SELECT arrayPartialReverseSort((x) -> -x, 2, [5, 9, 1, 3])", ""},
+            {"lambda_complex", "SELECT arrayPartialReverseSort((x, y) -> -y, 1, [0, 1, 2], [1, 2, 3]) as res", ""}},
+        .categories{"Array"}});
 }
 
 }
diff --git a/src/Functions/array/length.cpp b/src/Functions/array/length.cpp
index 65555a501e8..f09ad17892b 100644
--- a/src/Functions/array/length.cpp
+++ b/src/Functions/array/length.cpp
@@ -66,8 +66,8 @@ using FunctionLength = FunctionStringOrArrayToT<LengthImpl, NameLength, UInt64,
 REGISTER_FUNCTION(Length)
 {
     factory.registerFunction<FunctionLength>(
-        {
-            R"(
+        FunctionDocumentation{
+            .description=R"(
 Calculates the length of the string or array.
 
 For String or FixedString argument: calculates the number of bytes in string.
@@ -87,18 +87,18 @@ and it is not the same as the visible string width.
 It is ok to have ASCII NUL bytes in strings, and they will be counted as well.
 [example:nul]
 )",
-            Documentation::Examples{
-                {"string1", "SELECT length('Hello, world!')"},
-                {"arr1", "SELECT length(['Hello'], ['world'])"},
+            .examples{
+                {"string1", "SELECT length('Hello, world!')", ""},
+                {"arr1", "SELECT length(['Hello'], ['world'])", ""},
                 {"constexpr", "WITH 'hello' || toString(number) AS str\n"
                               "SELECT str, \n"
                               "       isConstant(length(str)) AS str_length_is_constant, \n"
                               "       isConstant(length(str::FixedString(6))) AS fixed_str_length_is_constant\n"
-                              "FROM numbers(3)"},
-                {"unicode", "SELECT 'ёлка' AS str1, length(str1), lengthUTF8(str1), normalizeUTF8NFKD(str1) AS str2, length(str2), lengthUTF8(str2)"},
-                {"nul", R"(SELECT 'abc\0\0\0' AS str, length(str))"},
+                              "FROM numbers(3)", ""},
+                {"unicode", "SELECT 'ёлка' AS str1, length(str1), lengthUTF8(str1), normalizeUTF8NFKD(str1) AS str2, length(str2), lengthUTF8(str2)", ""},
+                {"nul", R"(SELECT 'abc\0\0\0' AS str, length(str))", ""},
                 },
-            Documentation::Categories{"String", "Array"}
+            .categories{"String", "Array"}
         },
         FunctionFactory::CaseInsensitive);
 }
diff --git a/src/Functions/ascii.cpp b/src/Functions/ascii.cpp
index 5632f0d15df..b43c3221391 100644
--- a/src/Functions/ascii.cpp
+++ b/src/Functions/ascii.cpp
@@ -82,14 +82,14 @@ using FunctionAscii = FunctionStringOrArrayToT<AsciiImpl, AsciiName, AsciiImpl::
 REGISTER_FUNCTION(Ascii)
 {
     factory.registerFunction<FunctionAscii>(
-        {
-        R"(
+        FunctionDocumentation{
+        .description=R"(
 Returns the ASCII code point of the first character of str.  The result type is Int32.
 
 If s is empty, the result is 0. If the first character is not an ASCII character or not part of the Latin-1 Supplement range of UTF-16, the result is undefined)
         )",
-        Documentation::Examples{{"ascii", "SELECT ascii('234')"}},
-        Documentation::Categories{"String"}
+        .examples{{"ascii", "SELECT ascii('234')", ""}},
+        .categories{"String"}
         }, FunctionFactory::CaseInsensitive);
 }
 
diff --git a/src/Functions/asin.cpp b/src/Functions/asin.cpp
index 264d2389974..3049b025d5e 100644
--- a/src/Functions/asin.cpp
+++ b/src/Functions/asin.cpp
@@ -14,9 +14,9 @@ using FunctionAsin = FunctionMathUnary<UnaryFunctionVectorized<AsinName, asin>>;
 
 REGISTER_FUNCTION(Asin)
 {
-    factory.registerFunction<FunctionAsin>(
+    factory.registerFunction<FunctionAsin>(FunctionDocumentation
         {
-            R"(
+            .description=R"(
 Calculates the arcsine of the argument.
 
 Takes arbitrary numeric type, which includes floating point and integer numbers, as well as big integers and decimals and returns Float64.
@@ -35,11 +35,11 @@ For arguments outside of this range, it returns nan:
 Every self-respectful data scientist knows how to apply arcsine to improve ads click-through rate with ClickHouse.
 For more details, see [https://en.wikipedia.org/wiki/Inverse_trigonometric_functions].
 )",
-            Documentation::Examples{
-                {"inverse", "SELECT asin(1.0) = pi() / 2, sin(asin(1)), asin(sin(1))"},
-                {"float32", "SELECT toTypeName(asin(1.0::Float32))"},
-                {"nan", "SELECT asin(1.1), asin(-2), asin(inf), asin(nan)"}},
-            Documentation::Categories{"Mathematical", "Trigonometric"}
+            .examples{
+                {"inverse", "SELECT asin(1.0) = pi() / 2, sin(asin(1)), asin(sin(1))", ""},
+                {"float32", "SELECT toTypeName(asin(1.0::Float32))", ""},
+                {"nan", "SELECT asin(1.1), asin(-2), asin(inf), asin(nan)", ""}},
+            .categories{"Mathematical", "Trigonometric"}
         },
         FunctionFactory::CaseInsensitive);
 }
diff --git a/src/Functions/canonicalRand.cpp b/src/Functions/canonicalRand.cpp
index 0f168142177..8eb064f4a32 100644
--- a/src/Functions/canonicalRand.cpp
+++ b/src/Functions/canonicalRand.cpp
@@ -47,13 +47,13 @@ public:
 
 REGISTER_FUNCTION(CanonicalRand)
 {
-    factory.registerFunction<FunctionCanonicalRand>({
-        R"(
+    factory.registerFunction<FunctionCanonicalRand>(FunctionDocumentation{
+        .description=R"(
 The function generates pseudo random results with independent and identically distributed uniformly distributed values in [0, 1).
 Non-deterministic. Return type is Float64.
         )",
-        Documentation::Examples{{"randCanonical", "SELECT randCanonical()"}},
-        Documentation::Categories{"Mathematical"}});
+        .examples{{"randCanonical", "SELECT randCanonical()", ""}},
+        .categories{"Mathematical"}});
 }
 
 }
diff --git a/src/Functions/concatWithSeparator.cpp b/src/Functions/concatWithSeparator.cpp
index 90f635ca97e..bfd1bc392db 100644
--- a/src/Functions/concatWithSeparator.cpp
+++ b/src/Functions/concatWithSeparator.cpp
@@ -155,21 +155,21 @@ using FunctionConcatWithSeparatorAssumeInjective = ConcatWithSeparatorImpl<NameC
 
 REGISTER_FUNCTION(ConcatWithSeparator)
 {
-    factory.registerFunction<FunctionConcatWithSeparator>({
-        R"(
+    factory.registerFunction<FunctionConcatWithSeparator>(FunctionDocumentation{
+        .description=R"(
 Returns the concatenation strings separated by string separator. Syntax: concatWithSeparator(sep, expr1, expr2, expr3...)
         )",
-        Documentation::Examples{{"concatWithSeparator", "SELECT concatWithSeparator('a', '1', '2', '3')"}},
-        Documentation::Categories{"String"}});
+        .examples{{"concatWithSeparator", "SELECT concatWithSeparator('a', '1', '2', '3')", ""}},
+        .categories{"String"}});
 
-    factory.registerFunction<FunctionConcatWithSeparatorAssumeInjective>({
-        R"(
+    factory.registerFunction<FunctionConcatWithSeparatorAssumeInjective>(FunctionDocumentation{
+        .description=R"(
 Same as concatWithSeparator, the difference is that you need to ensure that concatWithSeparator(sep, expr1, expr2, expr3...) → result is injective, it will be used for optimization of GROUP BY.
 
 The function is named “injective” if it always returns different result for different values of arguments. In other words: different arguments never yield identical result.
         )",
-        Documentation::Examples{{"concatWithSeparatorAssumeInjective", "SELECT concatWithSeparatorAssumeInjective('a', '1', '2', '3')"}},
-        Documentation::Categories{"String"}});
+        .examples{{"concatWithSeparatorAssumeInjective", "SELECT concatWithSeparatorAssumeInjective('a', '1', '2', '3')", ""}},
+        .categories{"String"}});
 
     /// Compatibility with Spark:
     factory.registerAlias("concat_ws", "concatWithSeparator", FunctionFactory::CaseInsensitive);
diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp
index b28c97d45b2..8361e9db166 100644
--- a/src/Functions/dateDiff.cpp
+++ b/src/Functions/dateDiff.cpp
@@ -457,16 +457,16 @@ REGISTER_FUNCTION(DateDiff)
 
 REGISTER_FUNCTION(TimeDiff)
 {
-    factory.registerFunction<FunctionTimeDiff>({R"(
+    factory.registerFunction<FunctionTimeDiff>(FunctionDocumentation{.description=R"(
 Returns the difference between two dates or dates with time values. The difference is calculated in seconds units (see toRelativeSecondNum).
 It is same as `dateDiff` and was added only for MySQL support. `dateDiff` is preferred.
 
 Example:
 [example:typical]
 )",
-    Documentation::Examples{
-        {"typical", "SELECT timeDiff(UTCTimestamp(), now());"}},
-    Documentation::Categories{"Dates and Times"}}, FunctionFactory::CaseInsensitive);
+    .examples{
+        {"typical", "SELECT timeDiff(UTCTimestamp(), now());", ""}},
+    .categories{"Dates and Times"}}, FunctionFactory::CaseInsensitive);
 }
 
 REGISTER_FUNCTION(Age)
diff --git a/src/Functions/divideDecimal.cpp b/src/Functions/divideDecimal.cpp
index b1e1296e790..1d0db232062 100644
--- a/src/Functions/divideDecimal.cpp
+++ b/src/Functions/divideDecimal.cpp
@@ -58,69 +58,54 @@ struct DivideDecimalsImpl
 
 REGISTER_FUNCTION(DivideDecimals)
 {
-    factory.registerFunction<FunctionsDecimalArithmetics<DivideDecimalsImpl>>(Documentation(
-        R"(
+    factory.registerFunction<FunctionsDecimalArithmetics<DivideDecimalsImpl>>(FunctionDocumentation{
+            .description = R"(
 Performs division on two decimals. Result value will be of type [Decimal256](../../sql-reference/data-types/decimal.md).
 Result scale can be explicitly specified by `result_scale` argument (const Integer in range `[0, 76]`). If not specified, the result scale is the max scale of given arguments.
 
 :::note
 These function work significantly slower than usual `divide`.
 In case you don't really need controlled precision and/or need fast computation, consider using [divide](#divide).
-:::
-
-**Syntax**
-
-```sql
-divideDecimal(a, b[, result_scale])
-```
-
-**Arguments**
-
--   `a` — First value: [Decimal](../../sql-reference/data-types/decimal.md).
--   `b` — Second value: [Decimal](../../sql-reference/data-types/decimal.md).
--   `result_scale` — Scale of result: [Int/UInt](../../sql-reference/data-types/int-uint.md).
-
-**Returned value**
-
--   The result of division with given scale.
-
-Type: [Decimal256](../../sql-reference/data-types/decimal.md).
-
-**Example**
-
-```text
+:::)",
+            .syntax = "divideDecimal(a, b[, result_scale])",
+            .arguments = {
+                {"a", "First value: [Decimal](../../sql-reference/data-types/decimal.md)"},
+                {"b", "Second value: [Decimal](../../sql-reference/data-types/decimal.md)."}
+            },
+            .returned_value = "The result of division with given scale. Type: [Decimal256](../../sql-reference/data-types/decimal.md).",
+            .examples = {
+                {"", "divideDecimal(toDecimal256(-12, 0), toDecimal32(2.1, 1), 10)",
+R"(
 ┌─divideDecimal(toDecimal256(-12, 0), toDecimal32(2.1, 1), 10)─┐
 │                                                -5.7142857142 │
 └──────────────────────────────────────────────────────────────┘
-```
+)"}, {"Difference to regular division",
 
-**Difference from regular division:**
-```sql
+R"(
 SELECT toDecimal64(-12, 1) / toDecimal32(2.1, 1);
 SELECT toDecimal64(-12, 1) as a, toDecimal32(2.1, 1) as b, divideDecimal(a, b, 1), divideDecimal(a, b, 5);
-```
-
-```text
+)",
+R"(
 ┌─divide(toDecimal64(-12, 1), toDecimal32(2.1, 1))─┐
 │                                             -5.7 │
 └──────────────────────────────────────────────────┘
 ┌───a─┬───b─┬─divideDecimal(toDecimal64(-12, 1), toDecimal32(2.1, 1), 1)─┬─divideDecimal(toDecimal64(-12, 1), toDecimal32(2.1, 1), 5)─┐
 │ -12 │ 2.1 │                                                       -5.7 │                                                   -5.71428 │
 └─────┴─────┴────────────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────┘
-```
-
-```sql
+)"
+},
+                {"",
+R"(
 SELECT toDecimal64(-12, 0) / toDecimal32(2.1, 1);
 SELECT toDecimal64(-12, 0) as a, toDecimal32(2.1, 1) as b, divideDecimal(a, b, 1), divideDecimal(a, b, 5);
-```
-
-```text
+)",
+R"(
 DB::Exception: Decimal result's scale is less than argument's one: While processing toDecimal64(-12, 0) / toDecimal32(2.1, 1). (ARGUMENT_OUT_OF_BOUND)
 ┌───a─┬───b─┬─divideDecimal(toDecimal64(-12, 0), toDecimal32(2.1, 1), 1)─┬─divideDecimal(toDecimal64(-12, 0), toDecimal32(2.1, 1), 5)─┐
 │ -12 │ 2.1 │                                                       -5.7 │                                                   -5.71428 │
 └─────┴─────┴────────────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────┘
-```
-)"));
+)"}
+    }});
 }
 
 }
diff --git a/src/Functions/factorial.cpp b/src/Functions/factorial.cpp
index b1a9d4a8415..b814e8198e6 100644
--- a/src/Functions/factorial.cpp
+++ b/src/Functions/factorial.cpp
@@ -97,15 +97,15 @@ template <> struct FunctionUnaryArithmeticMonotonicity<NameFactorial>
 
 REGISTER_FUNCTION(Factorial)
 {
-    factory.registerFunction<FunctionFactorial>(
+    factory.registerFunction<FunctionFactorial>(FunctionDocumentation
         {
-            R"(
+            .description=R"(
 Computes the factorial of an integer value. It works with any native integer type including UInt(8|16|32|64) and Int(8|16|32|64). The return type is UInt64.
 
 The factorial of 0 is 1. Likewise, the factorial() function returns 1 for any negative value. The maximum positive value for the input argument is 20, a value of 21 or greater will cause exception throw.
 )",
-            Documentation::Examples{{"factorial", "SELECT factorial(10)"}},
-            Documentation::Categories{"Mathematical"}},
+            .examples{{"factorial", "SELECT factorial(10)", ""}},
+            .categories{"Mathematical"}},
         FunctionFactory::CaseInsensitive);
 }
 
diff --git a/src/Functions/formatReadableDecimalSize.cpp b/src/Functions/formatReadableDecimalSize.cpp
index 184b574abdf..b6fd0de8f7b 100644
--- a/src/Functions/formatReadableDecimalSize.cpp
+++ b/src/Functions/formatReadableDecimalSize.cpp
@@ -21,13 +21,13 @@ namespace
 REGISTER_FUNCTION(FormatReadableDecimalSize)
 {
     factory.registerFunction<FunctionFormatReadable<Impl>>(
-    {
-        R"(
+    FunctionDocumentation{
+        .description=R"(
 Accepts the size (number of bytes). Returns a rounded size with a suffix (KB, MB, etc.) as a string.
 )",
-        Documentation::Examples{
-            {"formatReadableDecimalSize", "SELECT formatReadableDecimalSize(1000)"}},
-        Documentation::Categories{"OtherFunctions"}
+        .examples{
+            {"formatReadableDecimalSize", "SELECT formatReadableDecimalSize(1000)", ""}},
+        .categories{"OtherFunctions"}
     },
     FunctionFactory::CaseSensitive);
 }
diff --git a/src/Functions/generateULID.cpp b/src/Functions/generateULID.cpp
index a8a2a2174cb..9c5c9403185 100644
--- a/src/Functions/generateULID.cpp
+++ b/src/Functions/generateULID.cpp
@@ -74,17 +74,17 @@ public:
 
 REGISTER_FUNCTION(GenerateULID)
 {
-    factory.registerFunction<FunctionGenerateULID>(
+    factory.registerFunction<FunctionGenerateULID>(FunctionDocumentation
     {
-        R"(
+        .description=R"(
 Generates a Universally Unique Lexicographically Sortable Identifier (ULID).
 This function takes an optional argument, the value of which is discarded to generate different values in case the function is called multiple times.
 The function returns a value of type FixedString(26).
 )",
-        Documentation::Examples{
-            {"ulid", "SELECT generateULID()"},
-            {"multiple", "SELECT generateULID(1), generateULID(2)"}},
-        Documentation::Categories{"ULID"}
+        .examples{
+            {"ulid", "SELECT generateULID()", ""},
+            {"multiple", "SELECT generateULID(1), generateULID(2)", ""}},
+        .categories{"ULID"}
     },
     FunctionFactory::CaseSensitive);
 }
diff --git a/src/Functions/getSubcolumn.cpp b/src/Functions/getSubcolumn.cpp
index b2f63ea4c75..f041165385e 100644
--- a/src/Functions/getSubcolumn.cpp
+++ b/src/Functions/getSubcolumn.cpp
@@ -53,14 +53,14 @@ private:
 
 REGISTER_FUNCTION(GetSubcolumn)
 {
-    factory.registerFunction<FunctionGetSubcolumn>({
-        R"(
+    factory.registerFunction<FunctionGetSubcolumn>(FunctionDocumentation{
+        .description=R"(
 Receives the expression or identifier and constant string with the name of subcolumn.
 
 Returns requested subcolumn extracted from the expression.
 )",
-        Documentation::Examples{{"getSubcolumn", "SELECT getSubcolumn(array_col, 'size0'), getSubcolumn(tuple_col, 'elem_name')"}},
-        Documentation::Categories{"OtherFunctions"}
+        .examples{{"getSubcolumn", "SELECT getSubcolumn(array_col, 'size0'), getSubcolumn(tuple_col, 'elem_name')", ""}},
+        .categories{"OtherFunctions"}
     });
 }
 
diff --git a/src/Functions/hasToken.cpp b/src/Functions/hasToken.cpp
index ba1ff9b3ab5..de67e80e752 100644
--- a/src/Functions/hasToken.cpp
+++ b/src/Functions/hasToken.cpp
@@ -1,10 +1,10 @@
-#include "FunctionFactory.h"
-#include "FunctionsStringSearch.h"
-#include "HasTokenImpl.h"
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionsStringSearch.h>
+#include <Functions/HasTokenImpl.h>
 
 #include <Common/Volnitsky.h>
 
-namespace
+namespace DB
 {
 struct NameHasToken
 {
@@ -19,14 +19,15 @@ struct NameHasTokenOrNull
 using FunctionHasToken = DB::FunctionsStringSearch<DB::HasTokenImpl<NameHasToken, DB::VolnitskyCaseSensitiveToken, false>>;
 using FunctionHasTokenOrNull = DB::
     FunctionsStringSearch<DB::HasTokenImpl<NameHasTokenOrNull, DB::VolnitskyCaseSensitiveToken, false>, DB::ExecutionErrorPolicy::Null>;
-}
 
 REGISTER_FUNCTION(HasToken)
 {
-    factory.registerFunction<FunctionHasToken>(
-        {"Performs lookup of needle in haystack using tokenbf_v1 index."}, DB::FunctionFactory::CaseSensitive);
+    factory.registerFunction<FunctionHasToken>(FunctionDocumentation
+        {.description="Performs lookup of needle in haystack using tokenbf_v1 index."}, DB::FunctionFactory::CaseSensitive);
 
-    factory.registerFunction<FunctionHasTokenOrNull>(
-        {"Performs lookup of needle in haystack using tokenbf_v1 index. Returns null if needle is ill-formed."},
+    factory.registerFunction<FunctionHasTokenOrNull>(FunctionDocumentation
+        {.description="Performs lookup of needle in haystack using tokenbf_v1 index. Returns null if needle is ill-formed."},
         DB::FunctionFactory::CaseSensitive);
 }
+
+}
diff --git a/src/Functions/hasTokenCaseInsensitive.cpp b/src/Functions/hasTokenCaseInsensitive.cpp
index 4fc19ca0784..a6e8ecf3e9d 100644
--- a/src/Functions/hasTokenCaseInsensitive.cpp
+++ b/src/Functions/hasTokenCaseInsensitive.cpp
@@ -1,10 +1,10 @@
-#include "FunctionFactory.h"
-#include "FunctionsStringSearch.h"
-#include "HasTokenImpl.h"
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionsStringSearch.h>
+#include <Functions/HasTokenImpl.h>
 
 #include <Common/Volnitsky.h>
 
-namespace
+namespace DB
 {
 struct NameHasTokenCaseInsensitive
 {
@@ -21,14 +21,16 @@ using FunctionHasTokenCaseInsensitive
 using FunctionHasTokenCaseInsensitiveOrNull = DB::FunctionsStringSearch<
     DB::HasTokenImpl<NameHasTokenCaseInsensitiveOrNull, DB::VolnitskyCaseInsensitiveToken, false>,
     DB::ExecutionErrorPolicy::Null>;
-}
 
 REGISTER_FUNCTION(HasTokenCaseInsensitive)
 {
     factory.registerFunction<FunctionHasTokenCaseInsensitive>(
-        {"Performs case insensitive lookup of needle in haystack using tokenbf_v1 index."}, DB::FunctionFactory::CaseInsensitive);
+        FunctionDocumentation{.description="Performs case insensitive lookup of needle in haystack using tokenbf_v1 index."},
+        DB::FunctionFactory::CaseInsensitive);
 
     factory.registerFunction<FunctionHasTokenCaseInsensitiveOrNull>(
-        {"Performs case insensitive lookup of needle in haystack using tokenbf_v1 index. Returns null if needle is ill-formed."},
+        FunctionDocumentation{.description="Performs case insensitive lookup of needle in haystack using tokenbf_v1 index. Returns null if needle is ill-formed."},
         DB::FunctionFactory::CaseInsensitive);
 }
+
+}
diff --git a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
index 4078b88c23e..3e5a1de4fae 100644
--- a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
+++ b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
@@ -135,8 +135,8 @@ struct NameExtractKeyValuePairsWithEscaping
 REGISTER_FUNCTION(ExtractKeyValuePairs)
 {
     factory.registerFunction<ExtractKeyValuePairs<NameExtractKeyValuePairs, false>>(
-        Documentation(
-            R"(Extracts key-value pairs from any string. The string does not need to be 100% structured in a key value pair format;
+        FunctionDocumentation{
+            .description=R"(Extracts key-value pairs from any string. The string does not need to be 100% structured in a key value pair format;
 
             It can contain noise (e.g. log files). The key-value pair format to be interpreted should be specified via function arguments.
 
@@ -197,12 +197,12 @@ REGISTER_FUNCTION(ExtractKeyValuePairs)
             ┌─kv────────────────────┐
             │ {'age':'a\\x0A\\n\\0'} │
             └───────────────────────┘
-            ```)")
+            ```)"}
     );
 
     factory.registerFunction<ExtractKeyValuePairs<NameExtractKeyValuePairsWithEscaping, true>>(
-        Documentation(
-            R"(Same as `extractKeyValuePairs` but with escaping support.
+        FunctionDocumentation{
+            .description=R"(Same as `extractKeyValuePairs` but with escaping support.
 
             Escape sequences supported: `\x`, `\N`, `\a`, `\b`, `\e`, `\f`, `\n`, `\r`, `\t`, `\v` and `\0`.
             Non standard escape sequences are returned as it is (including the backslash) unless they are one of the following:
@@ -226,7 +226,7 @@ REGISTER_FUNCTION(ExtractKeyValuePairs)
             ┌─kv───────────────┐
             │ {'age':'a\n\n\0'} │
             └──────────────────┘
-            ```)")
+            ```)"}
     );
 }
 
diff --git a/src/Functions/modulo.cpp b/src/Functions/modulo.cpp
index 273bb8832ea..8b8919f7b26 100644
--- a/src/Functions/modulo.cpp
+++ b/src/Functions/modulo.cpp
@@ -174,15 +174,15 @@ using FunctionPositiveModulo = BinaryArithmeticOverloadResolver<PositiveModuloIm
 
 REGISTER_FUNCTION(PositiveModulo)
 {
-    factory.registerFunction<FunctionPositiveModulo>(
+    factory.registerFunction<FunctionPositiveModulo>(FunctionDocumentation
         {
-            R"(
+            .description=R"(
 Calculates the remainder when dividing `a` by `b`. Similar to function `modulo` except that `positiveModulo` always return non-negative number.
 Returns the difference between `a` and the nearest integer not greater than `a` divisible by `b`.
 In other words, the function returning the modulus (modulo) in the terms of Modular Arithmetic.
         )",
-            Documentation::Examples{{"positiveModulo", "SELECT positiveModulo(-1, 10);"}},
-            Documentation::Categories{"Arithmetic"}},
+            .examples{{"positiveModulo", "SELECT positiveModulo(-1, 10);", ""}},
+            .categories{"Arithmetic"}},
         FunctionFactory::CaseInsensitive);
 
     factory.registerAlias("positive_modulo", "positiveModulo", FunctionFactory::CaseInsensitive);
diff --git a/src/Functions/mortonDecode.cpp b/src/Functions/mortonDecode.cpp
index 337fd5e3a38..126635a4521 100644
--- a/src/Functions/mortonDecode.cpp
+++ b/src/Functions/mortonDecode.cpp
@@ -380,8 +380,8 @@ private:
 
 REGISTER_FUNCTION(MortonDecode)
 {
-    factory.registerFunction<FunctionMortonDecode>({
-        R"(
+    factory.registerFunction<FunctionMortonDecode>(FunctionDocumentation{
+        .description=R"(
 Decodes a Morton encoding (ZCurve) into the corresponding unsigned integer tuple
 
 The function has two modes of operation:
@@ -418,15 +418,15 @@ The function accepts a column of codes as a second argument:
 The range tuple must be a constant:
 [example:from_table_range]
 )",
-        Documentation::Examples{
-            {"simple", "SELECT mortonDecode(4, 2149)"},
-            {"range_shrank", "SELECT mortonDecode((1,2), 1572864)"},
-            {"identity", "SELECT mortonDecode(1, 1)"},
-            {"identity_shrank", "SELECT mortonDecode(tuple(2), 32768)"},
-            {"from_table", "SELECT mortonDecode(2, code) FROM table"},
-            {"from_table_range", "SELECT mortonDecode((1,2), code) FROM table"},
+        .examples{
+            {"simple", "SELECT mortonDecode(4, 2149)", ""},
+            {"range_shrank", "SELECT mortonDecode((1,2), 1572864)", ""},
+            {"identity", "SELECT mortonDecode(1, 1)", ""},
+            {"identity_shrank", "SELECT mortonDecode(tuple(2), 32768)", ""},
+            {"from_table", "SELECT mortonDecode(2, code) FROM table", ""},
+            {"from_table_range", "SELECT mortonDecode((1,2), code) FROM table", ""},
             },
-        Documentation::Categories {"ZCurve", "Morton coding"}
+        .categories {"ZCurve", "Morton coding"}
     });
 }
 
diff --git a/src/Functions/mortonEncode.cpp b/src/Functions/mortonEncode.cpp
index 4bdd237fa9c..17d8077304d 100644
--- a/src/Functions/mortonEncode.cpp
+++ b/src/Functions/mortonEncode.cpp
@@ -335,8 +335,8 @@ private:
 
 REGISTER_FUNCTION(MortonEncode)
 {
-    factory.registerFunction<FunctionMortonEncode>({
-    R"(
+    factory.registerFunction<FunctionMortonEncode>(FunctionDocumentation{
+    .description=R"(
 Calculates Morton encoding (ZCurve) for a list of unsigned integers
 
 The function has two modes of operation:
@@ -378,15 +378,15 @@ Two arguments will have a range of maximum 2^32 (64/2) each
 Three arguments: range of max 2^21 (64/3) each
 And so on, all overflow will be clamped to zero
 )",
-        Documentation::Examples{
-            {"simple", "SELECT mortonEncode(1, 2, 3)"},
-            {"range_expanded", "SELECT mortonEncode((1,2), 1024, 16)"},
-            {"identity", "SELECT mortonEncode(1)"},
-            {"identity_expanded", "SELECT mortonEncode(tuple(2), 128)"},
-            {"from_table", "SELECT mortonEncode(n1, n2) FROM table"},
-            {"from_table_range", "SELECT mortonEncode((1,2), n1, n2) FROM table"},
+        .examples{
+            {"simple", "SELECT mortonEncode(1, 2, 3)", ""},
+            {"range_expanded", "SELECT mortonEncode((1,2), 1024, 16)", ""},
+            {"identity", "SELECT mortonEncode(1)", ""},
+            {"identity_expanded", "SELECT mortonEncode(tuple(2), 128)", ""},
+            {"from_table", "SELECT mortonEncode(n1, n2) FROM table", ""},
+            {"from_table_range", "SELECT mortonEncode((1,2), n1, n2) FROM table", ""},
             },
-        Documentation::Categories {"ZCurve", "Morton coding"}
+        .categories {"ZCurve", "Morton coding"}
     });
 }
 
diff --git a/src/Functions/multiplyDecimal.cpp b/src/Functions/multiplyDecimal.cpp
index 1ec2d4acc9b..ed6487c6683 100644
--- a/src/Functions/multiplyDecimal.cpp
+++ b/src/Functions/multiplyDecimal.cpp
@@ -58,8 +58,8 @@ struct MultiplyDecimalsImpl
 
 REGISTER_FUNCTION(MultiplyDecimals)
 {
-    factory.registerFunction<FunctionsDecimalArithmetics<MultiplyDecimalsImpl>>(Documentation(
-        R"(
+    factory.registerFunction<FunctionsDecimalArithmetics<MultiplyDecimalsImpl>>(FunctionDocumentation{
+        .description=R"(
 Performs multiplication on two decimals. Result value will be of type [Decimal256](../../sql-reference/data-types/decimal.md).
 Result scale can be explicitly specified by `result_scale` argument (const Integer in range `[0, 76]`). If not specified, the result scale is the max scale of given arguments.
 
@@ -127,7 +127,7 @@ SELECT
 Received exception from server (version 22.11.1):
 Code: 407. DB::Exception: Received from localhost:9000. DB::Exception: Decimal math overflow: While processing toDecimal64(-12.647987876, 9) AS a, toDecimal64(123.967645643, 9) AS b, a * b. (DECIMAL_OVERFLOW)
 ```
-)"));
+)"});
 
 }
 
diff --git a/src/Functions/nested.cpp b/src/Functions/nested.cpp
index b22330cd881..0a094176a55 100644
--- a/src/Functions/nested.cpp
+++ b/src/Functions/nested.cpp
@@ -165,12 +165,12 @@ private:
 
 REGISTER_FUNCTION(Nested)
 {
-    factory.registerFunction<FunctionNested>({
-        R"(
+    factory.registerFunction<FunctionNested>(FunctionDocumentation{
+        .description=R"(
 Returns the array of tuples from multiple arrays.
 )",
-        Documentation::Examples{{"nested", "SELECT nested(['keys', 'values'], ['key_1', 'key_2'], ['value_1','value_2'])"}},
-        Documentation::Categories{"OtherFunctions"}
+        .examples{{"nested", "SELECT nested(['keys', 'values'], ['key_1', 'key_2'], ['value_1','value_2'])", ""}},
+        .categories{"OtherFunctions"}
     });
 }
 
diff --git a/src/Functions/randDistribution.cpp b/src/Functions/randDistribution.cpp
index f92a2c7c034..725ae0b4e64 100644
--- a/src/Functions/randDistribution.cpp
+++ b/src/Functions/randDistribution.cpp
@@ -310,96 +310,96 @@ public:
 REGISTER_FUNCTION(Distribution)
 {
     factory.registerFunction<FunctionRandomDistribution<UniformDistribution>>(
-    {
-    R"(
+    FunctionDocumentation{
+    .description=R"(
 Returns a random number from the uniform distribution in the specified range.
 Accepts two parameters - minimum bound and maximum bound.
 
 Typical usage:
 [example:typical]
 )",
-    Documentation::Examples{
-        {"typical", "SELECT randUniform(0, 1) FROM numbers(100000);"}},
-    Documentation::Categories{"Distribution"}
+    .examples{
+        {"typical", "SELECT randUniform(0, 1) FROM numbers(100000);", ""}},
+    .categories{"Distribution"}
     });
 
     factory.registerFunction<FunctionRandomDistribution<NormalDistribution>>(
-    {
-    R"(
+    FunctionDocumentation{
+    .description=R"(
 Returns a random number from the normal distribution.
 Accepts two parameters - mean and variance.
 
 Typical usage:
 [example:typical]
 )",
-    Documentation::Examples{
-        {"typical", "SELECT randNormal(0, 5) FROM numbers(100000);"}},
-    Documentation::Categories{"Distribution"}
+    .examples{
+        {"typical", "SELECT randNormal(0, 5) FROM numbers(100000);", ""}},
+    .categories{"Distribution"}
     });
 
 
     factory.registerFunction<FunctionRandomDistribution<LogNormalDistribution>>(
-    {
-    R"(
+    FunctionDocumentation{
+    .description=R"(
 Returns a random number from the lognormal distribution (a distribution of a random variable whose logarithm is normally distributed).
 Accepts two parameters - mean and variance.
 
 Typical usage:
 [example:typical]
 )",
-    Documentation::Examples{
-        {"typical", "SELECT randLogNormal(0, 5) FROM numbers(100000);"}},
-    Documentation::Categories{"Distribution"}
+    .examples{
+        {"typical", "SELECT randLogNormal(0, 5) FROM numbers(100000);", ""}},
+    .categories{"Distribution"}
     });
 
 
     factory.registerFunction<FunctionRandomDistribution<ExponentialDistribution>>(
-    {
-    R"(
+    FunctionDocumentation{
+    .description=R"(
 Returns a random number from the exponential distribution.
 Accepts one parameter - lambda value.
 
 Typical usage:
 [example:typical]
 )",
-    Documentation::Examples{
-        {"typical", "SELECT randExponential(0, 5) FROM numbers(100000);"}},
-    Documentation::Categories{"Distribution"}
+    .examples{
+        {"typical", "SELECT randExponential(0, 5) FROM numbers(100000);", ""}},
+    .categories{"Distribution"}
     });
 
 
     factory.registerFunction<FunctionRandomDistribution<ChiSquaredDistribution>>(
-    {
-    R"(
+    FunctionDocumentation{
+    .description=R"(
 Returns a random number from the chi-squared distribution (a distribution of a sum of the squares of k independent standard normal random variables).
 Accepts one parameter - degree of freedom.
 
 Typical usage:
 [example:typical]
 )",
-    Documentation::Examples{
-        {"typical", "SELECT randChiSquared(5) FROM numbers(100000);"}},
-    Documentation::Categories{"Distribution"}
+    .examples{
+        {"typical", "SELECT randChiSquared(5) FROM numbers(100000);", ""}},
+    .categories{"Distribution"}
     });
 
     factory.registerFunction<FunctionRandomDistribution<StudentTDistribution>>(
-    {
-    R"(
+    FunctionDocumentation{
+    .description=R"(
 Returns a random number from the t-distribution.
 Accepts one parameter - degree of freedom.
 
 Typical usage:
 [example:typical]
 )",
-    Documentation::Examples{
-        {"typical", "SELECT randStudentT(5) FROM numbers(100000);"}},
-    Documentation::Categories{"Distribution"}
+    .examples{
+        {"typical", "SELECT randStudentT(5) FROM numbers(100000);", ""}},
+    .categories{"Distribution"}
     });
 
 
     factory.registerFunction<FunctionRandomDistribution<FisherFDistribution>>(
-    {
-    R"(
+    FunctionDocumentation{
+    .description=R"(
 Returns a random number from the f-distribution.
 The F-distribution is the distribution of X = (S1 / d1) / (S2 / d2) where d1 and d2 are degrees of freedom.
 Accepts two parameters - degrees of freedom.
@@ -407,69 +407,69 @@ Accepts two parameters - degrees of freedom.
 Typical usage:
 [example:typical]
 )",
-    Documentation::Examples{
-        {"typical", "SELECT randFisherF(5) FROM numbers(100000);"}},
-    Documentation::Categories{"Distribution"}
+    .examples{
+        {"typical", "SELECT randFisherF(5) FROM numbers(100000);", ""}},
+    .categories{"Distribution"}
     });
 
 
     factory.registerFunction<FunctionRandomDistribution<BernoulliDistribution>>(
-    {
-    R"(
+    FunctionDocumentation{
+    .description=R"(
 Returns a random number from the Bernoulli distribution.
 Accepts one parameter - probability of success.
 
 Typical usage:
 [example:typical]
 )",
-    Documentation::Examples{
-        {"typical", "SELECT randBernoulli(0.1) FROM numbers(100000);"}},
-    Documentation::Categories{"Distribution"}
+    .examples{
+        {"typical", "SELECT randBernoulli(0.1) FROM numbers(100000);", ""}},
+    .categories{"Distribution"}
     });
 
 
     factory.registerFunction<FunctionRandomDistribution<BinomialDistribution>>(
-    {
-    R"(
+    FunctionDocumentation{
+    .description=R"(
 Returns a random number from the binomial distribution.
 Accepts two parameters - number of experiments and probability of success in each experiment.
 
 Typical usage:
 [example:typical]
 )",
-    Documentation::Examples{
-        {"typical", "SELECT randBinomial(10, 0.1) FROM numbers(100000);"}},
-    Documentation::Categories{"Distribution"}
+    .examples{
+        {"typical", "SELECT randBinomial(10, 0.1) FROM numbers(100000);", ""}},
+    .categories{"Distribution"}
     });
 
 
     factory.registerFunction<FunctionRandomDistribution<NegativeBinomialDistribution>>(
-    {
-    R"(
+    FunctionDocumentation{
+    .description=R"(
 Returns a random number from the negative binomial distribution.
 Accepts two parameters - number of experiments and probability of success in each experiment.
 
 Typical usage:
 [example:typical]
 )",
-    Documentation::Examples{
-        {"typical", "SELECT randNegativeBinomial(10, 0.1) FROM numbers(100000);"}},
-    Documentation::Categories{"Distribution"}
+    .examples{
+        {"typical", "SELECT randNegativeBinomial(10, 0.1) FROM numbers(100000);", ""}},
+    .categories{"Distribution"}
     });
 
 
     factory.registerFunction<FunctionRandomDistribution<PoissonDistribution>>(
-    {
-    R"(
+    FunctionDocumentation{
+    .description=R"(
 Returns a random number from the poisson distribution.
 Accepts one parameter - the mean number of occurrences.
 
 Typical usage:
 [example:typical]
 )",
-    Documentation::Examples{
-        {"typical", "SELECT randPoisson(3) FROM numbers(100000);"}},
-    Documentation::Categories{"Distribution"}
+    .examples{
+        {"typical", "SELECT randPoisson(3) FROM numbers(100000);", ""}},
+    .categories{"Distribution"}
     });
 }
 
diff --git a/src/Functions/regexpExtract.cpp b/src/Functions/regexpExtract.cpp
index 22f1b3599ba..2b3f0b2088d 100644
--- a/src/Functions/regexpExtract.cpp
+++ b/src/Functions/regexpExtract.cpp
@@ -8,7 +8,7 @@
 #include <Functions/Regexps.h>
 #include <Interpreters/Context.h>
 #include <base/StringRef.h>
-#include <Common/Documentation.h>
+#include <Common/FunctionDocumentation.h>
 
 namespace DB
 {
@@ -244,7 +244,7 @@ private:
 REGISTER_FUNCTION(RegexpExtract)
 {
     factory.registerFunction<FunctionRegexpExtract>(
-        Documentation{"Extracts the first string in haystack that matches the regexp pattern and corresponds to the regex group index."});
+        FunctionDocumentation{.description="Extracts the first string in haystack that matches the regexp pattern and corresponds to the regex group index."});
 
     /// For Spark compatibility.
     factory.registerAlias("REGEXP_EXTRACT", "regexpExtract", FunctionFactory::CaseInsensitive);
diff --git a/src/Functions/reverseDNSQuery.cpp b/src/Functions/reverseDNSQuery.cpp
index 65ce5e859c5..b4d963a6a15 100644
--- a/src/Functions/reverseDNSQuery.cpp
+++ b/src/Functions/reverseDNSQuery.cpp
@@ -99,43 +99,19 @@ public:
 REGISTER_FUNCTION(ReverseDNSQuery)
 {
     factory.registerFunction<ReverseDNSQuery>(
-        Documentation(
-            R"(Performs a reverse DNS query to get the PTR records associated with the IP address.
-
-                **Syntax**
-
-                ``` sql
-                reverseDNSQuery(address)
-                ```
-
-                This function performs reverse DNS resolutions on both IPv4 and IPv6.
-
-                **Arguments**
-
-                -   `address` — An IPv4 or IPv6 address. [String](../../sql-reference/data-types/string.md).
-
-                **Returned value**
-
-                -   Associated domains (PTR records).
-
-                Type: Type: [Array(String)](../../sql-reference/data-types/array.md).
-
-                **Example**
-
-                Query:
-
-                ``` sql
-                SELECT reverseDNSQuery('192.168.0.2');
-                ```
-
-                Result:
-
-                ``` text
-                ┌─reverseDNSQuery('192.168.0.2')────────────┐
-                │ ['test2.example.com','test3.example.com'] │
-                └───────────────────────────────────────────┘
-                ```
-                )")
+            FunctionDocumentation{
+                .description = R"(Performs a reverse DNS query to get the PTR records associated with the IP address)",
+                .syntax = "reverseDNSQuery(address)",
+                .arguments = {{"address", "An IPv4 or IPv6 address. [String](../../sql-reference/data-types/string.md)"}},
+                .returned_value = "Associated domains (PTR records). [String](../../sql-reference/data-types/string.md).",
+                .examples = {{"",
+                              "SELECT reverseDNSQuery('192.168.0.2');",
+R"(
+┌─reverseDNSQuery('192.168.0.2')────────────┐
+│ ['test2.example.com','test3.example.com'] │
+└───────────────────────────────────────────┘
+)"}}
+            }
     );
 }
 
diff --git a/src/Functions/serverConstants.cpp b/src/Functions/serverConstants.cpp
index 96615d0a4c9..19a0d82caf4 100644
--- a/src/Functions/serverConstants.cpp
+++ b/src/Functions/serverConstants.cpp
@@ -184,15 +184,15 @@ REGISTER_FUNCTION(GetOSKernelVersion)
 
 REGISTER_FUNCTION(DisplayName)
 {
-    factory.registerFunction<FunctionDisplayName>(
+    factory.registerFunction<FunctionDisplayName>(FunctionDocumentation
         {
-            R"(
+            .description=R"(
 Returns the value of `display_name` from config or server FQDN if not set.
 
 [example:displayName]
 )",
-            Documentation::Examples{{"displayName", "SELECT displayName();"}},
-            Documentation::Categories{"Constant", "Miscellaneous"}
+            .examples{{"displayName", "SELECT displayName();", ""}},
+            .categories{"Constant", "Miscellaneous"}
         },
         FunctionFactory::CaseSensitive);
 }
diff --git a/src/Functions/soundex.cpp b/src/Functions/soundex.cpp
index 5318339295d..0cddfc90f7c 100644
--- a/src/Functions/soundex.cpp
+++ b/src/Functions/soundex.cpp
@@ -112,7 +112,7 @@ struct NameSoundex
 REGISTER_FUNCTION(Soundex)
 {
     factory.registerFunction<FunctionStringToString<SoundexImpl, NameSoundex>>(
-        Documentation{"Returns Soundex code of a string."}, FunctionFactory::CaseInsensitive);
+        FunctionDocumentation{.description="Returns Soundex code of a string."}, FunctionFactory::CaseInsensitive);
 }
 
 
diff --git a/src/Functions/tryDecrypt.cpp b/src/Functions/tryDecrypt.cpp
index 0992d5f4308..eccccb1e4a4 100644
--- a/src/Functions/tryDecrypt.cpp
+++ b/src/Functions/tryDecrypt.cpp
@@ -1,4 +1,4 @@
-#include <Common/Documentation.h>
+#include <Common/FunctionDocumentation.h>
 #include "config.h"
 
 #if USE_SSL
@@ -23,8 +23,8 @@ namespace DB
 
 REGISTER_FUNCTION(TryDecrypt)
 {
-    factory.registerFunction<FunctionDecrypt<TryDecryptImpl>>(Documentation(
-        "Similar to `decrypt`, but returns NULL if decryption fails because of using the wrong key."));
+    factory.registerFunction<FunctionDecrypt<TryDecryptImpl>>(FunctionDocumentation{
+        .description="Similar to `decrypt`, but returns NULL if decryption fails because of using the wrong key."});
 }
 
 }
diff --git a/src/Functions/vectorFunctions.cpp b/src/Functions/vectorFunctions.cpp
index a7ab09612cf..a063dd9b05c 100644
--- a/src/Functions/vectorFunctions.cpp
+++ b/src/Functions/vectorFunctions.cpp
@@ -1553,33 +1553,33 @@ REGISTER_FUNCTION(VectorFunctions)
     factory.registerFunction<FunctionTupleDivide>();
     factory.registerFunction<FunctionTupleNegate>();
 
-    factory.registerFunction<FunctionAddTupleOfIntervals>(
+    factory.registerFunction<FunctionAddTupleOfIntervals>(FunctionDocumentation
         {
-            R"(
+            .description=R"(
 Consecutively adds a tuple of intervals to a Date or a DateTime.
 [example:tuple]
 )",
-            Documentation::Examples{
-                {"tuple", "WITH toDate('2018-01-01') AS date SELECT addTupleOfIntervals(date, (INTERVAL 1 DAY, INTERVAL 1 YEAR))"},
+            .examples{
+                {"tuple", "WITH toDate('2018-01-01') AS date SELECT addTupleOfIntervals(date, (INTERVAL 1 DAY, INTERVAL 1 YEAR))", ""},
                 },
-            Documentation::Categories{"Tuple", "Interval", "Date", "DateTime"}
+            .categories{"Tuple", "Interval", "Date", "DateTime"}
         });
 
-    factory.registerFunction<FunctionSubtractTupleOfIntervals>(
+    factory.registerFunction<FunctionSubtractTupleOfIntervals>(FunctionDocumentation
         {
-            R"(
+            .description=R"(
 Consecutively subtracts a tuple of intervals from a Date or a DateTime.
 [example:tuple]
 )",
-            Documentation::Examples{
-                {"tuple", "WITH toDate('2018-01-01') AS date SELECT subtractTupleOfIntervals(date, (INTERVAL 1 DAY, INTERVAL 1 YEAR))"},
+            .examples{
+                {"tuple", "WITH toDate('2018-01-01') AS date SELECT subtractTupleOfIntervals(date, (INTERVAL 1 DAY, INTERVAL 1 YEAR))", ""},
                 },
-            Documentation::Categories{"Tuple", "Interval", "Date", "DateTime"}
+            .categories{"Tuple", "Interval", "Date", "DateTime"}
         });
 
-    factory.registerFunction<FunctionTupleAddInterval>(
+    factory.registerFunction<FunctionTupleAddInterval>(FunctionDocumentation
         {
-            R"(
+            .description=R"(
 Adds an interval to another interval or tuple of intervals. The returned value is tuple of intervals.
 [example:tuple]
 [example:interval1]
@@ -1587,16 +1587,16 @@ Adds an interval to another interval or tuple of intervals. The returned value i
 If the types of the first interval (or the interval in the tuple) and the second interval are the same they will be merged into one interval.
 [example:interval2]
 )",
-            Documentation::Examples{
-                {"tuple", "SELECT addInterval((INTERVAL 1 DAY, INTERVAL 1 YEAR), INTERVAL 1 MONTH)"},
-                {"interval1", "SELECT addInterval(INTERVAL 1 DAY, INTERVAL 1 MONTH)"},
-                {"interval2", "SELECT addInterval(INTERVAL 1 DAY, INTERVAL 1 DAY)"},
+            .examples{
+                {"tuple", "SELECT addInterval((INTERVAL 1 DAY, INTERVAL 1 YEAR), INTERVAL 1 MONTH)", ""},
+                {"interval1", "SELECT addInterval(INTERVAL 1 DAY, INTERVAL 1 MONTH)", ""},
+                {"interval2", "SELECT addInterval(INTERVAL 1 DAY, INTERVAL 1 DAY)", ""},
                 },
-            Documentation::Categories{"Tuple", "Interval"}
+            .categories{"Tuple", "Interval"}
         });
-    factory.registerFunction<FunctionTupleSubtractInterval>(
+    factory.registerFunction<FunctionTupleSubtractInterval>(FunctionDocumentation
         {
-            R"(
+            .description=R"(
 Adds an negated interval to another interval or tuple of intervals. The returned value is tuple of intervals.
 [example:tuple]
 [example:interval1]
@@ -1604,12 +1604,12 @@ Adds an negated interval to another interval or tuple of intervals. The returned
 If the types of the first interval (or the interval in the tuple) and the second interval are the same they will be merged into one interval.
 [example:interval2]
 )",
-            Documentation::Examples{
-                {"tuple", "SELECT subtractInterval((INTERVAL 1 DAY, INTERVAL 1 YEAR), INTERVAL 1 MONTH)"},
-                {"interval1", "SELECT subtractInterval(INTERVAL 1 DAY, INTERVAL 1 MONTH)"},
-                {"interval2", "SELECT subtractInterval(INTERVAL 2 DAY, INTERVAL 1 DAY)"},
+            .examples{
+                {"tuple", "SELECT subtractInterval((INTERVAL 1 DAY, INTERVAL 1 YEAR), INTERVAL 1 MONTH)", ""},
+                {"interval1", "SELECT subtractInterval(INTERVAL 1 DAY, INTERVAL 1 MONTH)", ""},
+                {"interval2", "SELECT subtractInterval(INTERVAL 2 DAY, INTERVAL 1 DAY)", ""},
                 },
-            Documentation::Categories{"Tuple", "Interval"}
+            .categories{"Tuple", "Interval"}
         });
 
     factory.registerFunction<FunctionTupleMultiplyByNumber>();
diff --git a/src/Functions/widthBucket.cpp b/src/Functions/widthBucket.cpp
index e14ef90223d..e95f7c05756 100644
--- a/src/Functions/widthBucket.cpp
+++ b/src/Functions/widthBucket.cpp
@@ -252,8 +252,8 @@ public:
 
 REGISTER_FUNCTION(WidthBucket)
 {
-    factory.registerFunction<FunctionWidthBucket>({
-        R"(
+    factory.registerFunction<FunctionWidthBucket>(FunctionDocumentation{
+        .description=R"(
 Returns the number of the bucket in which `operand` falls in a histogram having `count` equal-width buckets spanning the range `low` to `high`. Returns `0` if `operand < low`, and returns `count+1` if `operand >= high`.
 
 `operand`, `low`, `high` can be any native number type. `count` can only be unsigned native integer and its value cannot be zero.
@@ -279,10 +279,10 @@ Result:
 └──────────────────────────────────┘
 ```
 )",
-        Documentation::Examples{
-            {"simple", "SELECT widthBucket(10.15, -8.6, 23, 18)"},
+        .examples{
+            {"simple", "SELECT widthBucket(10.15, -8.6, 23, 18)", ""},
         },
-        Documentation::Categories{"Mathematical"},
+        .categories{"Mathematical"},
     });
 
     factory.registerAlias("width_bucket", "widthBucket", FunctionFactory::CaseInsensitive);
diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp
index f3a297a11d1..0c609d84da3 100644
--- a/src/Storages/System/StorageSystemFunctions.cpp
+++ b/src/Storages/System/StorageSystemFunctions.cpp
@@ -68,9 +68,9 @@ namespace
             {
                 auto documentation = factory.getDocumentation(name);
                 res_columns[6]->insert(documentation.description);
-                res_columns[7]->insertDefault();
-                res_columns[8]->insertDefault();
-                res_columns[9]->insertDefault();
+                res_columns[7]->insert(documentation.syntax);
+                res_columns[8]->insert(documentation.argumentsAsString());
+                res_columns[9]->insert(documentation.returned_value);
                 res_columns[10]->insert(documentation.examplesAsString());
                 res_columns[11]->insert(documentation.categoriesAsString());
             }
diff --git a/src/TableFunctions/ITableFunction.h b/src/TableFunctions/ITableFunction.h
index f59092acf69..fe71005cb9c 100644
--- a/src/TableFunctions/ITableFunction.h
+++ b/src/TableFunctions/ITableFunction.h
@@ -4,7 +4,7 @@
 #include <Storages/IStorage_fwd.h>
 #include <Storages/ColumnsDescription.h>
 #include <Access/Common/AccessType.h>
-#include <Common/Documentation.h>
+#include <Common/FunctionDocumentation.h>
 #include <Analyzer/IQueryTreeNode.h>
 
 #include <memory>
@@ -97,7 +97,7 @@ private:
 /// Properties of table function that are independent of argument types and parameters.
 struct TableFunctionProperties
 {
-    Documentation documentation;
+    FunctionDocumentation documentation;
 
     /** It is determined by the possibility of modifying any data or making requests to arbitrary hostnames.
       *
diff --git a/src/TableFunctions/TableFunctionDeltaLake.cpp b/src/TableFunctions/TableFunctionDeltaLake.cpp
index 27ae9e6df2d..a77acce775c 100644
--- a/src/TableFunctions/TableFunctionDeltaLake.cpp
+++ b/src/TableFunctions/TableFunctionDeltaLake.cpp
@@ -20,12 +20,7 @@ using TableFunctionDeltaLake = ITableFunctionDataLake<TableFunctionDeltaLakeName
 
 void registerTableFunctionDeltaLake(TableFunctionFactory & factory)
 {
-    factory.registerFunction<TableFunctionDeltaLake>(
-        {.documentation
-         = {R"(The table function can be used to read the DeltaLake table stored on object store.)",
-            Documentation::Examples{{"deltaLake", "SELECT * FROM deltaLake(url, access_key_id, secret_access_key)"}},
-            Documentation::Categories{"DataLake"}},
-         .allow_readonly = false});
+    factory.registerFunction<TableFunctionDeltaLake>({.allow_readonly = false});
 }
 
 }
diff --git a/src/TableFunctions/TableFunctionExplain.cpp b/src/TableFunctions/TableFunctionExplain.cpp
index f93ae00cd7d..335ddb6f0ff 100644
--- a/src/TableFunctions/TableFunctionExplain.cpp
+++ b/src/TableFunctions/TableFunctionExplain.cpp
@@ -147,20 +147,7 @@ InterpreterExplainQuery TableFunctionExplain::getInterpreter(ContextPtr context)
 
 void registerTableFunctionExplain(TableFunctionFactory & factory)
 {
-    factory.registerFunction<TableFunctionExplain>({.documentation = {R"(
-Returns result of EXPLAIN query.
-
-The function should not be called directly but can be invoked via `SELECT * FROM (EXPLAIN <query>)`.
-
-You can use this query to process the result of EXPLAIN further using SQL (e.g., in tests).
-
-Example:
-[example:1]
-
-)",
-{{"1", "SELECT explain FROM (EXPLAIN AST SELECT * FROM system.numbers) WHERE explain LIKE '%Asterisk%'"}}
-}});
-
+    factory.registerFunction<TableFunctionExplain>();
 }
 
 }
diff --git a/src/TableFunctions/TableFunctionFormat.cpp b/src/TableFunctions/TableFunctionFormat.cpp
index 1e37775f574..f5aff4bd098 100644
--- a/src/TableFunctions/TableFunctionFormat.cpp
+++ b/src/TableFunctions/TableFunctionFormat.cpp
@@ -96,9 +96,9 @@ StoragePtr TableFunctionFormat::executeImpl(const ASTPtr & /*ast_function*/, Con
     return res;
 }
 
-static const Documentation format_table_function_documentation =
+static const FunctionDocumentation format_table_function_documentation =
 {
-    R"(
+    .description=R"(
 Extracts table structure from data and parses it according to specified input format.
 Syntax: `format(format_name, data)`.
 Parameters:
@@ -106,7 +106,7 @@ Parameters:
     - `data ` - String literal or constant expression that returns a string containing data in specified format.
 Returned value: A table with data parsed from `data` argument according specified format and extracted schema.
 )",
-    Documentation::Examples
+    .examples
     {
         {
             "First example",
@@ -131,7 +131,7 @@ Result:
 │ 124 │ World │
 └─────┴───────┘
 ```
-)"
+)", ""
         },
         {
             "Second example",
@@ -154,10 +154,10 @@ Result:
 │ a    │ Nullable(String)  │              │                    │         │                  │                │
 └──────┴───────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
 ```
-)"
+)", ""
         },
     },
-    Documentation::Categories{"format", "table-functions"}
+    .categories{"format", "table-functions"}
 };
 
 void registerTableFunctionFormat(TableFunctionFactory & factory)
diff --git a/src/TableFunctions/TableFunctionHudi.cpp b/src/TableFunctions/TableFunctionHudi.cpp
index a49fa32458d..436e708b72d 100644
--- a/src/TableFunctions/TableFunctionHudi.cpp
+++ b/src/TableFunctions/TableFunctionHudi.cpp
@@ -21,9 +21,9 @@ void registerTableFunctionHudi(TableFunctionFactory & factory)
 {
     factory.registerFunction<TableFunctionHudi>(
         {.documentation
-         = {R"(The table function can be used to read the Hudi table stored on object store.)",
-            Documentation::Examples{{"hudi", "SELECT * FROM hudi(url, access_key_id, secret_access_key)"}},
-            Documentation::Categories{"DataLake"}},
+         = {.description=R"(The table function can be used to read the Hudi table stored on object store.)",
+            .examples{{"hudi", "SELECT * FROM hudi(url, access_key_id, secret_access_key)", ""}},
+            .categories{"DataLake"}},
          .allow_readonly = false});
 }
 }
diff --git a/src/TableFunctions/TableFunctionIceberg.cpp b/src/TableFunctions/TableFunctionIceberg.cpp
index 423dbe882c0..5407d647f0f 100644
--- a/src/TableFunctions/TableFunctionIceberg.cpp
+++ b/src/TableFunctions/TableFunctionIceberg.cpp
@@ -23,9 +23,9 @@ void registerTableFunctionIceberg(TableFunctionFactory & factory)
 {
     factory.registerFunction<TableFunctionIceberg>(
         {.documentation
-         = {R"(The table function can be used to read the Iceberg table stored on object store.)",
-            Documentation::Examples{{"iceberg", "SELECT * FROM iceberg(url, access_key_id, secret_access_key)"}},
-            Documentation::Categories{"DataLake"}},
+         = {.description=R"(The table function can be used to read the Iceberg table stored on object store.)",
+            .examples{{"iceberg", "SELECT * FROM iceberg(url, access_key_id, secret_access_key)", ""}},
+            .categories{"DataLake"}},
          .allow_readonly = false});
 }
 
diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp
index 5d3b2d10822..01b44bc8380 100644
--- a/src/TableFunctions/TableFunctionS3.cpp
+++ b/src/TableFunctions/TableFunctionS3.cpp
@@ -270,9 +270,9 @@ void registerTableFunctionGCS(TableFunctionFactory & factory)
 {
     factory.registerFunction<TableFunctionGCS>(
         {.documentation
-         = {R"(The table function can be used to read the data stored on Google Cloud Storage.)",
-            Documentation::Examples{{"gcs", "SELECT * FROM gcs(url, hmac_key, hmac_secret)"}},
-            Documentation::Categories{"DataLake"}},
+         = {.description=R"(The table function can be used to read the data stored on Google Cloud Storage.)",
+            .examples{{"gcs", "SELECT * FROM gcs(url, hmac_key, hmac_secret)", ""}},
+            .categories{"DataLake"}},
          .allow_readonly = false});
 }
 
@@ -280,9 +280,9 @@ void registerTableFunctionS3(TableFunctionFactory & factory)
 {
     factory.registerFunction<TableFunctionS3>(
         {.documentation
-         = {R"(The table function can be used to read the data stored on AWS S3.)",
-            Documentation::Examples{{"s3", "SELECT * FROM s3(url, access_key_id, secret_access_key)"}},
-            Documentation::Categories{"DataLake"}},
+         = {.description=R"(The table function can be used to read the data stored on AWS S3.)",
+            .examples{{"s3", "SELECT * FROM s3(url, access_key_id, secret_access_key)", ""}},
+            .categories{"DataLake"}},
          .allow_readonly = false});
 }
 
diff --git a/src/TableFunctions/TableFunctionZeros.cpp b/src/TableFunctions/TableFunctionZeros.cpp
index d1c67659f56..c3dc819981b 100644
--- a/src/TableFunctions/TableFunctionZeros.cpp
+++ b/src/TableFunctions/TableFunctionZeros.cpp
@@ -48,37 +48,8 @@ StoragePtr TableFunctionZeros<multithreaded>::executeImpl(const ASTPtr & ast_fun
 
 void registerTableFunctionZeros(TableFunctionFactory & factory)
 {
-    factory.registerFunction<TableFunctionZeros<true>>({.documentation = {R"(
-Generates a stream of zeros (a table with one column 'zero' of type 'UInt8') of specified size.
-
-This table function is used in performance tests, where you want to spend as little time as possible to data generation while testing some other parts of queries.
-
-In contrast to the `zeros_mt`, this table function is using single thread for data generation.
-
-Example:
-[example:1]
-This query will test the speed of `randomPrintableASCII` function using single thread.
-
-See also the `system.zeros` table.
-)",
-{{"1", "SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomPrintableASCII(10))"}}
-}});
-
-    factory.registerFunction<TableFunctionZeros<false>>({.documentation = {R"(
-Generates a stream of zeros (a table with one column 'zero' of type 'UInt8') of specified size.
-
-This table function is used in performance tests, where you want to spend as little time as possible to data generation while testing some other parts of queries.
-
-In contrast to the `zeros`, this table function is using multiple threads for data generation, according to the `max_threads` setting.
-
-Example:
-[example:1]
-This query will test the speed of `randomPrintableASCII` function using multiple threads.
-
-See also the `system.zeros` table.
-)",
-{{"1", "SELECT count() FROM zeros_mt(1000000000) WHERE NOT ignore(randomPrintableASCII(10))"}}
-}});
+    factory.registerFunction<TableFunctionZeros<true>>();
+    factory.registerFunction<TableFunctionZeros<false>>();
 }
 
 template <bool multithreaded>

From e4d122323f98491feb404ce6ebcfb55b1a4f326b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 5 May 2023 23:43:22 +0200
Subject: [PATCH 208/689] Add a new test

---
 .../test_in_memory_parts_still_read.py        | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py

diff --git a/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py b/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
new file mode 100644
index 00000000000..d32a35bf95c
--- /dev/null
+++ b/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
@@ -0,0 +1,44 @@
+import pytest
+
+from helpers.cluster import ClickHouseCluster
+
+
+cluster = ClickHouseCluster(__file__)
+
+# Version 23.4 is the latest version to support writing in-memory parts.
+node = cluster.add_instance(
+    "node_old",
+    image="clickhouse/clickhouse-server",
+    tag="23.4",
+    stay_alive=True,
+    with_installed_binary=True,
+)
+
+
+@pytest.fixture(scope="module")
+def start_cluster():
+    try:
+        cluster.start()
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def test_in_memory_parts_still_read(start_cluster):
+    node.query(
+        "CREATE TABLE t (x UInt64, s String, a Array(Tuple(Map(String, LowCardinality(String)), Date32, DateTime64(3))) ENGINE = MergeTree ORDER BY s SETTINGS min_rows_for_compact_part = 1000000, min_bytes_for_compact_part = '1G', in_memory_parts_enable_wal = 1"
+    )
+    node.query("INSERT INTO t SELECT generateRandom() LIMIT 100")
+
+    assert node.query("SELECT count() FROM t WHERE NOT ignore(*)") == "100\n"
+
+    node.restart_with_latest_version()
+    assert node.query("SELECT count() FROM t WHERE NOT ignore(*)") == "100\n"
+
+    node.query("INSERT INTO t SELECT generateRandom() LIMIT 100")
+
+    assert node.query("SELECT count() FROM t WHERE NOT ignore(*)") == "200\n"
+
+    node.restart_with_original_version()
+    assert node.query("SELECT count() FROM t WHERE NOT ignore(*)") == "200\n"

From 510e63c0949a69f0c2528e47136cf1bf7f00f2e6 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 00:00:06 +0200
Subject: [PATCH 209/689] Fix bad test

---
 tests/queries/0_stateless/02723_zookeeper_name.sql | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/02723_zookeeper_name.sql b/tests/queries/0_stateless/02723_zookeeper_name.sql
index 7ddbf4edd47..e6374767951 100644
--- a/tests/queries/0_stateless/02723_zookeeper_name.sql
+++ b/tests/queries/0_stateless/02723_zookeeper_name.sql
@@ -11,12 +11,12 @@ INSERT INTO t1 SELECT * FROM generateRandom('k UInt32, v UInt32') LIMIT 1;
 INSERT INTO t2 SELECT * FROM generateRandom('k UInt32, v UInt32') LIMIT 1;
 
 SELECT
-    table,zookeeper_name,count()
+    table, zookeeper_name, count()
 FROM system.replicas
 INNER JOIN system.parts USING (database, table)
 WHERE database = currentDatabase()
-GROUP BY
-    table,zookeeper_name
+GROUP BY table, zookeeper_name
+ORDER BY table, zookeeper_name
 FORMAT CSV;
 
 DROP TABLE t1;

From be2834da5889f0c933636ce84d52a8fe26f14215 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 00:02:45 +0200
Subject: [PATCH 210/689] Remove obsolete test about deprecated feature

---
 .../__init__.py                               |   0
 .../configs/merge_tree.xml                    |   7 -
 .../configs/remote_servers.xml                |  26 -
 .../test_part_moves_between_shards/test.py    | 647 ------------------
 4 files changed, 680 deletions(-)
 delete mode 100644 tests/integration/test_part_moves_between_shards/__init__.py
 delete mode 100644 tests/integration/test_part_moves_between_shards/configs/merge_tree.xml
 delete mode 100644 tests/integration/test_part_moves_between_shards/configs/remote_servers.xml
 delete mode 100644 tests/integration/test_part_moves_between_shards/test.py

diff --git a/tests/integration/test_part_moves_between_shards/__init__.py b/tests/integration/test_part_moves_between_shards/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_part_moves_between_shards/configs/merge_tree.xml b/tests/integration/test_part_moves_between_shards/configs/merge_tree.xml
deleted file mode 100644
index cf54b8be04d..00000000000
--- a/tests/integration/test_part_moves_between_shards/configs/merge_tree.xml
+++ /dev/null
@@ -1,7 +0,0 @@
-<clickhouse>
-    <merge_tree>
-        <assign_part_uuids>1</assign_part_uuids>
-        <part_moves_between_shards_enable>1</part_moves_between_shards_enable>
-        <part_moves_between_shards_delay_seconds>3</part_moves_between_shards_delay_seconds>
-    </merge_tree>
-</clickhouse>
diff --git a/tests/integration/test_part_moves_between_shards/configs/remote_servers.xml b/tests/integration/test_part_moves_between_shards/configs/remote_servers.xml
deleted file mode 100644
index d0734608aa9..00000000000
--- a/tests/integration/test_part_moves_between_shards/configs/remote_servers.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-<clickhouse>
-    <remote_servers>
-        <test_cluster>
-            <shard>
-                <replica>
-                    <host>s0r0</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>s0r1</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-            <shard>
-                <replica>
-                    <host>s1r0</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>s1r1</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </test_cluster>
-    </remote_servers>
-</clickhouse>
diff --git a/tests/integration/test_part_moves_between_shards/test.py b/tests/integration/test_part_moves_between_shards/test.py
deleted file mode 100644
index 8fef44305ea..00000000000
--- a/tests/integration/test_part_moves_between_shards/test.py
+++ /dev/null
@@ -1,647 +0,0 @@
-import pytest
-import random
-import threading
-import time
-
-from helpers.client import QueryRuntimeException
-from helpers.cluster import ClickHouseCluster
-from helpers.test_tools import TSV
-
-transient_ch_errors = [23, 32, 210]
-
-cluster = ClickHouseCluster(__file__)
-
-s0r0 = cluster.add_instance(
-    "s0r0",
-    main_configs=["configs/remote_servers.xml", "configs/merge_tree.xml"],
-    stay_alive=True,
-    with_zookeeper=True,
-)
-
-s0r1 = cluster.add_instance(
-    "s0r1",
-    main_configs=["configs/remote_servers.xml", "configs/merge_tree.xml"],
-    stay_alive=True,
-    with_zookeeper=True,
-)
-
-s1r0 = cluster.add_instance(
-    "s1r0",
-    main_configs=["configs/remote_servers.xml", "configs/merge_tree.xml"],
-    stay_alive=True,
-    with_zookeeper=True,
-)
-
-s1r1 = cluster.add_instance(
-    "s1r1",
-    main_configs=["configs/remote_servers.xml", "configs/merge_tree.xml"],
-    stay_alive=True,
-    with_zookeeper=True,
-)
-
-
-@pytest.fixture(scope="module")
-def started_cluster():
-    try:
-        cluster.start()
-        yield cluster
-    finally:
-        cluster.shutdown()
-
-
-def test_move(started_cluster):
-    for shard_ix, rs in enumerate([[s0r0, s0r1], [s1r0, s1r1]]):
-        for replica_ix, r in enumerate(rs):
-            r.query(
-                """
-            DROP TABLE IF EXISTS test_move;
-            CREATE TABLE test_move(v UInt64)
-            ENGINE ReplicatedMergeTree('/clickhouse/shard_{}/tables/test_move', '{}')
-            ORDER BY tuple()
-            """.format(
-                    shard_ix, r.name
-                )
-            )
-
-    s0r0.query("SYSTEM STOP MERGES test_move")
-    s0r1.query("SYSTEM STOP MERGES test_move")
-
-    s0r0.query("INSERT INTO test_move VALUES (1)")
-    s0r0.query("INSERT INTO test_move VALUES (2)")
-
-    assert "2" == s0r0.query("SELECT count() FROM test_move").strip()
-    assert "0" == s1r0.query("SELECT count() FROM test_move").strip()
-
-    s0r0.query(
-        "ALTER TABLE test_move MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_move'"
-    )
-
-    print(s0r0.query("SELECT * FROM system.part_moves_between_shards"))
-
-    s0r0.query("SYSTEM START MERGES test_move")
-    s0r0.query("OPTIMIZE TABLE test_move FINAL")
-
-    wait_for_state("DONE", s0r0, "test_move")
-
-    for n in [s0r0, s0r1]:
-        assert "1" == n.query("SELECT count() FROM test_move").strip()
-
-    for n in [s1r0, s1r1]:
-        assert "1" == n.query("SELECT count() FROM test_move").strip()
-
-    # Move part back
-    s1r0.query(
-        "ALTER TABLE test_move MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_0/tables/test_move'"
-    )
-
-    wait_for_state("DONE", s1r0, "test_move")
-
-    for n in [s0r0, s0r1]:
-        assert "2" == n.query("SELECT count() FROM test_move").strip()
-
-    for n in [s1r0, s1r1]:
-        assert "0" == n.query("SELECT count() FROM test_move").strip()
-
-
-def test_deduplication_while_move(started_cluster):
-    for shard_ix, rs in enumerate([[s0r0, s0r1], [s1r0, s1r1]]):
-        for replica_ix, r in enumerate(rs):
-            r.query(
-                """
-            DROP TABLE IF EXISTS test_deduplication;
-            CREATE TABLE test_deduplication(v UInt64)
-            ENGINE ReplicatedMergeTree('/clickhouse/shard_{}/tables/test_deduplication', '{}')
-            ORDER BY tuple()
-            """.format(
-                    shard_ix, r.name
-                )
-            )
-
-            r.query(
-                """
-            DROP TABLE IF EXISTS test_deduplication_d;
-            CREATE TABLE test_deduplication_d AS test_deduplication
-            ENGINE Distributed('test_cluster', '', test_deduplication)
-            """
-            )
-
-    s0r0.query("SYSTEM STOP MERGES test_deduplication")
-    s0r1.query("SYSTEM STOP MERGES test_deduplication")
-
-    s0r0.query("INSERT INTO test_deduplication VALUES (1)")
-    s0r0.query("INSERT INTO test_deduplication VALUES (2)")
-    s0r1.query("SYSTEM SYNC REPLICA test_deduplication", timeout=20)
-
-    assert "2" == s0r0.query("SELECT count() FROM test_deduplication").strip()
-    assert "0" == s1r0.query("SELECT count() FROM test_deduplication").strip()
-
-    s0r0.query(
-        "ALTER TABLE test_deduplication MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_deduplication'"
-    )
-    s0r0.query("SYSTEM START MERGES test_deduplication")
-
-    expected = """
-1
-2
-"""
-
-    def deduplication_invariant_test():
-        n = random.choice(list(started_cluster.instances.values()))
-        assert TSV(
-            n.query(
-                "SELECT * FROM test_deduplication_d ORDER BY v",
-                settings={"allow_experimental_query_deduplication": 1},
-            )
-        ) == TSV(expected)
-
-        # https://github.com/ClickHouse/ClickHouse/issues/34089
-        assert TSV(
-            n.query(
-                "SELECT count() FROM test_deduplication_d",
-                settings={"allow_experimental_query_deduplication": 1},
-            )
-        ) == TSV("2")
-
-        assert TSV(
-            n.query(
-                "SELECT count() FROM test_deduplication_d",
-                settings={"allow_experimental_query_deduplication": 1},
-            )
-        ) == TSV("2")
-
-    deduplication_invariant = ConcurrentInvariant(deduplication_invariant_test)
-    deduplication_invariant.start()
-
-    wait_for_state("DONE", s0r0, "test_deduplication")
-
-    deduplication_invariant.stop_and_assert_no_exception()
-
-
-def test_part_move_step_by_step(started_cluster):
-    for shard_ix, rs in enumerate([[s0r0, s0r1], [s1r0, s1r1]]):
-        for replica_ix, r in enumerate(rs):
-            r.query(
-                """
-            DROP TABLE IF EXISTS test_part_move_step_by_step;
-            CREATE TABLE test_part_move_step_by_step(v UInt64)
-            ENGINE ReplicatedMergeTree('/clickhouse/shard_{}/tables/test_part_move_step_by_step', '{}')
-            ORDER BY tuple()
-            """.format(
-                    shard_ix, r.name
-                )
-            )
-
-            r.query(
-                """
-            DROP TABLE IF EXISTS test_part_move_step_by_step_d;
-            CREATE TABLE test_part_move_step_by_step_d AS test_part_move_step_by_step
-            ENGINE Distributed('test_cluster', currentDatabase(), test_part_move_step_by_step)
-            """
-            )
-
-    s0r0.query("SYSTEM STOP MERGES test_part_move_step_by_step")
-    s0r1.query("SYSTEM STOP MERGES test_part_move_step_by_step")
-
-    s0r0.query("INSERT INTO test_part_move_step_by_step VALUES (1)")
-    s0r0.query("INSERT INTO test_part_move_step_by_step VALUES (2)")
-    s0r1.query("SYSTEM SYNC REPLICA test_part_move_step_by_step", timeout=20)
-
-    assert "2" == s0r0.query("SELECT count() FROM test_part_move_step_by_step").strip()
-    assert "0" == s1r0.query("SELECT count() FROM test_part_move_step_by_step").strip()
-
-    expected = """
-1
-2
-"""
-
-    def deduplication_invariant_test():
-        n = random.choice(list(started_cluster.instances.values()))
-        try:
-            assert TSV(
-                n.query(
-                    "SELECT * FROM test_part_move_step_by_step_d ORDER BY v",
-                    settings={"allow_experimental_query_deduplication": 1},
-                )
-            ) == TSV(expected)
-        except QueryRuntimeException as e:
-            # ignore transient errors that are caused by us restarting nodes
-            if e.returncode not in transient_ch_errors:
-                raise e
-
-    deduplication_invariant = ConcurrentInvariant(deduplication_invariant_test)
-    deduplication_invariant.start()
-
-    # Stop a source replica to prevent SYNC_SOURCE succeeding.
-    s0r1.stop_clickhouse()
-
-    s0r0.query(
-        "ALTER TABLE test_part_move_step_by_step MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_part_move_step_by_step'"
-    )
-
-    # Should hang on SYNC_SOURCE until all source replicas acknowledge new pinned UUIDs.
-    wait_for_state(
-        "SYNC_SOURCE",
-        s0r0,
-        "test_part_move_step_by_step",
-        "Some replicas haven\\'t processed event",
-    )
-    deduplication_invariant.assert_no_exception()
-
-    # Start all replicas in source shard but stop a replica in destination shard
-    # to prevent SYNC_DESTINATION succeeding.
-    s1r1.stop_clickhouse()
-    s0r1.start_clickhouse()
-
-    # After SYNC_SOURCE step no merges will be assigned.
-    s0r0.query(
-        "SYSTEM START MERGES test_part_move_step_by_step; OPTIMIZE TABLE test_part_move_step_by_step;"
-    )
-    s0r1.query(
-        "SYSTEM START MERGES test_part_move_step_by_step; OPTIMIZE TABLE test_part_move_step_by_step;"
-    )
-
-    wait_for_state(
-        "SYNC_DESTINATION",
-        s0r0,
-        "test_part_move_step_by_step",
-        "Some replicas haven\\'t processed event",
-    )
-    deduplication_invariant.assert_no_exception()
-
-    # Start previously stopped replica in destination shard to let SYNC_DESTINATION
-    # succeed.
-    # Stop the other replica in destination shard to prevent DESTINATION_FETCH succeed.
-    s1r0.stop_clickhouse()
-    s1r1.start_clickhouse()
-    wait_for_state(
-        "DESTINATION_FETCH",
-        s0r0,
-        "test_part_move_step_by_step",
-        "Some replicas haven\\'t processed event",
-    )
-    deduplication_invariant.assert_no_exception()
-
-    # Start previously stopped replica in destination shard to let DESTINATION_FETCH
-    # succeed.
-    # Stop the other replica in destination shard to prevent DESTINATION_ATTACH succeed.
-    s1r1.stop_clickhouse()
-    s1r0.start_clickhouse()
-    wait_for_state(
-        "DESTINATION_ATTACH",
-        s0r0,
-        "test_part_move_step_by_step",
-        "Some replicas haven\\'t processed event",
-    )
-    deduplication_invariant.assert_no_exception()
-
-    # Start all replicas in destination shard to let DESTINATION_ATTACH succeed.
-    # Stop a source replica to prevent SOURCE_DROP succeeding.
-    s0r0.stop_clickhouse()
-    s1r1.start_clickhouse()
-    wait_for_state(
-        "SOURCE_DROP",
-        s0r1,
-        "test_part_move_step_by_step",
-        "Some replicas haven\\'t processed event",
-    )
-    deduplication_invariant.assert_no_exception()
-
-    s0r0.start_clickhouse()
-    wait_for_state("DONE", s0r1, "test_part_move_step_by_step")
-    deduplication_invariant.assert_no_exception()
-
-    # No hung tasks in replication queue. Would timeout otherwise.
-    for instance in started_cluster.instances.values():
-        instance.query("SYSTEM SYNC REPLICA test_part_move_step_by_step")
-
-    assert "1" == s0r0.query("SELECT count() FROM test_part_move_step_by_step").strip()
-    assert "1" == s1r0.query("SELECT count() FROM test_part_move_step_by_step").strip()
-
-    deduplication_invariant.stop_and_assert_no_exception()
-
-
-def test_part_move_step_by_step_kill(started_cluster):
-    for shard_ix, rs in enumerate([[s0r0, s0r1], [s1r0, s1r1]]):
-        for replica_ix, r in enumerate(rs):
-            r.query(
-                """
-            DROP TABLE IF EXISTS test_part_move_step_by_step_kill;
-            CREATE TABLE test_part_move_step_by_step_kill(v UInt64)
-            ENGINE ReplicatedMergeTree('/clickhouse/shard_{}/tables/test_part_move_step_by_step_kill', '{}')
-            ORDER BY tuple()
-            """.format(
-                    shard_ix, r.name
-                )
-            )
-
-            r.query(
-                """
-            DROP TABLE IF EXISTS test_part_move_step_by_step_kill_d;
-            CREATE TABLE test_part_move_step_by_step_kill_d AS test_part_move_step_by_step_kill
-            ENGINE Distributed('test_cluster', currentDatabase(), test_part_move_step_by_step_kill)
-            """
-            )
-
-    s0r0.query("SYSTEM STOP MERGES test_part_move_step_by_step_kill")
-    s0r1.query("SYSTEM STOP MERGES test_part_move_step_by_step_kill")
-
-    s0r0.query("INSERT INTO test_part_move_step_by_step_kill VALUES (1)")
-    s0r0.query("INSERT INTO test_part_move_step_by_step_kill VALUES (2)")
-    s0r1.query("SYSTEM SYNC REPLICA test_part_move_step_by_step_kill", timeout=20)
-
-    assert (
-        "2"
-        == s0r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip()
-    )
-    assert (
-        "0"
-        == s1r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip()
-    )
-
-    expected = """
-1
-2
-"""
-
-    def deduplication_invariant_test():
-        n = random.choice(list(started_cluster.instances.values()))
-        try:
-            assert TSV(
-                n.query(
-                    "SELECT * FROM test_part_move_step_by_step_kill_d ORDER BY v",
-                    settings={"allow_experimental_query_deduplication": 1},
-                )
-            ) == TSV(expected)
-        except QueryRuntimeException as e:
-            # ignore transient errors that are caused by us restarting nodes
-            if e.returncode not in transient_ch_errors:
-                raise e
-
-    deduplication_invariant = ConcurrentInvariant(deduplication_invariant_test)
-    deduplication_invariant.start()
-
-    # Stop a source replica to prevent SYNC_SOURCE succeeding.
-    s0r1.stop_clickhouse()
-
-    s0r0.query(
-        "ALTER TABLE test_part_move_step_by_step_kill MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_part_move_step_by_step_kill'"
-    )
-
-    # Should hang on SYNC_SOURCE until all source replicas acknowledge new pinned UUIDs.
-    wait_for_state(
-        "SYNC_SOURCE",
-        s0r0,
-        "test_part_move_step_by_step_kill",
-        "Some replicas haven\\'t processed event",
-    )
-    deduplication_invariant.assert_no_exception()
-
-    # Start all replicas in source shard but stop a replica in destination shard
-    # to prevent SYNC_DESTINATION succeeding.
-    s1r1.stop_clickhouse()
-    s0r1.start_clickhouse()
-
-    # After SYNC_SOURCE step no merges will be assigned.
-    s0r0.query(
-        "SYSTEM START MERGES test_part_move_step_by_step_kill; OPTIMIZE TABLE test_part_move_step_by_step_kill;"
-    )
-    s0r1.query(
-        "SYSTEM START MERGES test_part_move_step_by_step_kill; OPTIMIZE TABLE test_part_move_step_by_step_kill;"
-    )
-
-    wait_for_state(
-        "SYNC_DESTINATION",
-        s0r0,
-        "test_part_move_step_by_step_kill",
-        "Some replicas haven\\'t processed event",
-    )
-    deduplication_invariant.assert_no_exception()
-
-    # Start previously stopped replica in destination shard to let SYNC_DESTINATION
-    # succeed.
-    # Stop the other replica in destination shard to prevent DESTINATION_FETCH succeed.
-    s1r0.stop_clickhouse()
-    s1r1.start_clickhouse()
-    wait_for_state(
-        "DESTINATION_FETCH",
-        s0r0,
-        "test_part_move_step_by_step_kill",
-        "Some replicas haven\\'t processed event",
-    )
-
-    # Start previously stopped replica in destination shard to let DESTINATION_FETCH
-    # succeed.
-    # Stop the other replica in destination shard to prevent DESTINATION_ATTACH succeed.
-    s1r1.stop_clickhouse()
-    s1r0.start_clickhouse()
-    wait_for_state(
-        "DESTINATION_ATTACH",
-        s0r0,
-        "test_part_move_step_by_step_kill",
-        "Some replicas haven\\'t processed event",
-    )
-    deduplication_invariant.assert_no_exception()
-
-    # Rollback here.
-    s0r0.query(
-        """
-        KILL PART_MOVE_TO_SHARD
-        WHERE task_uuid = (SELECT task_uuid FROM system.part_moves_between_shards WHERE table = 'test_part_move_step_by_step_kill')
-    """
-    )
-
-    wait_for_state(
-        "DESTINATION_ATTACH",
-        s0r0,
-        "test_part_move_step_by_step_kill",
-        assert_exception_msg="Some replicas haven\\'t processed event",
-        assert_rollback=True,
-    )
-
-    s1r1.start_clickhouse()
-
-    wait_for_state(
-        "CANCELLED", s0r0, "test_part_move_step_by_step_kill", assert_rollback=True
-    )
-    deduplication_invariant.assert_no_exception()
-
-    # No hung tasks in replication queue. Would timeout otherwise.
-    for instance in started_cluster.instances.values():
-        instance.query("SYSTEM SYNC REPLICA test_part_move_step_by_step_kill")
-
-    assert (
-        "2"
-        == s0r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip()
-    )
-    assert (
-        "0"
-        == s1r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip()
-    )
-
-    deduplication_invariant.stop_and_assert_no_exception()
-
-
-def test_move_not_permitted(started_cluster):
-    # Verify that invariants for part compatibility are checked.
-
-    # Tests are executed in order. Make sure cluster is up if previous test
-    # failed.
-    s0r0.start_clickhouse()
-    s1r0.start_clickhouse()
-
-    for ix, n in enumerate([s0r0, s1r0]):
-        n.query(
-            """
-        DROP TABLE IF EXISTS not_permitted_columns;
-        
-        CREATE TABLE not_permitted_columns(v_{ix} UInt64)
-        ENGINE ReplicatedMergeTree('/clickhouse/shard_{ix}/tables/not_permitted_columns', 'r')
-        ORDER BY tuple();
-        """.format(
-                ix=ix
-            )
-        )
-
-        partition = "date"
-        if ix > 0:
-            partition = "v"
-
-        n.query(
-            """
-        DROP TABLE IF EXISTS not_permitted_partition;
-        CREATE TABLE not_permitted_partition(date Date, v UInt64)
-        ENGINE ReplicatedMergeTree('/clickhouse/shard_{ix}/tables/not_permitted_partition', 'r')
-        PARTITION BY ({partition})
-        ORDER BY tuple();
-        """.format(
-                ix=ix, partition=partition
-            )
-        )
-
-    s0r0.query("INSERT INTO not_permitted_columns VALUES (1)")
-    s0r0.query("INSERT INTO not_permitted_partition VALUES ('2021-09-03', 1)")
-
-    with pytest.raises(
-        QueryRuntimeException,
-        match="DB::Exception: Source and destination are the same",
-    ):
-        s0r0.query(
-            "ALTER TABLE not_permitted_columns MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_0/tables/not_permitted_columns'"
-        )
-
-    with pytest.raises(
-        QueryRuntimeException,
-        match="DB::Exception: Table columns structure in ZooKeeper is different from local table structure.",
-    ):
-        s0r0.query(
-            "ALTER TABLE not_permitted_columns MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/not_permitted_columns'"
-        )
-
-    with pytest.raises(
-        QueryRuntimeException,
-        match="DB::Exception: Existing table metadata in ZooKeeper differs in partition key expression.",
-    ):
-        s0r0.query(
-            "ALTER TABLE not_permitted_partition MOVE PART '20210903_0_0_0' TO SHARD '/clickhouse/shard_1/tables/not_permitted_partition'"
-        )
-
-
-def wait_for_state(
-    desired_state,
-    instance,
-    test_table,
-    assert_exception_msg=None,
-    assert_rollback=False,
-):
-    last_debug_print_time = time.time()
-
-    print("Waiting to reach state: {}".format(desired_state))
-    if assert_exception_msg:
-        print("  with exception contents: {}".format(assert_exception_msg))
-    if assert_rollback:
-        print("     and rollback: {}".format(assert_rollback))
-
-    while True:
-        tasks = TSV.toMat(
-            instance.query(
-                "SELECT state, num_tries, last_exception, rollback FROM system.part_moves_between_shards WHERE table = '{}'".format(
-                    test_table
-                )
-            )
-        )
-        assert len(tasks) == 1, "only one task expected in this test"
-
-        if time.time() - last_debug_print_time > 30:
-            last_debug_print_time = time.time()
-            print("Current state: ", tasks)
-
-        [state, num_tries, last_exception, rollback] = tasks[0]
-
-        if state == desired_state:
-            if assert_exception_msg and int(num_tries) < 3:
-                # Let the task be retried a few times when expecting an exception
-                # to make sure the exception is persistent and the code doesn't
-                # accidentally continue to run when we expect it not to.
-                continue
-
-            if assert_exception_msg:
-                assert assert_exception_msg in last_exception
-
-            if assert_rollback:
-                assert int(rollback) == 1, "rollback bit isn't set"
-
-            break
-        elif state in ["DONE", "CANCELLED"]:
-            raise Exception(
-                "Reached terminal state {}, but was waiting for {}".format(
-                    state, desired_state
-                )
-            )
-
-        time.sleep(0.1)
-
-
-class ConcurrentInvariant:
-    def __init__(self, invariant_test, loop_sleep=0.1):
-        self.invariant_test = invariant_test
-        self.loop_sleep = loop_sleep
-
-        self.started = False
-        self.exiting = False
-        self.exception = None
-        self.thread = threading.Thread(target=self._loop)
-
-    def start(self):
-        if self.started:
-            raise Exception("invariant thread already started")
-
-        self.started = True
-        self.thread.start()
-
-    def stop_and_assert_no_exception(self):
-        self._assert_started()
-
-        self.exiting = True
-        self.thread.join()
-
-        if self.exception:
-            raise self.exception
-
-    def assert_no_exception(self):
-        self._assert_started()
-
-        if self.exception:
-            raise self.exception
-
-    def _loop(self):
-        try:
-            while not self.exiting:
-                self.invariant_test()
-                time.sleep(self.loop_sleep)
-        except Exception as e:
-            self.exiting = True
-            self.exception = e
-
-    def _assert_started(self):
-        if not self.started:
-            raise Exception("invariant thread not started, forgot to call start?")

From ddb99279f786d52d47095818ce95181ba4622b63 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Fri, 5 May 2023 04:29:05 +0000
Subject: [PATCH 211/689] Test for input_format_parquet_preserve_order

---
 .../02725_parquet_preserve_order.reference      |  12 ++++++++++++
 .../0_stateless/02725_parquet_preserve_order.sh |  16 ++++++++++++++++
 .../0_stateless/data_parquet/02725_data.parquet | Bin 0 -> 576 bytes
 3 files changed, 28 insertions(+)
 create mode 100644 tests/queries/0_stateless/02725_parquet_preserve_order.reference
 create mode 100755 tests/queries/0_stateless/02725_parquet_preserve_order.sh
 create mode 100644 tests/queries/0_stateless/data_parquet/02725_data.parquet

diff --git a/tests/queries/0_stateless/02725_parquet_preserve_order.reference b/tests/queries/0_stateless/02725_parquet_preserve_order.reference
new file mode 100644
index 00000000000..e9c8f99bb33
--- /dev/null
+++ b/tests/queries/0_stateless/02725_parquet_preserve_order.reference
@@ -0,0 +1,12 @@
+0
+1
+2
+(Expression)
+ExpressionTransform
+  (ReadFromStorage)
+  File 0 → 1
+(Expression)
+ExpressionTransform × 2
+  (ReadFromStorage)
+  Resize 1 → 2
+    File 0 → 1
diff --git a/tests/queries/0_stateless/02725_parquet_preserve_order.sh b/tests/queries/0_stateless/02725_parquet_preserve_order.sh
new file mode 100755
index 00000000000..ea3e4219e35
--- /dev/null
+++ b/tests/queries/0_stateless/02725_parquet_preserve_order.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+
+# This file has a row group with 2 rows, then a row group with 1 row.
+# It'll be read into two blocks. The first block will sleep 2x longer than the second.
+# So reordering is very likely if the order-preservation doesn't work.
+
+$CLICKHOUSE_LOCAL -q "select number+sleepEachRow(3) from file('$CURDIR/data_parquet/02725_data.parquet') settings input_format_parquet_preserve_order=1"
+
+$CLICKHOUSE_LOCAL -q "explain pipeline select number+sleepEachRow(3) from file('$CURDIR/data_parquet/02725_data.parquet') settings input_format_parquet_preserve_order=1, max_threads=2"
+$CLICKHOUSE_LOCAL -q "explain pipeline select number+sleepEachRow(3) from file('$CURDIR/data_parquet/02725_data.parquet') settings input_format_parquet_preserve_order=0, parallelize_output_from_storages=1, max_threads=2"
diff --git a/tests/queries/0_stateless/data_parquet/02725_data.parquet b/tests/queries/0_stateless/data_parquet/02725_data.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..5b4842c9dbdf52bac11c9f9867508ea582191608
GIT binary patch
literal 576
zcmWG=3^EjD5mgYC@&Qr|LJSPT42%p7j0_MU${@-n%Ao_45&$yTWXu>eIAC%T98g6d
z%7BrXNo@wBj3|qwsQ?3;qy$@DX>L+#ktm-Si`W!Ku?dW7c4{2B^h<(lU=U?u01^T~
zih%)aqmYj%6WC1+AR!QdxrYhKJuo>54j2ol(14MNL2U;!Huo@zwJ?fxFsj{SQro};
zb`eY;k_p&cB&lP{Ai-9goRONF2y!cnDv%5DrmBq38es=u=rM?~N$N<-P%|h%!N;P;
z0SrPB1_q$fRP_}oEI{D^3=k0}1`VZx#G=B|)Dqp~f&zuI)S}|d{5%CiJyShH-C#e*
VfM5^*5J?#spf~w|sUQHDb^xl%LbCt>

literal 0
HcmV?d00001


From 846f2d57af344e52708b326cdadc20fd6e362273 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 00:55:56 +0200
Subject: [PATCH 212/689] Fix error found by AST Fuzzer

---
 .../test_server_initialization/test.py        |  14 --
 .../00960_live_view_watch_events_live.py      |  52 -------
 ...0960_live_view_watch_events_live.reference |   0
 .../00962_live_view_periodic_refresh.py       |  46 ------
 ...00962_live_view_periodic_refresh.reference |   0
 ..._live_view_periodic_refresh_and_timeout.py |  45 ------
 ...iew_periodic_refresh_and_timeout.reference |   0
 ...2_live_view_periodic_refresh_dictionary.py |  73 ---------
 ...view_periodic_refresh_dictionary.reference |   0
 .../00964_live_view_watch_events_heartbeat.py |  56 -------
 ...live_view_watch_events_heartbeat.reference |   0
 .../00965_live_view_watch_heartbeat.py        |  54 -------
 .../00965_live_view_watch_heartbeat.reference |   0
 .../00966_live_view_watch_events_http.py      |  49 ------
 ...0966_live_view_watch_events_http.reference |   0
 .../0_stateless/00967_live_view_watch_http.py |  49 ------
 .../00967_live_view_watch_http.reference      |   0
 ...t_format_jsoneachrowwithprogress.reference |   4 -
 ..._select_format_jsoneachrowwithprogress.sql |  16 --
 ...h_format_jsoneachrowwithprogress.reference |   6 -
 ...w_watch_format_jsoneachrowwithprogress.sql |  22 ---
 ...0_live_view_watch_events_http_heartbeat.py |  63 --------
 ...view_watch_events_http_heartbeat.reference |   0
 .../00971_live_view_watch_http_heartbeat.py   |  57 -------
 ...1_live_view_watch_http_heartbeat.reference |   0
 .../00972_live_view_select_1.reference        |   1 -
 .../0_stateless/00972_live_view_select_1.sql  |  11 --
 .../00973_live_view_select.reference          |   4 -
 .../0_stateless/00973_live_view_select.sql    |  22 ---
 .../00973_live_view_select_prewhere.reference |   2 -
 .../00973_live_view_select_prewhere.sql       |  25 ---
 ...3_live_view_with_subquery_select.reference |  18 ---
 .../00973_live_view_with_subquery_select.sql  |  22 ---
 ...e_view_with_subquery_select_join.reference |   6 -
 ...73_live_view_with_subquery_select_join.sql |  30 ----
 ...th_subquery_select_join_no_alias.reference |   6 -
 ...iew_with_subquery_select_join_no_alias.sql |  30 ----
 ...view_with_subquery_select_nested.reference |  18 ---
 ..._live_view_with_subquery_select_nested.sql |  22 ---
 ...y_select_nested_with_aggregation.reference |   4 -
 ...ubquery_select_nested_with_aggregation.sql |  22 ---
 ...ted_with_aggregation_table_alias.reference |   4 -
 ...ct_nested_with_aggregation_table_alias.sql |  22 ---
 ...with_subquery_select_table_alias.reference |  18 ---
 ..._view_with_subquery_select_table_alias.sql |  22 ---
 ...subquery_select_with_aggregation.reference |   4 -
 ..._with_subquery_select_with_aggregation.sql |  22 ---
 ...ect_with_aggregation_in_subquery.reference |   4 -
 ...ry_select_with_aggregation_in_subquery.sql |  22 ---
 ...ive_view_select_with_aggregation.reference |   2 -
 ...0974_live_view_select_with_aggregation.sql |  20 ---
 .../00975_live_view_create.reference          |   0
 .../0_stateless/00975_live_view_create.sql    |  11 --
 .../00976_live_view_select_version.reference  |   3 -
 .../00976_live_view_select_version.sql        |  16 --
 .../00977_live_view_watch_events.reference    |   3 -
 .../00977_live_view_watch_events.sql          |  22 ---
 .../00978_live_view_watch.reference           |   3 -
 .../0_stateless/00978_live_view_watch.sql     |  22 ---
 ...9_live_view_watch_continuous_aggregates.py |  84 ----------
 ...view_watch_continuous_aggregates.reference |   0
 .../0_stateless/00979_live_view_watch_live.py |  58 -------
 .../00979_live_view_watch_live.reference      |   0
 ...0979_live_view_watch_live_with_subquery.py |  60 --------
 ...ve_view_watch_live_with_subquery.reference |   0
 .../00991_live_view_watch_event_live.python   |  88 -----------
 ...00991_live_view_watch_event_live.reference |   7 -
 .../00991_live_view_watch_http.python         |  70 ---------
 .../00991_live_view_watch_http.reference      |   4 -
 .../01056_create_table_as.reference           |   1 -
 .../0_stateless/01056_create_table_as.sql     |  57 -------
 ...1071_live_view_detach_dependency.reference |   0
 .../01071_live_view_detach_dependency.sql     |  12 --
 .../01190_full_attach_syntax.reference        |  13 --
 .../0_stateless/01190_full_attach_syntax.sql  |  60 --------
 ...01235_live_view_over_distributed.reference |   4 -
 .../01235_live_view_over_distributed.sql      |  23 ---
 ..._over_live_view_over_distributed.reference |   8 -
 ...ibuted_over_live_view_over_distributed.sql |  25 ---
 ...with_subquery_select_table_alias.reference |   4 -
 ...buted_with_subquery_select_table_alias.sql |  23 ---
 .../01246_insert_into_watch_live_view.py      |  87 -----------
 ...1246_insert_into_watch_live_view.reference |   0
 ...463_test_alter_live_view_refresh.reference |   1 -
 .../01463_test_alter_live_view_refresh.sql    |  15 --
 .../01602_show_create_view.reference          |   7 -
 .../0_stateless/01602_show_create_view.sql    |  53 -------
 ...query_views_log_current_database.reference |  94 ------------
 ...01927_query_views_log_current_database.sql | 145 ------------------
 .../02048_views_with_comment.reference        |   3 -
 .../0_stateless/02048_views_with_comment.sql  |  12 --
 91 files changed, 2087 deletions(-)
 delete mode 100755 tests/queries/0_stateless/00960_live_view_watch_events_live.py
 delete mode 100644 tests/queries/0_stateless/00960_live_view_watch_events_live.reference
 delete mode 100755 tests/queries/0_stateless/00962_live_view_periodic_refresh.py
 delete mode 100644 tests/queries/0_stateless/00962_live_view_periodic_refresh.reference
 delete mode 100755 tests/queries/0_stateless/00962_live_view_periodic_refresh_and_timeout.py
 delete mode 100644 tests/queries/0_stateless/00962_live_view_periodic_refresh_and_timeout.reference
 delete mode 100755 tests/queries/0_stateless/00962_live_view_periodic_refresh_dictionary.py
 delete mode 100644 tests/queries/0_stateless/00962_live_view_periodic_refresh_dictionary.reference
 delete mode 100755 tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py
 delete mode 100644 tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference
 delete mode 100755 tests/queries/0_stateless/00965_live_view_watch_heartbeat.py
 delete mode 100644 tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference
 delete mode 100755 tests/queries/0_stateless/00966_live_view_watch_events_http.py
 delete mode 100644 tests/queries/0_stateless/00966_live_view_watch_events_http.reference
 delete mode 100755 tests/queries/0_stateless/00967_live_view_watch_http.py
 delete mode 100644 tests/queries/0_stateless/00967_live_view_watch_http.reference
 delete mode 100644 tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference
 delete mode 100644 tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql
 delete mode 100644 tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference
 delete mode 100644 tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql
 delete mode 100755 tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py
 delete mode 100644 tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference
 delete mode 100755 tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py
 delete mode 100644 tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference
 delete mode 100644 tests/queries/0_stateless/00972_live_view_select_1.reference
 delete mode 100644 tests/queries/0_stateless/00972_live_view_select_1.sql
 delete mode 100644 tests/queries/0_stateless/00973_live_view_select.reference
 delete mode 100644 tests/queries/0_stateless/00973_live_view_select.sql
 delete mode 100644 tests/queries/0_stateless/00973_live_view_select_prewhere.reference
 delete mode 100644 tests/queries/0_stateless/00973_live_view_select_prewhere.sql
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select.reference
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select.sql
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_join.reference
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_join.sql
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_join_no_alias.reference
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_join_no_alias.sql
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_nested.reference
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_nested.sql
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation.reference
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation.sql
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation_table_alias.reference
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation_table_alias.sql
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_table_alias.reference
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_table_alias.sql
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation.reference
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation.sql
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation_in_subquery.reference
 delete mode 100644 tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation_in_subquery.sql
 delete mode 100644 tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference
 delete mode 100644 tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql
 delete mode 100644 tests/queries/0_stateless/00975_live_view_create.reference
 delete mode 100644 tests/queries/0_stateless/00975_live_view_create.sql
 delete mode 100644 tests/queries/0_stateless/00976_live_view_select_version.reference
 delete mode 100644 tests/queries/0_stateless/00976_live_view_select_version.sql
 delete mode 100644 tests/queries/0_stateless/00977_live_view_watch_events.reference
 delete mode 100644 tests/queries/0_stateless/00977_live_view_watch_events.sql
 delete mode 100644 tests/queries/0_stateless/00978_live_view_watch.reference
 delete mode 100644 tests/queries/0_stateless/00978_live_view_watch.sql
 delete mode 100755 tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.py
 delete mode 100644 tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.reference
 delete mode 100755 tests/queries/0_stateless/00979_live_view_watch_live.py
 delete mode 100644 tests/queries/0_stateless/00979_live_view_watch_live.reference
 delete mode 100755 tests/queries/0_stateless/00979_live_view_watch_live_with_subquery.py
 delete mode 100644 tests/queries/0_stateless/00979_live_view_watch_live_with_subquery.reference
 delete mode 100644 tests/queries/0_stateless/00991_live_view_watch_event_live.python
 delete mode 100644 tests/queries/0_stateless/00991_live_view_watch_event_live.reference
 delete mode 100755 tests/queries/0_stateless/00991_live_view_watch_http.python
 delete mode 100644 tests/queries/0_stateless/00991_live_view_watch_http.reference
 delete mode 100644 tests/queries/0_stateless/01056_create_table_as.reference
 delete mode 100644 tests/queries/0_stateless/01056_create_table_as.sql
 delete mode 100644 tests/queries/0_stateless/01071_live_view_detach_dependency.reference
 delete mode 100644 tests/queries/0_stateless/01071_live_view_detach_dependency.sql
 delete mode 100644 tests/queries/0_stateless/01190_full_attach_syntax.reference
 delete mode 100644 tests/queries/0_stateless/01190_full_attach_syntax.sql
 delete mode 100644 tests/queries/0_stateless/01235_live_view_over_distributed.reference
 delete mode 100644 tests/queries/0_stateless/01235_live_view_over_distributed.sql
 delete mode 100644 tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.reference
 delete mode 100644 tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql
 delete mode 100644 tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.reference
 delete mode 100644 tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.sql
 delete mode 100755 tests/queries/0_stateless/01246_insert_into_watch_live_view.py
 delete mode 100644 tests/queries/0_stateless/01246_insert_into_watch_live_view.reference
 delete mode 100644 tests/queries/0_stateless/01463_test_alter_live_view_refresh.reference
 delete mode 100644 tests/queries/0_stateless/01463_test_alter_live_view_refresh.sql
 delete mode 100644 tests/queries/0_stateless/01602_show_create_view.reference
 delete mode 100644 tests/queries/0_stateless/01602_show_create_view.sql
 delete mode 100644 tests/queries/0_stateless/01927_query_views_log_current_database.reference
 delete mode 100644 tests/queries/0_stateless/01927_query_views_log_current_database.sql
 delete mode 100644 tests/queries/0_stateless/02048_views_with_comment.reference
 delete mode 100644 tests/queries/0_stateless/02048_views_with_comment.sql

diff --git a/tests/integration/test_server_initialization/test.py b/tests/integration/test_server_initialization/test.py
index 1b57e14a51b..867dddeb8e2 100644
--- a/tests/integration/test_server_initialization/test.py
+++ b/tests/integration/test_server_initialization/test.py
@@ -50,17 +50,3 @@ def test_partially_dropped_tables(started_cluster):
         )
         == "0\n"
     )
-
-
-def test_live_view_dependency(started_cluster):
-    instance = started_cluster.instances["dummy"]
-    instance.query("CREATE DATABASE a_load_first")
-    instance.query("CREATE DATABASE b_load_second")
-    instance.query(
-        "CREATE TABLE b_load_second.mt (a Int32) Engine=MergeTree order by tuple()"
-    )
-    instance.query(
-        "CREATE LIVE VIEW a_load_first.lv AS SELECT sum(a) FROM b_load_second.mt",
-        settings={"allow_experimental_live_view": 1},
-    )
-    instance.restart_clickhouse()
diff --git a/tests/queries/0_stateless/00960_live_view_watch_events_live.py b/tests/queries/0_stateless/00960_live_view_watch_events_live.py
deleted file mode 100755
index 46c561516ba..00000000000
--- a/tests/queries/0_stateless/00960_live_view_watch_events_live.py
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-import signal
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1, client(
-    name="client2>", log=log
-) as client2:
-    client1.expect(prompt)
-    client2.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-    client2.send("SET allow_experimental_live_view = 1")
-    client2.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send(" DROP TABLE IF EXISTS test.mt")
-    client1.expect(prompt)
-    client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()")
-    client1.expect(prompt)
-    client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt")
-    client1.expect(prompt)
-    client1.send("WATCH test.lv EVENTS")
-    client1.expect("version")
-    client1.expect("1.*" + end_of_block)
-    client2.send("INSERT INTO test.mt VALUES (1),(2),(3)")
-    client1.expect("2.*" + end_of_block)
-    client2.send("INSERT INTO test.mt VALUES (4),(5),(6)")
-    client1.expect("3.*" + end_of_block)
-    # send Ctrl-C
-    client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
-    if match.groups()[1]:
-        client1.send(client1.command)
-        client1.expect(prompt)
-    client1.send("DROP TABLE test.lv")
-    client1.expect(prompt)
-    client1.send("DROP TABLE test.mt")
-    client1.expect(prompt)
diff --git a/tests/queries/0_stateless/00960_live_view_watch_events_live.reference b/tests/queries/0_stateless/00960_live_view_watch_events_live.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00962_live_view_periodic_refresh.py b/tests/queries/0_stateless/00962_live_view_periodic_refresh.py
deleted file mode 100755
index ac399d3c4c8..00000000000
--- a/tests/queries/0_stateless/00962_live_view_periodic_refresh.py
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-import signal
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1, client(
-    name="client2>", log=log
-) as client2:
-    client1.expect(prompt)
-    client2.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-    client2.send("SET allow_experimental_live_view = 1")
-    client2.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send(
-        "CREATE LIVE VIEW test.lv WITH REFRESH 1"
-        " AS SELECT value FROM system.events WHERE event = 'OSCPUVirtualTimeMicroseconds'"
-    )
-    client1.expect(prompt)
-    client1.send("WATCH test.lv FORMAT JSONEachRow")
-    client1.expect(r'"_version":' + end_of_block)
-    client1.expect(r'"_version":' + end_of_block)
-    client1.expect(r'"_version":' + end_of_block)
-    # send Ctrl-C
-    client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
-    if match.groups()[1]:
-        client1.send(client1.command)
-        client1.expect(prompt)
-    client1.send("DROP TABLE test.lv")
-    client1.expect(prompt)
diff --git a/tests/queries/0_stateless/00962_live_view_periodic_refresh.reference b/tests/queries/0_stateless/00962_live_view_periodic_refresh.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00962_live_view_periodic_refresh_and_timeout.py b/tests/queries/0_stateless/00962_live_view_periodic_refresh_and_timeout.py
deleted file mode 100755
index 983b330e24a..00000000000
--- a/tests/queries/0_stateless/00962_live_view_periodic_refresh_and_timeout.py
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-import time
-import signal
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1, client(
-    name="client2>", log=log
-) as client2:
-    client1.expect(prompt)
-    client2.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-    client2.send("SET allow_experimental_live_view = 1")
-    client2.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send(
-        "CREATE LIVE VIEW test.lv WITH REFRESH 1"
-        " AS SELECT value FROM system.events WHERE event = 'OSCPUVirtualTimeMicroseconds'"
-    )
-    client1.expect(prompt)
-    client1.send("WATCH test.lv FORMAT JSONEachRow")
-    client1.expect(r'"_version":' + end_of_block)
-    client1.expect(r'"_version":' + end_of_block)
-    client1.expect(r'"_version":' + end_of_block)
-    # send Ctrl-C
-    client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
-    if match.groups()[1]:
-        client1.send(client1.command)
-        client1.expect(prompt)
diff --git a/tests/queries/0_stateless/00962_live_view_periodic_refresh_and_timeout.reference b/tests/queries/0_stateless/00962_live_view_periodic_refresh_and_timeout.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00962_live_view_periodic_refresh_dictionary.py b/tests/queries/0_stateless/00962_live_view_periodic_refresh_dictionary.py
deleted file mode 100755
index 9d2a26c83c0..00000000000
--- a/tests/queries/0_stateless/00962_live_view_periodic_refresh_dictionary.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-import signal
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1, client(
-    name="client2>", log=log
-) as client2:
-    client1.expect(prompt)
-    client2.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-    client2.send("SET allow_experimental_live_view = 1")
-    client2.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send("DROP TABLE IF EXISTS test.mt")
-    client1.expect(prompt)
-    client1.send("DROP DICTIONARY IF EXITS test.dict")
-    client1.expect(prompt)
-
-    client1.send(
-        "CREATE TABLE test.mt (a Int32, b Int32) Engine=MergeTree order by tuple()"
-    )
-    client1.expect(prompt)
-    client1.send(
-        "CREATE DICTIONARY test.dict(a Int32, b Int32) PRIMARY KEY a LAYOUT(FLAT()) "
-        + "SOURCE(CLICKHOUSE(db 'test' table 'mt')) LIFETIME(1)"
-    )
-    client1.expect(prompt)
-    client1.send("CREATE LIVE VIEW test.lv WITH REFRESH 1 AS SELECT * FROM test.dict")
-    client1.expect(prompt)
-
-    client2.send("INSERT INTO test.mt VALUES (1,2)")
-    client2.expect(prompt)
-
-    client1.send("WATCH test.lv FORMAT JSONEachRow")
-    client1.expect(r'"_version":"1"')
-
-    client2.send("INSERT INTO test.mt VALUES (2,2)")
-    client2.expect(prompt)
-    client1.expect(r'"_version":"2"')
-
-    client2.send("INSERT INTO test.mt VALUES (3,2)")
-    client2.expect(prompt)
-    client1.expect(r'"_version":"3"')
-
-    # send Ctrl-C
-    client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
-    if match.groups()[1]:
-        client1.send(client1.command)
-        client1.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send("DROP DICTIONARY IF EXISTS test.dict")
-    client1.expect(prompt)
-    client1.send("DROP TABLE IF EXISTS test.mt")
-    client1.expect(prompt)
diff --git a/tests/queries/0_stateless/00962_live_view_periodic_refresh_dictionary.reference b/tests/queries/0_stateless/00962_live_view_periodic_refresh_dictionary.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py b/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py
deleted file mode 100755
index c8902203a3d..00000000000
--- a/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-import signal
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1, client(
-    name="client2>", log=log
-) as client2:
-    client1.expect(prompt)
-    client2.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-    client2.send("SET allow_experimental_live_view = 1")
-    client2.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send("DROP TABLE IF EXISTS test.mt")
-    client1.expect(prompt)
-    client1.send("SET live_view_heartbeat_interval=1")
-    client1.expect(prompt)
-    client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()")
-    client1.expect(prompt)
-    client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt")
-    client1.expect(prompt)
-    client1.send("WATCH test.lv EVENTS FORMAT CSV")
-    client1.expect("Progress: 1.00 rows.*\)")
-    client2.send("INSERT INTO test.mt VALUES (1)")
-    client2.expect(prompt)
-    client1.expect("Progress: 2.00 rows.*\)")
-    client2.send("INSERT INTO test.mt VALUES (2),(3)")
-    client2.expect(prompt)
-    # wait for heartbeat
-    client1.expect("Progress: 3.00 rows.*\)")
-    # send Ctrl-C
-    client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
-    if match.groups()[1]:
-        client1.send(client1.command)
-        client1.expect(prompt)
-    client1.send("DROP TABLE test.lv")
-    client1.expect(prompt)
-    client1.send("DROP TABLE test.mt")
-    client1.expect(prompt)
diff --git a/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference b/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py b/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py
deleted file mode 100755
index b499f673cc0..00000000000
--- a/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-import signal
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1, client(
-    name="client2>", log=log
-) as client2:
-    client1.expect(prompt)
-    client2.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-    client2.send("SET allow_experimental_live_view = 1")
-    client2.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send(" DROP TABLE IF EXISTS test.mt")
-    client1.expect(prompt)
-    client1.send("SET live_view_heartbeat_interval=1")
-    client1.expect(prompt)
-    client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()")
-    client1.expect(prompt)
-    client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt")
-    client1.expect(prompt)
-    client1.send("WATCH test.lv")
-    client1.expect("_version")
-    client1.expect(r"0.*1" + end_of_block)
-    client2.send("INSERT INTO test.mt VALUES (1),(2),(3)")
-    client1.expect(r"6.*2" + end_of_block)
-    # wait for heartbeat
-    client1.expect("Progress: 2.00 rows.*\)")
-    # send Ctrl-C
-    client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
-    if match.groups()[1]:
-        client1.send(client1.command)
-        client1.expect(prompt)
-    client1.send("DROP TABLE test.lv")
-    client1.expect(prompt)
-    client1.send("DROP TABLE test.mt")
-    client1.expect(prompt)
diff --git a/tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference b/tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00966_live_view_watch_events_http.py b/tests/queries/0_stateless/00966_live_view_watch_events_http.py
deleted file mode 100755
index 1f2ddae23d6..00000000000
--- a/tests/queries/0_stateless/00966_live_view_watch_events_http.py
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-from httpclient import client as http_client
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1:
-    client1.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send(" DROP TABLE IF EXISTS test.mt")
-    client1.expect(prompt)
-    client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()")
-    client1.expect(prompt)
-    client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt")
-    client1.expect(prompt)
-
-    try:
-        with http_client(
-            {
-                "method": "GET",
-                "url": "/?allow_experimental_live_view=1&query=WATCH%20test.lv%20EVENTS",
-            },
-            name="client2>",
-            log=log,
-        ) as client2:
-            client2.expect(".*1\n")
-            client1.send("INSERT INTO test.mt VALUES (1),(2),(3)")
-            client1.expect(prompt)
-            client2.expect(".*2\n")
-    finally:
-        client1.send("DROP TABLE test.lv")
-        client1.expect(prompt)
-        client1.send("DROP TABLE test.mt")
-        client1.expect(prompt)
diff --git a/tests/queries/0_stateless/00966_live_view_watch_events_http.reference b/tests/queries/0_stateless/00966_live_view_watch_events_http.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00967_live_view_watch_http.py b/tests/queries/0_stateless/00967_live_view_watch_http.py
deleted file mode 100755
index 92e192cc7f2..00000000000
--- a/tests/queries/0_stateless/00967_live_view_watch_http.py
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-from httpclient import client as http_client
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1:
-    client1.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send(" DROP TABLE IF EXISTS test.mt")
-    client1.expect(prompt)
-    client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()")
-    client1.expect(prompt)
-    client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt")
-    client1.expect(prompt)
-
-    try:
-        with http_client(
-            {
-                "method": "GET",
-                "url": "/?allow_experimental_live_view=1&query=WATCH%20test.lv",
-            },
-            name="client2>",
-            log=log,
-        ) as client2:
-            client2.expect(".*0\t1\n")
-            client1.send("INSERT INTO test.mt VALUES (1),(2),(3)")
-            client1.expect(prompt)
-            client2.expect(".*6\t2\n")
-    finally:
-        client1.send("DROP TABLE test.lv")
-        client1.expect(prompt)
-        client1.send("DROP TABLE test.mt")
-        client1.expect(prompt)
diff --git a/tests/queries/0_stateless/00967_live_view_watch_http.reference b/tests/queries/0_stateless/00967_live_view_watch_http.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference b/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference
deleted file mode 100644
index 5f48ead3147..00000000000
--- a/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference
+++ /dev/null
@@ -1,4 +0,0 @@
-{"row":{"a":1}}
-{"row":{"a":2}}
-{"row":{"a":3}}
-{"progress":{"read_rows":"3","read_bytes":"36","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","result_rows":"0","result_bytes":"0"}}
diff --git a/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql b/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql
deleted file mode 100644
index 77b0f37e10f..00000000000
--- a/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql
+++ /dev/null
@@ -1,16 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT * FROM mt;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT * FROM lv FORMAT JSONEachRowWithProgress;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference b/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference
deleted file mode 100644
index 80ec35990d6..00000000000
--- a/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference
+++ /dev/null
@@ -1,6 +0,0 @@
-{"row":{"sum(a)":"0","_version":"1"}}
-{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","result_rows":"0","result_bytes":"0"}}
-{"row":{"sum(a)":"6","_version":"2"}}
-{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","result_rows":"0","result_bytes":"0"}}
-{"row":{"sum(a)":"21","_version":"3"}}
-{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","result_rows":"0","result_bytes":"0"}}
diff --git a/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql b/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql
deleted file mode 100644
index 2d5d5f48b34..00000000000
--- a/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql
+++ /dev/null
@@ -1,22 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT sum(a) FROM mt;
-
-WATCH lv LIMIT 0 FORMAT JSONEachRowWithProgress;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-WATCH lv LIMIT 0 FORMAT JSONEachRowWithProgress;
-
-INSERT INTO mt VALUES (4),(5),(6);
-
-WATCH lv LIMIT 0 FORMAT JSONEachRowWithProgress;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py b/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py
deleted file mode 100755
index febe439e63b..00000000000
--- a/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-from httpclient import client as http_client
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1:
-    client1.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send(" DROP TABLE IF EXISTS test.mt")
-    client1.expect(prompt)
-    client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()")
-    client1.expect(prompt)
-    client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt")
-    client1.expect(prompt)
-
-    with http_client(
-        {
-            "method": "GET",
-            "url": "/?allow_experimental_live_view=1&live_view_heartbeat_interval=1&query=WATCH%20test.lv%20EVENTS%20FORMAT%20JSONEachRowWithProgress",
-        },
-        name="client2>",
-        log=log,
-    ) as client2:
-        client2.expect(
-            '{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","result_rows":"0","result_bytes":"0"}}\n',
-            escape=True,
-        )
-        client2.expect('{"row":{"version":"1"}', escape=True)
-        client2.expect(
-            '{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","result_rows":"0","result_bytes":"0"}}',
-            escape=True,
-        )
-        # heartbeat is provided by progress message
-        client2.expect(
-            '{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","result_rows":"0","result_bytes":"0"}}',
-            escape=True,
-        )
-
-        client1.send("INSERT INTO test.mt VALUES (1),(2),(3)")
-        client1.expect(prompt)
-
-        client2.expect('{"row":{"version":"2"}}\n', escape=True)
-
-    client1.send("DROP TABLE test.lv")
-    client1.expect(prompt)
-    client1.send("DROP TABLE test.mt")
-    client1.expect(prompt)
diff --git a/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference b/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py b/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py
deleted file mode 100755
index 117f7b7c786..00000000000
--- a/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-from httpclient import client as http_client
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1:
-    client1.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send(" DROP TABLE IF EXISTS test.mt")
-    client1.expect(prompt)
-    client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()")
-    client1.expect(prompt)
-    client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt")
-    client1.expect(prompt)
-
-    with http_client(
-        {
-            "method": "GET",
-            "url": "/?allow_experimental_live_view=1&live_view_heartbeat_interval=1&query=WATCH%20test.lv%20FORMAT%20JSONEachRowWithProgress",
-        },
-        name="client2>",
-        log=log,
-    ) as client2:
-        client2.expect(
-            '"progress".*',
-        )
-        client2.expect('{"row":{"sum(a)":"0","_version":"1"}}\n', escape=True)
-        client2.expect('"progress".*\n')
-        # heartbeat is provided by progress message
-        client2.expect('"progress".*\n')
-
-        client1.send("INSERT INTO test.mt VALUES (1),(2),(3)")
-        client1.expect(prompt)
-
-        client2.expect('"progress".*"read_rows":"2".*\n')
-        client2.expect('{"row":{"sum(a)":"6","_version":"2"}}\n', escape=True)
-
-    client1.send("DROP TABLE test.lv")
-    client1.expect(prompt)
-    client1.send("DROP TABLE test.mt")
-    client1.expect(prompt)
diff --git a/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference b/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00972_live_view_select_1.reference b/tests/queries/0_stateless/00972_live_view_select_1.reference
deleted file mode 100644
index d00491fd7e5..00000000000
--- a/tests/queries/0_stateless/00972_live_view_select_1.reference
+++ /dev/null
@@ -1 +0,0 @@
-1
diff --git a/tests/queries/0_stateless/00972_live_view_select_1.sql b/tests/queries/0_stateless/00972_live_view_select_1.sql
deleted file mode 100644
index 20344466856..00000000000
--- a/tests/queries/0_stateless/00972_live_view_select_1.sql
+++ /dev/null
@@ -1,11 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-
-CREATE LIVE VIEW lv AS SELECT 1;
-
-SELECT * FROM lv;
-
-DROP TABLE lv;
diff --git a/tests/queries/0_stateless/00973_live_view_select.reference b/tests/queries/0_stateless/00973_live_view_select.reference
deleted file mode 100644
index 75236c0daf7..00000000000
--- a/tests/queries/0_stateless/00973_live_view_select.reference
+++ /dev/null
@@ -1,4 +0,0 @@
-6	1
-6	1
-12	2
-12	2
diff --git a/tests/queries/0_stateless/00973_live_view_select.sql b/tests/queries/0_stateless/00973_live_view_select.sql
deleted file mode 100644
index 0844c730b26..00000000000
--- a/tests/queries/0_stateless/00973_live_view_select.sql
+++ /dev/null
@@ -1,22 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT sum(a) FROM mt;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00973_live_view_select_prewhere.reference b/tests/queries/0_stateless/00973_live_view_select_prewhere.reference
deleted file mode 100644
index 3a6fe59ae6d..00000000000
--- a/tests/queries/0_stateless/00973_live_view_select_prewhere.reference
+++ /dev/null
@@ -1,2 +0,0 @@
-5	1
-10	2
diff --git a/tests/queries/0_stateless/00973_live_view_select_prewhere.sql b/tests/queries/0_stateless/00973_live_view_select_prewhere.sql
deleted file mode 100644
index c40ca5ab410..00000000000
--- a/tests/queries/0_stateless/00973_live_view_select_prewhere.sql
+++ /dev/null
@@ -1,25 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_analyzer = 1;
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS lv2;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT sum(a) AS sum_a FROM mt PREWHERE a > 1;
-CREATE LIVE VIEW lv2 AS SELECT sum(number) AS sum_number FROM system.numbers PREWHERE number > 1; -- { serverError 182 }
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *, _version FROM lv;
-SELECT *, _version FROM lv PREWHERE sum_a > 5; -- { serverError 182 }
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *, _version FROM lv;
-SELECT *, _version FROM lv PREWHERE sum_a > 10; -- { serverError 182 }
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select.reference b/tests/queries/0_stateless/00973_live_view_with_subquery_select.reference
deleted file mode 100644
index ebf18a51290..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select.reference
+++ /dev/null
@@ -1,18 +0,0 @@
-1	1
-2	1
-3	1
-1	1
-2	1
-3	1
-1	2
-2	2
-3	2
-1	2
-2	2
-3	2
-1	2
-2	2
-3	2
-1	2
-2	2
-3	2
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select.sql b/tests/queries/0_stateless/00973_live_view_with_subquery_select.sql
deleted file mode 100644
index 9d644f245f6..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select.sql
+++ /dev/null
@@ -1,22 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT a FROM (SELECT a FROM mt);
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_join.reference b/tests/queries/0_stateless/00973_live_view_with_subquery_select_join.reference
deleted file mode 100644
index 7a596e87ed6..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_join.reference
+++ /dev/null
@@ -1,6 +0,0 @@
-1	hello	2
-1	hello	2
-1	hello	3
-2	hello	3
-1	hello	3
-2	hello	3
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_join.sql b/tests/queries/0_stateless/00973_live_view_with_subquery_select_join.sql
deleted file mode 100644
index 562e6df1bfd..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_join.sql
+++ /dev/null
@@ -1,30 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS A;
-DROP TABLE IF EXISTS B;
-
-CREATE TABLE A (id Int32) Engine=Memory;
-CREATE TABLE B (id Int32, name String) Engine=Memory;
-
-CREATE LIVE VIEW lv AS SELECT id, name FROM ( SELECT A.id, B.name FROM A as A, B as B WHERE A.id = B.id );
-
-SELECT * FROM lv;
-
-INSERT INTO A VALUES (1);
-INSERT INTO B VALUES (1, 'hello');
-
-SELECT *,_version FROM lv ORDER BY id;
-SELECT *,_version FROM lv ORDER BY id;
-
-INSERT INTO A VALUES (2)
-INSERT INTO B VALUES (2, 'hello')
-
-SELECT *,_version FROM lv ORDER BY id;
-SELECT *,_version FROM lv ORDER BY id;
-
-DROP TABLE lv;
-DROP TABLE A;
-DROP TABLE B;
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_join_no_alias.reference b/tests/queries/0_stateless/00973_live_view_with_subquery_select_join_no_alias.reference
deleted file mode 100644
index 7a596e87ed6..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_join_no_alias.reference
+++ /dev/null
@@ -1,6 +0,0 @@
-1	hello	2
-1	hello	2
-1	hello	3
-2	hello	3
-1	hello	3
-2	hello	3
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_join_no_alias.sql b/tests/queries/0_stateless/00973_live_view_with_subquery_select_join_no_alias.sql
deleted file mode 100644
index 60078ccff30..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_join_no_alias.sql
+++ /dev/null
@@ -1,30 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS A;
-DROP TABLE IF EXISTS B;
-
-CREATE TABLE A (id Int32) Engine=Memory;
-CREATE TABLE B (id Int32, name String) Engine=Memory;
-
-CREATE LIVE VIEW lv AS SELECT id, name FROM ( SELECT A.id, B.name FROM A, B WHERE A.id = B.id);
-
-SELECT * FROM lv;
-
-INSERT INTO A VALUES (1);
-INSERT INTO B VALUES (1, 'hello');
-
-SELECT *,_version FROM lv ORDER BY id;
-SELECT *,_version FROM lv ORDER BY id;
-
-INSERT INTO A VALUES (2)
-INSERT INTO B VALUES (2, 'hello')
-
-SELECT *,_version FROM lv ORDER BY id;
-SELECT *,_version FROM lv ORDER BY id;
-
-DROP TABLE lv;
-DROP TABLE A;
-DROP TABLE B;
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested.reference b/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested.reference
deleted file mode 100644
index ebf18a51290..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested.reference
+++ /dev/null
@@ -1,18 +0,0 @@
-1	1
-2	1
-3	1
-1	1
-2	1
-3	1
-1	2
-2	2
-3	2
-1	2
-2	2
-3	2
-1	2
-2	2
-3	2
-1	2
-2	2
-3	2
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested.sql b/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested.sql
deleted file mode 100644
index 700de5955f9..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested.sql
+++ /dev/null
@@ -1,22 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT a FROM ( SELECT * FROM ( SELECT a FROM (SELECT a FROM mt) ) );
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation.reference b/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation.reference
deleted file mode 100644
index 75236c0daf7..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation.reference
+++ /dev/null
@@ -1,4 +0,0 @@
-6	1
-6	1
-12	2
-12	2
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation.sql b/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation.sql
deleted file mode 100644
index 3a9408a62fa..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation.sql
+++ /dev/null
@@ -1,22 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT * FROM ( SELECT sum(a) FROM ( SELECT a FROM (SELECT a FROM mt) ) );
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation_table_alias.reference b/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation_table_alias.reference
deleted file mode 100644
index 75236c0daf7..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation_table_alias.reference
+++ /dev/null
@@ -1,4 +0,0 @@
-6	1
-6	1
-12	2
-12	2
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation_table_alias.sql b/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation_table_alias.sql
deleted file mode 100644
index c0f218edbd8..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_nested_with_aggregation_table_alias.sql
+++ /dev/null
@@ -1,22 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT * FROM ( SELECT sum(boo.x) FROM ( SELECT foo.x FROM (SELECT a AS x FROM mt) AS foo) AS boo );
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_table_alias.reference b/tests/queries/0_stateless/00973_live_view_with_subquery_select_table_alias.reference
deleted file mode 100644
index ebf18a51290..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_table_alias.reference
+++ /dev/null
@@ -1,18 +0,0 @@
-1	1
-2	1
-3	1
-1	1
-2	1
-3	1
-1	2
-2	2
-3	2
-1	2
-2	2
-3	2
-1	2
-2	2
-3	2
-1	2
-2	2
-3	2
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_table_alias.sql b/tests/queries/0_stateless/00973_live_view_with_subquery_select_table_alias.sql
deleted file mode 100644
index 2d7da9d6df5..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_table_alias.sql
+++ /dev/null
@@ -1,22 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT foo.x FROM (SELECT a AS x FROM mt) AS foo;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation.reference b/tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation.reference
deleted file mode 100644
index 75236c0daf7..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation.reference
+++ /dev/null
@@ -1,4 +0,0 @@
-6	1
-6	1
-12	2
-12	2
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation.sql b/tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation.sql
deleted file mode 100644
index bdf74949f57..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation.sql
+++ /dev/null
@@ -1,22 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT sum(a) FROM (SELECT a FROM mt);
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation_in_subquery.reference b/tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation_in_subquery.reference
deleted file mode 100644
index 75236c0daf7..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation_in_subquery.reference
+++ /dev/null
@@ -1,4 +0,0 @@
-6	1
-6	1
-12	2
-12	2
diff --git a/tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation_in_subquery.sql b/tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation_in_subquery.sql
deleted file mode 100644
index a80af3e3337..00000000000
--- a/tests/queries/0_stateless/00973_live_view_with_subquery_select_with_aggregation_in_subquery.sql
+++ /dev/null
@@ -1,22 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT * FROM (SELECT sum(a) FROM mt);
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-SELECT *,_version FROM lv;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference b/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference
deleted file mode 100644
index 6d50f0e9c3a..00000000000
--- a/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference
+++ /dev/null
@@ -1,2 +0,0 @@
-6
-21
diff --git a/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql b/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql
deleted file mode 100644
index 09eea37de60..00000000000
--- a/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql
+++ /dev/null
@@ -1,20 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT * FROM mt;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT sum(a) FROM lv;
-
-INSERT INTO mt VALUES (4),(5),(6);
-
-SELECT sum(a) FROM lv;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00975_live_view_create.reference b/tests/queries/0_stateless/00975_live_view_create.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00975_live_view_create.sql b/tests/queries/0_stateless/00975_live_view_create.sql
deleted file mode 100644
index 1e7eb674bcc..00000000000
--- a/tests/queries/0_stateless/00975_live_view_create.sql
+++ /dev/null
@@ -1,11 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT * FROM mt;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00976_live_view_select_version.reference b/tests/queries/0_stateless/00976_live_view_select_version.reference
deleted file mode 100644
index 453bd800469..00000000000
--- a/tests/queries/0_stateless/00976_live_view_select_version.reference
+++ /dev/null
@@ -1,3 +0,0 @@
-1	1
-2	1
-3	1
diff --git a/tests/queries/0_stateless/00976_live_view_select_version.sql b/tests/queries/0_stateless/00976_live_view_select_version.sql
deleted file mode 100644
index 39ca587a49f..00000000000
--- a/tests/queries/0_stateless/00976_live_view_select_version.sql
+++ /dev/null
@@ -1,16 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT * FROM mt;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-SELECT *,_version FROM lv;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00977_live_view_watch_events.reference b/tests/queries/0_stateless/00977_live_view_watch_events.reference
deleted file mode 100644
index 01e79c32a8c..00000000000
--- a/tests/queries/0_stateless/00977_live_view_watch_events.reference
+++ /dev/null
@@ -1,3 +0,0 @@
-1
-2
-3
diff --git a/tests/queries/0_stateless/00977_live_view_watch_events.sql b/tests/queries/0_stateless/00977_live_view_watch_events.sql
deleted file mode 100644
index 3fe395b45fa..00000000000
--- a/tests/queries/0_stateless/00977_live_view_watch_events.sql
+++ /dev/null
@@ -1,22 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT sum(a) FROM mt;
-
-WATCH lv EVENTS LIMIT 0;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-WATCH lv EVENTS LIMIT 0;
-
-INSERT INTO mt VALUES (4),(5),(6);
-
-WATCH lv EVENTS LIMIT 0;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00978_live_view_watch.reference b/tests/queries/0_stateless/00978_live_view_watch.reference
deleted file mode 100644
index 6fbbedf1b21..00000000000
--- a/tests/queries/0_stateless/00978_live_view_watch.reference
+++ /dev/null
@@ -1,3 +0,0 @@
-0	1
-6	2
-21	3
diff --git a/tests/queries/0_stateless/00978_live_view_watch.sql b/tests/queries/0_stateless/00978_live_view_watch.sql
deleted file mode 100644
index a76b8aef0d0..00000000000
--- a/tests/queries/0_stateless/00978_live_view_watch.sql
+++ /dev/null
@@ -1,22 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS mt;
-
-CREATE TABLE mt (a Int32) Engine=MergeTree order by tuple();
-CREATE LIVE VIEW lv AS SELECT sum(a) FROM mt;
-
-WATCH lv LIMIT 0;
-
-INSERT INTO mt VALUES (1),(2),(3);
-
-WATCH lv LIMIT 0;
-
-INSERT INTO mt VALUES (4),(5),(6);
-
-WATCH lv LIMIT 0;
-
-DROP TABLE lv;
-DROP TABLE mt;
diff --git a/tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.py b/tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.py
deleted file mode 100755
index ef144d044c2..00000000000
--- a/tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.py
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-import signal
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1, client(
-    name="client2>", log=log
-) as client2:
-    client1.expect(prompt)
-    client2.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-    client2.send("SET allow_experimental_live_view = 1")
-    client2.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send("DROP TABLE IF EXISTS test.mt")
-    client1.expect(prompt)
-    client1.send(
-        "CREATE TABLE test.mt (time DateTime, location String, temperature UInt32) Engine=MergeTree order by tuple()"
-    )
-    client1.expect(prompt)
-    client1.send(
-        "CREATE LIVE VIEW test.lv AS SELECT toStartOfDay(time) AS day, location, avg(temperature) FROM test.mt GROUP BY day, location ORDER BY day, location"
-    )
-    client1.expect(prompt)
-    client1.send("WATCH test.lv FORMAT CSVWithNames")
-    client2.send(
-        "INSERT INTO test.mt VALUES ('2019-01-01 00:00:00','New York',60),('2019-01-01 00:10:00','New York',70)"
-    )
-    client2.expect(prompt)
-    client1.expect(r'"2019-01-01 00:00:00","New York",65')
-    client2.send(
-        "INSERT INTO test.mt VALUES ('2019-01-01 00:00:00','Moscow',30),('2019-01-01 00:10:00', 'Moscow', 40)"
-    )
-    client2.expect(prompt)
-    client1.expect(r'"2019-01-01 00:00:00","Moscow",35')
-    client1.expect(r'"2019-01-01 00:00:00","New York",65')
-    client2.send(
-        "INSERT INTO test.mt VALUES ('2019-01-02 00:00:00','New York',50),('2019-01-02 00:10:00','New York',60)"
-    )
-    client2.expect(prompt)
-    client1.expect(r'"2019-01-01 00:00:00","Moscow",35')
-    client1.expect(r'"2019-01-01 00:00:00","New York",65')
-    client1.expect(r'"2019-01-02 00:00:00","New York",55')
-    client2.send(
-        "INSERT INTO test.mt VALUES ('2019-01-02 00:00:00','Moscow',20),('2019-01-02 00:10:00', 'Moscow', 30)"
-    )
-    client2.expect(prompt)
-    client1.expect(r'"2019-01-01 00:00:00","Moscow",35')
-    client1.expect(r'"2019-01-01 00:00:00","New York",65')
-    client1.expect(r'"2019-01-02 00:00:00","Moscow",25')
-    client1.expect(r'"2019-01-02 00:00:00","New York",55')
-    client2.send(
-        "INSERT INTO test.mt VALUES ('2019-01-02 00:03:00','New York',40),('2019-01-02 00:06:00','New York',30)"
-    )
-    client2.expect(prompt)
-    client1.expect(r'"2019-01-01 00:00:00","Moscow",35')
-    client1.expect(r'"2019-01-01 00:00:00","New York",65')
-    client1.expect(r'"2019-01-02 00:00:00","Moscow",25')
-    client1.expect(r'"2019-01-02 00:00:00","New York",45')
-    # send Ctrl-C
-    client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
-    if match.groups()[1]:
-        client1.send(client1.command)
-        client1.expect(prompt)
-    client1.send("DROP TABLE test.lv")
-    client1.expect(prompt)
-    client1.send("DROP TABLE test.mt")
-    client1.expect(prompt)
diff --git a/tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.reference b/tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00979_live_view_watch_live.py b/tests/queries/0_stateless/00979_live_view_watch_live.py
deleted file mode 100755
index b099b56ae48..00000000000
--- a/tests/queries/0_stateless/00979_live_view_watch_live.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-import signal
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1, client(
-    name="client2>", log=log
-) as client2:
-    client1.expect(prompt)
-    client2.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-    client2.send("SET allow_experimental_live_view = 1")
-    client2.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send("DROP TABLE IF EXISTS test.mt")
-    client1.expect(prompt)
-    client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()")
-    client1.expect(prompt)
-    client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt")
-    client1.expect(prompt)
-    client1.send("WATCH test.lv")
-    client1.expect("_version")
-    client1.expect(r"0.*1" + end_of_block)
-    client2.send("INSERT INTO test.mt VALUES (1),(2),(3)")
-    client1.expect(r"6.*2" + end_of_block)
-    client2.expect(prompt)
-    client2.send("INSERT INTO test.mt VALUES (4),(5),(6)")
-    client1.expect(r"21.*3" + end_of_block)
-    client2.expect(prompt)
-    for i in range(1, 129):
-        client2.send("INSERT INTO test.mt VALUES (1)")
-        client1.expect(r"%d.*%d" % (21 + i, 3 + i) + end_of_block)
-        client2.expect(prompt)
-    # send Ctrl-C
-    client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
-    if match.groups()[1]:
-        client1.send(client1.command)
-        client1.expect(prompt)
-    client1.send("DROP TABLE test.lv")
-    client1.expect(prompt)
-    client1.send("DROP TABLE test.mt")
-    client1.expect(prompt)
diff --git a/tests/queries/0_stateless/00979_live_view_watch_live.reference b/tests/queries/0_stateless/00979_live_view_watch_live.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00979_live_view_watch_live_with_subquery.py b/tests/queries/0_stateless/00979_live_view_watch_live_with_subquery.py
deleted file mode 100755
index a7c1adac214..00000000000
--- a/tests/queries/0_stateless/00979_live_view_watch_live_with_subquery.py
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-import signal
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1, client(
-    name="client2>", log=log
-) as client2:
-    client1.expect(prompt)
-    client2.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-    client2.send("SET allow_experimental_live_view = 1")
-    client2.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send(" DROP TABLE IF EXISTS test.mt")
-    client1.expect(prompt)
-    client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()")
-    client1.expect(prompt)
-    client1.send(
-        "CREATE LIVE VIEW test.lv AS SELECT * FROM ( SELECT sum(A.a) FROM (SELECT * FROM test.mt) AS A )"
-    )
-    client1.expect(prompt)
-    client1.send("WATCH test.lv")
-    client1.expect("_version")
-    client1.expect(r"0.*1" + end_of_block)
-    client2.send("INSERT INTO test.mt VALUES (1),(2),(3)")
-    client1.expect(r"6.*2" + end_of_block)
-    client2.expect(prompt)
-    client2.send("INSERT INTO test.mt VALUES (4),(5),(6)")
-    client1.expect(r"21.*3" + end_of_block)
-    client2.expect(prompt)
-    for i in range(1, 129):
-        client2.send("INSERT INTO test.mt VALUES (1)")
-        client1.expect(r"%d.*%d" % (21 + i, 3 + i) + end_of_block)
-        client2.expect(prompt)
-    # send Ctrl-C
-    client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
-    if match.groups()[1]:
-        client1.send(client1.command)
-        client1.expect(prompt)
-    client1.send("DROP TABLE test.lv")
-    client1.expect(prompt)
-    client1.send("DROP TABLE test.mt")
-    client1.expect(prompt)
diff --git a/tests/queries/0_stateless/00979_live_view_watch_live_with_subquery.reference b/tests/queries/0_stateless/00979_live_view_watch_live_with_subquery.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/00991_live_view_watch_event_live.python b/tests/queries/0_stateless/00991_live_view_watch_event_live.python
deleted file mode 100644
index 9b7a3300c15..00000000000
--- a/tests/queries/0_stateless/00991_live_view_watch_event_live.python
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/usr/bin/env python3
-
-import subprocess
-import threading
-import queue as queue
-import os
-import sys
-import signal
-
-
-CLICKHOUSE_CLIENT = os.environ.get("CLICKHOUSE_CLIENT")
-CLICKHOUSE_CURL = os.environ.get("CLICKHOUSE_CURL")
-CLICKHOUSE_URL = os.environ.get("CLICKHOUSE_URL")
-
-
-def send_query(query):
-    cmd = list(CLICKHOUSE_CLIENT.split())
-    cmd += ["--query", query]
-    # print(cmd)
-    return subprocess.Popen(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
-    ).stdout
-
-
-def send_query_in_process_group(query):
-    cmd = list(CLICKHOUSE_CLIENT.split())
-    cmd += ["--query", query]
-    # print(cmd)
-    return subprocess.Popen(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid
-    )
-
-
-def read_lines_and_push_to_queue(pipe, queue):
-    try:
-        for line in iter(pipe.readline, ""):
-            line = line.strip()
-            print(line)
-            sys.stdout.flush()
-            queue.put(line)
-    except KeyboardInterrupt:
-        pass
-
-    queue.put(None)
-
-
-def test():
-    send_query("DROP TABLE IF EXISTS test.lv").read()
-    send_query("DROP TABLE IF EXISTS test.mt").read()
-    send_query(
-        "CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()"
-    ).read()
-    send_query("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt").read()
-
-    q = queue.Queue()
-    p = send_query_in_process_group("WATCH test.lv")
-    thread = threading.Thread(target=read_lines_and_push_to_queue, args=(p.stdout, q))
-    thread.start()
-
-    line = q.get()
-    print(line)
-    assert line == "0\t1"
-
-    send_query("INSERT INTO test.mt VALUES (1),(2),(3)").read()
-    line = q.get()
-    print(line)
-    assert line == "6\t2"
-
-    send_query("INSERT INTO test.mt VALUES (4),(5),(6)").read()
-    line = q.get()
-    print(line)
-    assert line == "21\t3"
-
-    # Send Ctrl+C to client.
-    os.killpg(os.getpgid(p.pid), signal.SIGINT)
-    # This insert shouldn't affect lv.
-    send_query("INSERT INTO test.mt VALUES (7),(8),(9)").read()
-    line = q.get()
-    print(line)
-    assert line is None
-
-    send_query("DROP TABLE if exists test.lv").read()
-    send_query("DROP TABLE if exists test.lv").read()
-
-    thread.join()
-
-
-test()
diff --git a/tests/queries/0_stateless/00991_live_view_watch_event_live.reference b/tests/queries/0_stateless/00991_live_view_watch_event_live.reference
deleted file mode 100644
index 1e94cdade41..00000000000
--- a/tests/queries/0_stateless/00991_live_view_watch_event_live.reference
+++ /dev/null
@@ -1,7 +0,0 @@
-0	1
-0	1
-6	2
-6	2
-21	3
-21	3
-None
diff --git a/tests/queries/0_stateless/00991_live_view_watch_http.python b/tests/queries/0_stateless/00991_live_view_watch_http.python
deleted file mode 100755
index 72c07b27d82..00000000000
--- a/tests/queries/0_stateless/00991_live_view_watch_http.python
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/usr/bin/env python3
-
-import subprocess
-import threading
-import queue as queue
-import os
-import sys
-
-
-CLICKHOUSE_CLIENT = os.environ.get("CLICKHOUSE_CLIENT")
-CLICKHOUSE_CURL = os.environ.get("CLICKHOUSE_CURL")
-CLICKHOUSE_URL = os.environ.get("CLICKHOUSE_URL")
-
-
-def send_query(query):
-    cmd = list(CLICKHOUSE_CLIENT.split())
-    cmd += ["--query", query]
-    # print(cmd)
-    return subprocess.Popen(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
-    ).stdout
-
-
-def send_http_query(query):
-    cmd = list(CLICKHOUSE_CURL.split())  # list(['curl', '-sSN', '--max-time', '10'])
-    cmd += ["-sSN", CLICKHOUSE_URL, "-d", query]
-    return subprocess.Popen(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
-    ).stdout
-
-
-def read_lines_and_push_to_queue(pipe, queue):
-    for line in iter(pipe.readline, ""):
-        line = line.strip()
-        print(line)
-        sys.stdout.flush()
-        queue.put(line)
-
-    queue.put(None)
-
-
-def test():
-    send_query("DROP TABLE IF EXISTS test.lv").read()
-    send_query("DROP TABLE IF EXISTS test.mt").read()
-    send_query(
-        "CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()"
-    ).read()
-    send_query("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt").read()
-
-    q = queue.Queue()
-    pipe = send_http_query("WATCH test.lv")
-    thread = threading.Thread(target=read_lines_and_push_to_queue, args=(pipe, q))
-    thread.start()
-
-    line = q.get()
-    print(line)
-    assert line == "0\t1"
-
-    send_query("INSERT INTO test.mt VALUES (1),(2),(3)").read()
-    line = q.get()
-    print(line)
-    assert line == "6\t2"
-
-    send_query("DROP TABLE if exists test.lv").read()
-    send_query("DROP TABLE if exists test.lv").read()
-
-    thread.join()
-
-
-test()
diff --git a/tests/queries/0_stateless/00991_live_view_watch_http.reference b/tests/queries/0_stateless/00991_live_view_watch_http.reference
deleted file mode 100644
index 489457d751b..00000000000
--- a/tests/queries/0_stateless/00991_live_view_watch_http.reference
+++ /dev/null
@@ -1,4 +0,0 @@
-0	1
-0	1
-6	2
-6	2
diff --git a/tests/queries/0_stateless/01056_create_table_as.reference b/tests/queries/0_stateless/01056_create_table_as.reference
deleted file mode 100644
index c6e14cb675e..00000000000
--- a/tests/queries/0_stateless/01056_create_table_as.reference
+++ /dev/null
@@ -1 +0,0 @@
-1	String
diff --git a/tests/queries/0_stateless/01056_create_table_as.sql b/tests/queries/0_stateless/01056_create_table_as.sql
deleted file mode 100644
index 2e146d67ca9..00000000000
--- a/tests/queries/0_stateless/01056_create_table_as.sql
+++ /dev/null
@@ -1,57 +0,0 @@
--- Tags: no-parallel
-
-DROP TABLE IF EXISTS t1;
-DROP TABLE IF EXISTS t2;
-DROP TABLE IF EXISTS t3;
-DROP TABLE IF EXISTS v;
-DROP TABLE IF EXISTS lv;
-
-CREATE TABLE t1 (key Int) Engine=Memory;
-CREATE TABLE t2 AS t1;
-DROP TABLE t2;
-CREATE TABLE t2 Engine=Memory AS t1;
-DROP TABLE t2;
-CREATE TABLE t2 AS t1 Engine=Memory;
-DROP TABLE t2;
-CREATE TABLE t3 AS numbers(10);
-DROP TABLE t3;
-
--- live view
-SET allow_experimental_live_view=1;
-CREATE LIVE VIEW lv AS SELECT * FROM t1;
-CREATE TABLE t3 AS lv; -- { serverError 80 }
-DROP TABLE lv;
-
--- view
-CREATE VIEW v AS SELECT * FROM t1;
-CREATE TABLE t3 AS v; -- { serverError 80 }
-DROP TABLE v;
-
--- dictionary
-DROP DICTIONARY IF EXISTS dict;
-DROP DATABASE if exists test_01056_dict_data;
-CREATE DATABASE test_01056_dict_data;
-CREATE TABLE test_01056_dict_data.dict_data (key Int, value UInt16) Engine=Memory();
-CREATE DICTIONARY dict
-(
-    `key` UInt64,
-    `value` UInt16
-)
-PRIMARY KEY key
-SOURCE(CLICKHOUSE(
-    HOST '127.0.0.1' PORT tcpPort()
-    TABLE 'dict_data' DB 'test_01056_dict_data' USER 'default' PASSWORD ''))
-LIFETIME(MIN 0 MAX 0)
-LAYOUT(SPARSE_HASHED());
-CREATE TABLE t3 AS dict; -- { serverError 80 }
-
-DROP TABLE IF EXISTS t1;
-DROP TABLE IF EXISTS t3;
-DROP DICTIONARY dict;
-DROP TABLE test_01056_dict_data.dict_data;
-
-DROP DATABASE test_01056_dict_data;
-
-CREATE TABLE t1 (x String) ENGINE = Memory AS SELECT 1;
-SELECT x, toTypeName(x) FROM t1;
-DROP TABLE t1;
diff --git a/tests/queries/0_stateless/01071_live_view_detach_dependency.reference b/tests/queries/0_stateless/01071_live_view_detach_dependency.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/01071_live_view_detach_dependency.sql b/tests/queries/0_stateless/01071_live_view_detach_dependency.sql
deleted file mode 100644
index 1caab4ca240..00000000000
--- a/tests/queries/0_stateless/01071_live_view_detach_dependency.sql
+++ /dev/null
@@ -1,12 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-DROP TABLE IF EXISTS test;
-DROP TABLE IF EXISTS lv;
-CREATE TABLE test (n Int8) ENGINE = Memory;
-CREATE LIVE VIEW lv AS SELECT * FROM test;
-DETACH TABLE lv;
-INSERT INTO test VALUES (42);
-DROP TABLE test;
-ATTACH TABLE lv;
-DROP TABLE lv;
diff --git a/tests/queries/0_stateless/01190_full_attach_syntax.reference b/tests/queries/0_stateless/01190_full_attach_syntax.reference
deleted file mode 100644
index 61a0603975a..00000000000
--- a/tests/queries/0_stateless/01190_full_attach_syntax.reference
+++ /dev/null
@@ -1,13 +0,0 @@
-CREATE DICTIONARY test_01190.dict\n(\n    `key` UInt64 DEFAULT 0,\n    `col` UInt8 DEFAULT 1\n)\nPRIMARY KEY key\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' PASSWORD \'[HIDDEN]\' DB \'test_01190\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(FLAT())
-CREATE DICTIONARY test_01190.dict\n(\n    `key` UInt64 DEFAULT 0,\n    `col` UInt8 DEFAULT 1\n)\nPRIMARY KEY key\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' PASSWORD \'[HIDDEN]\' DB \'test_01190\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(FLAT())
-CREATE TABLE test_01190.log\n(\n    `s` String\n)\nENGINE = Log
-CREATE TABLE test_01190.log\n(\n    `s` String\n)\nENGINE = Log
-test
-CREATE TABLE test_01190.mt\n(\n    `key` Array(UInt8),\n    `s` String,\n    `n` UInt64,\n    `d` Date MATERIALIZED \'2000-01-01\'\n)\nENGINE = MergeTree(d, (key, s, n), 1)
-[1,2]	Hello	2
-CREATE TABLE test_01190.mt\n(\n    `key` Array(UInt8),\n    `s` String,\n    `n` UInt64,\n    `d` Date\n)\nENGINE = MergeTree(d, (key, s, n), 1)
-CREATE MATERIALIZED VIEW test_01190.mv\n(\n    `s` String\n)\nENGINE = Null AS\nSELECT *\nFROM test_01190.log
-CREATE MATERIALIZED VIEW test_01190.mv\n(\n    `s` String\n)\nENGINE = Null AS\nSELECT *\nFROM test_01190.log
-CREATE MATERIALIZED VIEW test_01190.mv\n(\n    `key` Array(UInt8),\n    `s` String,\n    `n` UInt64,\n    `d` Date\n)\nENGINE = Null AS\nSELECT *\nFROM test_01190.mt
-CREATE LIVE VIEW test_01190.lv\n(\n    `1` UInt8\n) AS\nSELECT 1
-CREATE LIVE VIEW test_01190.lv\n(\n    `1` UInt8\n) AS\nSELECT 1
diff --git a/tests/queries/0_stateless/01190_full_attach_syntax.sql b/tests/queries/0_stateless/01190_full_attach_syntax.sql
deleted file mode 100644
index e66978e22e1..00000000000
--- a/tests/queries/0_stateless/01190_full_attach_syntax.sql
+++ /dev/null
@@ -1,60 +0,0 @@
--- Tags: no-parallel
-
-DROP DATABASE IF EXISTS test_01190;
-set allow_deprecated_database_ordinary=1;
-set allow_deprecated_syntax_for_merge_tree=1;
-CREATE DATABASE test_01190 ENGINE=Ordinary;     -- Full ATTACH requires UUID with Atomic
-USE test_01190;
-
-CREATE TABLE test_01190.table_for_dict (key UInt64, col UInt8) ENGINE = Memory;
-
-CREATE DICTIONARY test_01190.dict (key UInt64 DEFAULT 0, col UInt8 DEFAULT 1) PRIMARY KEY key SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'test_01190')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());
-
-SHOW CREATE DICTIONARY test_01190.dict;
-
-DETACH DICTIONARY test_01190.dict;
-ATTACH TABLE test_01190.dict; -- { serverError 80 }
--- Full ATTACH syntax is not allowed for dictionaries
-ATTACH DICTIONARY test_01190.dict (key UInt64 DEFAULT 0, col UInt8 DEFAULT 42) PRIMARY KEY key SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'test_01190')) LIFETIME(MIN 1 MAX 100) LAYOUT(FLAT()); -- { clientError 62 }
-ATTACH DICTIONARY test_01190.dict;
-SHOW CREATE DICTIONARY test_01190.dict;
-
-CREATE TABLE log ENGINE = Log AS SELECT 'test' AS s;
-SHOW CREATE log;
-DETACH TABLE log;
-ATTACH DICTIONARY log; -- { serverError 80 }
-ATTACH TABLE log (s String) ENGINE = Log();
-SHOW CREATE log;
-SELECT * FROM log;
-
-DROP TABLE IF EXISTS mt;
-CREATE TABLE mt (key Array(UInt8), s String, n UInt64, d Date MATERIALIZED '2000-01-01') ENGINE = MergeTree(d, (key, s, n), 1);
-INSERT INTO mt VALUES ([1, 2], 'Hello', 2);
-DETACH TABLE mt;
-ATTACH TABLE mt (key Array(UInt8), s String, n UInt64, d Date MATERIALIZED '2000-01-01') ENGINE = MergeTree ORDER BY (key, s, n) PARTITION BY toYYYYMM(d); -- { serverError 342 }
-ATTACH TABLE mt (key Array(UInt8), s String, n UInt64, d Date MATERIALIZED '2000-01-01') ENGINE = MergeTree(d, (key, s, n), 1);
-SHOW CREATE mt;
-SELECT * FROM mt;
-DETACH TABLE mt;
-ATTACH TABLE mt (key Array(UInt8), s String, n UInt64, d Date) ENGINE = MergeTree(d, (key, s, n), 1);   -- It works (with Ordinary database), but probably it shouldn't
-SHOW CREATE mt;
-
-CREATE MATERIALIZED VIEW mv ENGINE = Null AS SELECT * FROM log;
-SHOW CREATE mv;
-DETACH VIEW mv;
-ATTACH MATERIALIZED VIEW mv ENGINE = Null AS SELECT * FROM log;
-SHOW CREATE mv;
-DETACH VIEW mv;
-ATTACH MATERIALIZED VIEW mv ENGINE = Null AS SELECT * FROM mt;  -- It works (with Ordinary database), but probably it shouldn't
-SHOW CREATE mv;
-
-SET allow_experimental_live_view = 1;
-CREATE LIVE VIEW lv AS SELECT 1;
-SHOW CREATE lv;
-DETACH VIEW lv;
-ATTACH LIVE VIEW lv AS SELECT 1;
-SHOW CREATE lv;
-
-DROP DATABASE test_01190;
-
-
diff --git a/tests/queries/0_stateless/01235_live_view_over_distributed.reference b/tests/queries/0_stateless/01235_live_view_over_distributed.reference
deleted file mode 100644
index 00fc99d96ba..00000000000
--- a/tests/queries/0_stateless/01235_live_view_over_distributed.reference
+++ /dev/null
@@ -1,4 +0,0 @@
-2020-01-01
-2020-01-01
-2020-01-02
-2020-01-02
diff --git a/tests/queries/0_stateless/01235_live_view_over_distributed.sql b/tests/queries/0_stateless/01235_live_view_over_distributed.sql
deleted file mode 100644
index 32ec0a6127f..00000000000
--- a/tests/queries/0_stateless/01235_live_view_over_distributed.sql
+++ /dev/null
@@ -1,23 +0,0 @@
--- Tags: distributed, no-replicated-database, no-parallel, no-fasttest
-
-set insert_distributed_sync = 1;
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS visits;
-DROP TABLE IF EXISTS visits_layer;
-
-CREATE TABLE visits(StartDate Date) ENGINE MergeTree ORDER BY(StartDate);
-CREATE TABLE visits_layer(StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost,  currentDatabase(), 'visits', rand());
-
-CREATE LIVE VIEW lv AS SELECT * FROM visits_layer ORDER BY StartDate;
-
-INSERT INTO visits_layer (StartDate) VALUES ('2020-01-01');
-INSERT INTO visits_layer (StartDate) VALUES ('2020-01-02');
-
-SELECT * FROM lv;
-
-DROP TABLE visits;
-DROP TABLE visits_layer;
-
-DROP TABLE lv;
diff --git a/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.reference b/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.reference
deleted file mode 100644
index dfb4d0552f5..00000000000
--- a/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.reference
+++ /dev/null
@@ -1,8 +0,0 @@
-2020-01-01
-2020-01-01
-2020-01-02
-2020-01-02
-2020-01-01
-2020-01-01
-2020-01-02
-2020-01-02
diff --git a/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql b/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql
deleted file mode 100644
index 3822f22d353..00000000000
--- a/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql
+++ /dev/null
@@ -1,25 +0,0 @@
--- Tags: distributed, no-replicated-database, no-parallel, no-fasttest, no-random-settings
-
-SET allow_experimental_live_view = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS visits;
-DROP TABLE IF EXISTS visits_layer;
-
-CREATE TABLE visits (StartDate Date) ENGINE MergeTree ORDER BY(StartDate);
-CREATE TABLE visits_layer (StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost,  currentDatabase(), 'visits', rand());
-
-CREATE LIVE VIEW lv AS SELECT * FROM visits_layer ORDER BY StartDate;
-
-CREATE TABLE visits_layer_lv (StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost,  currentDatabase(), 'lv', rand());
-
-INSERT INTO visits_layer (StartDate) VALUES ('2020-01-01');
-INSERT INTO visits_layer (StartDate) VALUES ('2020-01-02');
-
-SELECT * FROM visits_layer_lv;
-
-DROP TABLE visits;
-DROP TABLE visits_layer;
-
-DROP TABLE lv;
-DROP TABLE visits_layer_lv;
diff --git a/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.reference b/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.reference
deleted file mode 100644
index 00fc99d96ba..00000000000
--- a/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.reference
+++ /dev/null
@@ -1,4 +0,0 @@
-2020-01-01
-2020-01-01
-2020-01-02
-2020-01-02
diff --git a/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.sql b/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.sql
deleted file mode 100644
index b7c1c6eea76..00000000000
--- a/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.sql
+++ /dev/null
@@ -1,23 +0,0 @@
--- Tags: distributed, no-replicated-database, no-parallel, no-fasttest
-
-SET allow_experimental_live_view = 1;
-SET insert_distributed_sync = 1;
-
-DROP TABLE IF EXISTS lv;
-DROP TABLE IF EXISTS visits;
-DROP TABLE IF EXISTS visits_layer;
-
-CREATE TABLE visits(StartDate Date) ENGINE MergeTree ORDER BY(StartDate);
-CREATE TABLE visits_layer(StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost,  currentDatabase(), 'visits', rand());
-
-CREATE LIVE VIEW lv AS SELECT foo.x FROM (SELECT StartDate AS x FROM visits_layer) AS foo ORDER BY foo.x;
-
-INSERT INTO visits_layer (StartDate) VALUES ('2020-01-01');
-INSERT INTO visits_layer (StartDate) VALUES ('2020-01-02');
-
-SELECT * FROM lv;
-
-DROP TABLE visits;
-DROP TABLE visits_layer;
-
-DROP TABLE lv;
diff --git a/tests/queries/0_stateless/01246_insert_into_watch_live_view.py b/tests/queries/0_stateless/01246_insert_into_watch_live_view.py
deleted file mode 100755
index 02d03abc8c3..00000000000
--- a/tests/queries/0_stateless/01246_insert_into_watch_live_view.py
+++ /dev/null
@@ -1,87 +0,0 @@
-#!/usr/bin/env python3
-# Tags: no-replicated-database, no-parallel, no-fasttest
-
-import os
-import sys
-import time
-import signal
-
-CURDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(CURDIR, "helpers"))
-
-from client import client, prompt, end_of_block
-
-log = None
-# uncomment the line below for debugging
-# log=sys.stdout
-
-with client(name="client1>", log=log) as client1, client(
-    name="client2>", log=log
-) as client2, client(name="client3>", log=log) as client3:
-    client1.expect(prompt)
-    client2.expect(prompt)
-    client3.expect(prompt)
-
-    client1.send("SET allow_experimental_live_view = 1")
-    client1.expect(prompt)
-    client3.send("SET allow_experimental_live_view = 1")
-    client3.expect(prompt)
-
-    client1.send("DROP TABLE IF EXISTS test.lv")
-    client1.expect(prompt)
-    client1.send("DROP TABLE IF EXISTS test.lv_sums")
-    client1.expect(prompt)
-    client1.send("DROP TABLE IF EXISTS test.mt")
-    client1.expect(prompt)
-    client1.send("DROP TABLE IF EXISTS test.sums")
-    client1.expect(prompt)
-    client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()")
-    client1.expect(prompt)
-    client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) AS s FROM test.mt")
-    client1.expect(prompt)
-    client1.send(
-        "CREATE TABLE test.sums (s Int32, version Int32) Engine=MergeTree ORDER BY tuple()"
-    )
-    client1.expect(prompt)
-    client3.send(
-        "CREATE LIVE VIEW test.lv_sums AS SELECT * FROM test.sums ORDER BY version"
-    )
-    client3.expect(prompt)
-
-    client3.send("WATCH test.lv_sums FORMAT CSVWithNames")
-
-    client1.send("INSERT INTO test.sums WATCH test.lv")
-    client1.expect(r"INSERT INTO")
-
-    client3.expect("0,1.*\n")
-
-    client2.send("INSERT INTO test.mt VALUES (1),(2),(3)")
-    client2.expect(prompt)
-    client3.expect("6,2.*\n")
-
-    client2.send("INSERT INTO test.mt VALUES (4),(5),(6)")
-    client2.expect(prompt)
-    client3.expect("21,3.*\n")
-
-    # send Ctrl-C
-    client3.send("\x03", eol="")
-    match = client3.expect("(%s)|([#\$] )" % prompt)
-    if match.groups()[1]:
-        client3.send(client3.command)
-        client3.expect(prompt)
-
-    # send Ctrl-C
-    client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
-    if match.groups()[1]:
-        client1.send(client1.command)
-        client1.expect(prompt)
-
-    client2.send("DROP TABLE test.lv")
-    client2.expect(prompt)
-    client2.send("DROP TABLE test.lv_sums")
-    client2.expect(prompt)
-    client2.send("DROP TABLE test.sums")
-    client2.expect(prompt)
-    client2.send("DROP TABLE test.mt")
-    client2.expect(prompt)
diff --git a/tests/queries/0_stateless/01246_insert_into_watch_live_view.reference b/tests/queries/0_stateless/01246_insert_into_watch_live_view.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/01463_test_alter_live_view_refresh.reference b/tests/queries/0_stateless/01463_test_alter_live_view_refresh.reference
deleted file mode 100644
index 4d98c7b6838..00000000000
--- a/tests/queries/0_stateless/01463_test_alter_live_view_refresh.reference
+++ /dev/null
@@ -1 +0,0 @@
-ALTER LIVE VIEW live1 REFRESH
diff --git a/tests/queries/0_stateless/01463_test_alter_live_view_refresh.sql b/tests/queries/0_stateless/01463_test_alter_live_view_refresh.sql
deleted file mode 100644
index 3c855c984b3..00000000000
--- a/tests/queries/0_stateless/01463_test_alter_live_view_refresh.sql
+++ /dev/null
@@ -1,15 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-fasttest
-
-CREATE TABLE test0 (
-        c0 UInt64
-    ) ENGINE = MergeTree() PARTITION BY c0 ORDER BY c0;
-
-SET allow_experimental_live_view=1;
-
-CREATE LIVE VIEW live1 AS SELECT * FROM test0;
-
-select 'ALTER LIVE VIEW live1 REFRESH';
-ALTER LIVE VIEW live1 REFRESH; -- success
-
-DROP TABLE test0;
-DROP VIEW live1;
diff --git a/tests/queries/0_stateless/01602_show_create_view.reference b/tests/queries/0_stateless/01602_show_create_view.reference
deleted file mode 100644
index 5d4bd2cd972..00000000000
--- a/tests/queries/0_stateless/01602_show_create_view.reference
+++ /dev/null
@@ -1,7 +0,0 @@
-CREATE VIEW test_1602.v\n(\n    `EventDate` DateTime,\n    `CounterID` UInt32,\n    `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl
-CREATE MATERIALIZED VIEW test_1602.vv\n(\n    `EventDate` DateTime,\n    `CounterID` UInt32,\n    `UserID` UInt32\n)\nENGINE = MergeTree\nPARTITION BY toYYYYMM(EventDate)\nORDER BY (CounterID, EventDate, intHash32(UserID))\nSETTINGS index_granularity = 8192 AS\nSELECT *\nFROM test_1602.tbl
-CREATE LIVE VIEW test_1602.vvv\n(\n    `EventDate` DateTime,\n    `CounterID` UInt32,\n    `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl
-CREATE VIEW test_1602.VIEW\n(\n    `EventDate` DateTime,\n    `CounterID` UInt32,\n    `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl
-CREATE VIEW test_1602.DATABASE\n(\n    `EventDate` DateTime,\n    `CounterID` UInt32,\n    `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl
-CREATE VIEW test_1602.DICTIONARY\n(\n    `EventDate` DateTime,\n    `CounterID` UInt32,\n    `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl
-CREATE VIEW test_1602.TABLE\n(\n    `EventDate` DateTime,\n    `CounterID` UInt32,\n    `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl
diff --git a/tests/queries/0_stateless/01602_show_create_view.sql b/tests/queries/0_stateless/01602_show_create_view.sql
deleted file mode 100644
index 11b9c235290..00000000000
--- a/tests/queries/0_stateless/01602_show_create_view.sql
+++ /dev/null
@@ -1,53 +0,0 @@
--- Tags: no-parallel
-
-DROP DATABASE IF EXISTS test_1602;
-
-CREATE DATABASE test_1602;
-
-CREATE TABLE test_1602.tbl (`EventDate` DateTime, `CounterID` UInt32, `UserID` UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SETTINGS index_granularity = 8192;
-
-CREATE VIEW test_1602.v AS SELECT * FROM test_1602.tbl; 
-
-CREATE VIEW test_1602.DATABASE AS SELECT * FROM test_1602.tbl; 
-
-CREATE VIEW test_1602.DICTIONARY AS SELECT * FROM test_1602.tbl; 
-
-CREATE VIEW test_1602.TABLE AS SELECT * FROM test_1602.tbl; 
-
-CREATE MATERIALIZED VIEW test_1602.vv (`EventDate` DateTime, `CounterID` UInt32, `UserID` UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SETTINGS index_granularity = 8192 AS SELECT * FROM test_1602.tbl;
-
-CREATE VIEW test_1602.VIEW AS SELECT * FROM test_1602.tbl; 
-
-SET allow_experimental_live_view=1;
-
-CREATE LIVE VIEW test_1602.vvv AS SELECT * FROM test_1602.tbl;
-
-SHOW CREATE VIEW test_1602.v;
-
-SHOW CREATE VIEW test_1602.vv;
-
-SHOW CREATE VIEW test_1602.vvv;
-
-SHOW CREATE VIEW test_1602.not_exist_view; -- { serverError 390 }
-
-SHOW CREATE VIEW test_1602.tbl; -- { serverError 36 }
-
-SHOW CREATE TEMPORARY VIEW; -- { serverError 60 }
-
-SHOW CREATE VIEW; -- { clientError 62 }
-
-SHOW CREATE DATABASE; -- { clientError 62 }
-
-SHOW CREATE DICTIONARY; -- { clientError 62 }
-
-SHOW CREATE TABLE; -- { clientError 62 }
-
-SHOW CREATE test_1602.VIEW;
-
-SHOW CREATE test_1602.DATABASE;
-
-SHOW CREATE test_1602.DICTIONARY;
-
-SHOW CREATE test_1602.TABLE;
-
-DROP DATABASE IF EXISTS test_1602;
diff --git a/tests/queries/0_stateless/01927_query_views_log_current_database.reference b/tests/queries/0_stateless/01927_query_views_log_current_database.reference
deleted file mode 100644
index eaa1e98c55c..00000000000
--- a/tests/queries/0_stateless/01927_query_views_log_current_database.reference
+++ /dev/null
@@ -1,94 +0,0 @@
-Row 1:
-──────
-stage:                Query log rows
-read_rows:            400
-written_rows:         201
-databases:            ['_table_function','default']
-tables:               ['_table_function.numbers','default.table_a','default.table_b','default.table_b_live_view','default.table_c']
-views:                ['default.matview_a_to_b','default.matview_b_to_c','default.table_b_live_view']
-sleep_calls:          200
-sleep_us:             298
-profile_select_rows:  400
-profile_select_bytes: 5200
-profile_insert_rows:  201
-profile_insert_bytes: 2808
-Row 1:
-──────
-stage:                Depending views
-view_name:            default.matview_a_to_b
-view_type:            Materialized
-status:               QueryFinish
-view_target:          default.table_b
-view_query:           SELECT toFloat64(a) AS a, b + sleepEachRow(0.000001) AS count FROM default.table_a
-read_rows:            100
-written_rows:         100
-sleep_calls:          100
-sleep_us:             99
-profile_select_rows:  100
-profile_select_bytes: 2000
-profile_insert_rows:  100
-profile_insert_bytes: 800
-
-Row 2:
-──────
-stage:                Depending views
-view_name:            default.matview_b_to_c
-view_type:            Materialized
-status:               QueryFinish
-view_target:          default.table_c
-view_query:           SELECT sum(a + sleepEachRow(0.000002)) AS a FROM default.table_b
-read_rows:            100
-written_rows:         1
-sleep_calls:          100
-sleep_us:             199
-profile_select_rows:  100
-profile_select_bytes: 800
-profile_insert_rows:  1
-profile_insert_bytes: 8
-
-Row 3:
-──────
-stage:                Depending views
-view_name:            default.table_b_live_view
-view_type:            Live
-status:               QueryFinish
-view_target:          default.table_b_live_view
-view_query:           SELECT sum(a + b) FROM default.table_b
-read_rows:            100
-written_rows:         0
-sleep_calls:          0
-sleep_us:             0
-profile_select_rows:  100
-profile_select_bytes: 1600
-profile_insert_rows:  0
-profile_insert_bytes: 0
-Row 1:
-──────
-stage:                Query log rows 2
-read_rows:            100
-written_rows:         100
-databases:            ['_table_function','default']
-tables:               ['_table_function.numbers','default.table_d','default.table_e','default.table_f']
-views:                ['default.matview_join_d_e']
-sleep_calls:          50
-sleep_us:             150
-profile_select_rows:  100
-profile_select_bytes: 800
-profile_insert_rows:  100
-profile_insert_bytes: 1600
-Row 1:
-──────
-stage:                Depending views 2
-view_name:            default.matview_join_d_e
-view_type:            Materialized
-status:               QueryFinish
-view_target:          default.table_f
-view_query:           SELECT table_d.a AS a, table_e.count + sleepEachRow(0.000003) AS count FROM default.table_d LEFT JOIN default.table_e ON table_d.a = table_e.a
-read_rows:            50
-written_rows:         50
-sleep_calls:          50
-sleep_us:             150
-profile_select_rows:  50
-profile_select_bytes: 400
-profile_insert_rows:  50
-profile_insert_bytes: 800
diff --git a/tests/queries/0_stateless/01927_query_views_log_current_database.sql b/tests/queries/0_stateless/01927_query_views_log_current_database.sql
deleted file mode 100644
index fbfbeab0167..00000000000
--- a/tests/queries/0_stateless/01927_query_views_log_current_database.sql
+++ /dev/null
@@ -1,145 +0,0 @@
-SET allow_experimental_live_view = 1;
-SET log_queries=0;
-SET log_query_threads=0;
-
--- SETUP TABLES
-CREATE TABLE table_a (a String, b Int64) ENGINE = MergeTree ORDER BY b;
-CREATE TABLE table_b (a Float64,  b Int64) ENGINE = MergeTree ORDER BY tuple();
-CREATE TABLE table_c (a Float64) ENGINE = MergeTree ORDER BY a;
-
-CREATE TABLE table_d (a Float64, count Int64) ENGINE MergeTree ORDER BY a;
-CREATE TABLE table_e (a Float64, count Int64) ENGINE MergeTree ORDER BY a;
-CREATE TABLE table_f (a Float64, count Int64) ENGINE MergeTree ORDER BY a;
-
--- SETUP MATERIALIZED VIEWS
-CREATE MATERIALIZED VIEW matview_a_to_b TO table_b AS SELECT toFloat64(a) AS a, b + sleepEachRow(0.000001) AS count FROM table_a;
-CREATE MATERIALIZED VIEW matview_b_to_c TO table_c AS SELECT SUM(a + sleepEachRow(0.000002)) as a FROM table_b;
-CREATE MATERIALIZED VIEW matview_join_d_e TO table_f AS SELECT table_d.a as a, table_e.count + sleepEachRow(0.000003) as count FROM table_d LEFT JOIN table_e ON table_d.a = table_e.a;
-
--- SETUP LIVE VIEW
----- table_b_live_view (Int64)
-DROP TABLE IF EXISTS table_b_live_view;
-CREATE LIVE VIEW table_b_live_view AS SELECT sum(a + b) FROM table_b;
-
--- ENABLE LOGS
-SET log_query_views=1;
-SET log_queries_min_type='QUERY_FINISH';
-SET log_queries=1;
-
--- INSERT 1
-INSERT INTO table_a SELECT '111', * FROM numbers(100);
-
--- INSERT 2
-INSERT INTO table_d SELECT 0.5, * FROM numbers(50);
-
-SYSTEM FLUSH LOGS;
-
-
--- CHECK LOGS OF INSERT 1
--- Note that live views currently don't report written rows
-SELECT
-    'Query log rows' as stage,
-    read_rows,
-    written_rows,
-    arraySort(databases) as databases,
-    arraySort(tables) as tables,
-    arraySort(views) as views,
-    ProfileEvents['SleepFunctionCalls'] as sleep_calls,
-    ProfileEvents['SleepFunctionMicroseconds'] as sleep_us,
-    ProfileEvents['SelectedRows'] as profile_select_rows,
-    ProfileEvents['SelectedBytes'] as profile_select_bytes,
-    ProfileEvents['InsertedRows'] as profile_insert_rows,
-    ProfileEvents['InsertedBytes'] as profile_insert_bytes
-FROM system.query_log
-WHERE query like '-- INSERT 1%INSERT INTO table_a%'
-  AND current_database = currentDatabase()
-  AND event_date >= yesterday()
-FORMAT Vertical;
-
-SELECT
-    'Depending views' as stage,
-    view_name,
-    view_type,
-    status,
-    view_target,
-    view_query,
-    read_rows,
-    written_rows,
-    ProfileEvents['SleepFunctionCalls'] as sleep_calls,
-    ProfileEvents['SleepFunctionMicroseconds'] as sleep_us,
-    ProfileEvents['SelectedRows'] as profile_select_rows,
-    ProfileEvents['SelectedBytes'] as profile_select_bytes,
-    ProfileEvents['InsertedRows'] as profile_insert_rows,
-    ProfileEvents['InsertedBytes'] as profile_insert_bytes
-FROM system.query_views_log
-WHERE initial_query_id =
-      (
-          SELECT initial_query_id
-          FROM system.query_log
-          WHERE query like '-- INSERT 1%INSERT INTO table_a%'
-            AND current_database = currentDatabase()
-            AND event_date >= yesterday()
-          LIMIT 1
-      )
-ORDER BY view_name
-FORMAT Vertical;
-
--- CHECK LOGS OF INSERT 2
-SELECT
-    'Query log rows 2' as stage,
-    read_rows,
-    written_rows,
-    arraySort(databases) as databases,
-    arraySort(tables) as tables,
-    arraySort(views) as views,
-    ProfileEvents['SleepFunctionCalls'] as sleep_calls,
-    ProfileEvents['SleepFunctionMicroseconds'] as sleep_us,
-    ProfileEvents['SelectedRows'] as profile_select_rows,
-    ProfileEvents['SelectedBytes'] as profile_select_bytes,
-    ProfileEvents['InsertedRows'] as profile_insert_rows,
-    ProfileEvents['InsertedBytes'] as profile_insert_bytes
-FROM system.query_log
-WHERE query like '-- INSERT 2%INSERT INTO table_d%'
-  AND current_database = currentDatabase()
-  AND event_date >= yesterday()
-FORMAT Vertical;
-
-SELECT
-    'Depending views 2' as stage,
-    view_name,
-    view_type,
-    status,
-    view_target,
-    view_query,
-    read_rows,
-    written_rows,
-    ProfileEvents['SleepFunctionCalls'] as sleep_calls,
-    ProfileEvents['SleepFunctionMicroseconds'] as sleep_us,
-    ProfileEvents['SelectedRows'] as profile_select_rows,
-    ProfileEvents['SelectedBytes'] as profile_select_bytes,
-    ProfileEvents['InsertedRows'] as profile_insert_rows,
-    ProfileEvents['InsertedBytes'] as profile_insert_bytes
-FROM system.query_views_log
-WHERE initial_query_id =
-      (
-          SELECT initial_query_id
-          FROM system.query_log
-          WHERE query like '-- INSERT 2%INSERT INTO table_d%'
-            AND current_database = currentDatabase()
-            AND event_date >= yesterday()
-          LIMIT 1
-      )
-ORDER BY view_name
-FORMAT Vertical;
-
--- TEARDOWN
-DROP TABLE table_b_live_view;
-DROP TABLE matview_a_to_b;
-DROP TABLE matview_b_to_c;
-DROP TABLE matview_join_d_e;
-DROP TABLE table_f;
-DROP TABLE table_e;
-DROP TABLE table_d;
-DROP TABLE table_c;
-DROP TABLE table_b;
-DROP TABLE table_a;
diff --git a/tests/queries/0_stateless/02048_views_with_comment.reference b/tests/queries/0_stateless/02048_views_with_comment.reference
deleted file mode 100644
index ad9817ad120..00000000000
--- a/tests/queries/0_stateless/02048_views_with_comment.reference
+++ /dev/null
@@ -1,3 +0,0 @@
-live_view_comment_test	LiveView	live view
-materialized_view_comment_test	MaterializedView	materialized view
-view_comment_test	View	simple view
diff --git a/tests/queries/0_stateless/02048_views_with_comment.sql b/tests/queries/0_stateless/02048_views_with_comment.sql
deleted file mode 100644
index a7c991d119b..00000000000
--- a/tests/queries/0_stateless/02048_views_with_comment.sql
+++ /dev/null
@@ -1,12 +0,0 @@
--- Make sure that any kind of `VIEW` can be created with a `COMMENT` clause
--- and value of that clause is visible as `comment` column of `system.tables` table.
-
-CREATE VIEW view_comment_test AS (SELECT 1) COMMENT 'simple view';
-CREATE MATERIALIZED VIEW materialized_view_comment_test TO test1 (a UInt64) AS (SELECT 1) COMMENT 'materialized view';
-
-SET allow_experimental_live_view=1;
-CREATE LIVE VIEW live_view_comment_test AS (SELECT 1) COMMENT 'live view';
-
-SYSTEM FLUSH LOGS;
-
-SELECT name, engine, comment FROM system.tables WHERE database == currentDatabase() ORDER BY name;

From e43d353fba082a1d3e396060d31b35642558b18a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 02:19:47 +0200
Subject: [PATCH 213/689] Fix wrong assert

---
 .../queries/0_stateless/02731_nothing_deserialization.reference  | 0
 tests/queries/0_stateless/02731_nothing_deserialization.sql      | 1 +
 2 files changed, 1 insertion(+)
 create mode 100644 tests/queries/0_stateless/02731_nothing_deserialization.reference
 create mode 100644 tests/queries/0_stateless/02731_nothing_deserialization.sql

diff --git a/tests/queries/0_stateless/02731_nothing_deserialization.reference b/tests/queries/0_stateless/02731_nothing_deserialization.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/02731_nothing_deserialization.sql b/tests/queries/0_stateless/02731_nothing_deserialization.sql
new file mode 100644
index 00000000000..7526bce3578
--- /dev/null
+++ b/tests/queries/0_stateless/02731_nothing_deserialization.sql
@@ -0,0 +1 @@
+SELECT CAST('\x01\x00' AS AggregateFunction(nothingArrayIf, Array(Nullable(Nothing)), Nullable(Nothing))); -- { serverError INCORRECT_DATA }

From e9912e2f7e7231f07b29ac865af59b5e22d8dec3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 02:19:58 +0200
Subject: [PATCH 214/689] Fix wrong assert

---
 src/AggregateFunctions/AggregateFunctionNothing.h | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/AggregateFunctions/AggregateFunctionNothing.h b/src/AggregateFunctions/AggregateFunctionNothing.h
index 8491e8edfaa..8280d72c085 100644
--- a/src/AggregateFunctions/AggregateFunctionNothing.h
+++ b/src/AggregateFunctions/AggregateFunctionNothing.h
@@ -13,6 +13,11 @@ namespace DB
 {
 struct Settings;
 
+namespace ErrorCodes
+{
+    extern const int INCORRECT_DATA;
+}
+
 
 /** Aggregate function that takes arbitrary number of arbitrary arguments and does nothing.
   */
@@ -69,7 +74,8 @@ public:
     {
         [[maybe_unused]] char symbol;
         readChar(symbol, buf);
-        assert(symbol == '\0');
+        if (symbol != '\0')
+            throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect state of aggregate function 'nothing', it should contain exactly one zero byte.");
     }
 
     void insertResultInto(AggregateDataPtr __restrict, IColumn & to, Arena *) const override

From f302c1986a186d92437fbcbdc0fe88f1a5aedc3e Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Sat, 6 May 2023 02:56:06 +0200
Subject: [PATCH 215/689] use flag has_exclusive_blobs to track the part's
 originate

---
 src/Storages/MergeTree/DataPartsExchange.cpp |  1 +
 src/Storages/MergeTree/IMergeTreeDataPart.h  |  2 ++
 src/Storages/MergeTree/MergeTask.cpp         |  1 +
 src/Storages/MergeTree/MergeTreeData.cpp     |  1 +
 src/Storages/MergeTree/MutateTask.cpp        |  4 ++++
 src/Storages/StorageReplicatedMergeTree.cpp  | 16 ++++++++--------
 src/Storages/StorageReplicatedMergeTree.h    |  2 +-
 7 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp
index 46c6d09eca4..7b6f8036c48 100644
--- a/src/Storages/MergeTree/DataPartsExchange.cpp
+++ b/src/Storages/MergeTree/DataPartsExchange.cpp
@@ -918,6 +918,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk(
 
         new_data_part->version.setCreationTID(Tx::PrehistoricTID, nullptr);
         new_data_part->is_temp = true;
+        new_data_part->has_exclusive_blobs = false;
         new_data_part->modification_time = time(nullptr);
         new_data_part->loadColumnsChecksumsIndexes(true, false);
     }
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h
index a36634d2cf9..61048d8e257 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.h
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.h
@@ -218,6 +218,8 @@ public:
     /// FIXME Why do we need this flag? What's difference from Temporary and DeleteOnDestroy state? Can we get rid of this?
     bool is_temp = false;
 
+    std::optional<bool> has_exclusive_blobs = std::nullopt;
+
     /// If true it means that there are no ZooKeeper node for this part, so it should be deleted only from filesystem
     bool is_duplicate = false;
 
diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp
index df759b3bd45..2905ab68ef3 100644
--- a/src/Storages/MergeTree/MergeTask.cpp
+++ b/src/Storages/MergeTree/MergeTask.cpp
@@ -209,6 +209,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare()
     global_ctx->new_data_part->uuid = global_ctx->future_part->uuid;
     global_ctx->new_data_part->partition.assign(global_ctx->future_part->getPartition());
     global_ctx->new_data_part->is_temp = global_ctx->parent_part == nullptr;
+    global_ctx->new_data_part->has_exclusive_blobs = true;
 
     ctx->need_remove_expired_values = false;
     ctx->force_ttl = false;
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 83f5c0d359c..f229a19b45c 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -8233,6 +8233,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createEmptyPart(
 
     new_data_part->minmax_idx = std::move(minmax_idx);
     new_data_part->is_temp = true;
+    new_data_part->has_exclusive_blobs = true;
 
     auto new_data_part_storage = new_data_part->getDataPartStoragePtr();
     new_data_part_storage->beginTransaction();
diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp
index a3fa210ac42..ac0f0e6d0a9 100644
--- a/src/Storages/MergeTree/MutateTask.cpp
+++ b/src/Storages/MergeTree/MutateTask.cpp
@@ -1838,6 +1838,8 @@ bool MutateTask::prepare()
     if (!isWidePart(ctx->source_part) || !isFullPartStorage(ctx->source_part->getDataPartStorage())
         || (ctx->interpreter && ctx->interpreter->isAffectingAllColumns()))
     {
+        ctx->new_data_part->has_exclusive_blobs = true;
+
         task = std::make_unique<MutateAllPartColumnsTask>(ctx);
     }
     else /// TODO: check that we modify only non-key columns in this case.
@@ -1865,6 +1867,8 @@ bool MutateTask::prepare()
             ctx->for_file_renames,
             ctx->mrk_extension);
 
+        ctx->new_data_part->has_exclusive_blobs = std::nullopt; // that means that unlocking in zookeper needed
+
         task = std::make_unique<MutateSomePartColumnsTask>(ctx);
     }
 
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 1c049991100..e7cee59bda8 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -8263,7 +8263,7 @@ StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, co
         return std::make_pair(true, NameSet{});
     }
 
-    if (part.getState() == MergeTreeDataPartState::Temporary && part.is_temp)
+    if (part.getState() == MergeTreeDataPartState::Temporary || part.getState() == MergeTreeDataPartState::Outdated)
     {
         /// Part {} is in temporary state and has it_temp flag. it means that it is under construction.
         /// That path hasn't been added to active set, no commit procedure has begun.
@@ -8274,16 +8274,16 @@ StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, co
         /// When the part has been merged then remote data can be removed, part owns it.
         /// In opposition, when the part has been mutated in generally it hardlinks the files.
         /// Therefore remote data is shared, it has to be unlocked in the keper.
+        /// In order to track all that cases has_exclusive_blobs optional value is used
+        /// If has_exclusive_blobs is determined, then CH traked the parts origin from the creation
 
-        std::lock_guard lock(currently_fetching_parts_mutex);
-        if (!currently_fetching_parts.contains(part.name))
+        if (part.has_exclusive_blobs.has_value())
         {
-            LOG_TRACE(log, "Part {} is temporary result of failed fetch."
-                           "Remove it without unlocking shared state preserving shared data.", part.name);
-            return std::make_pair(false, NameSet{});
+            LOG_INFO(log, "Looks like CH knows the origin of that part. "
+                          "Part {} can be deleted without unlocking shared data in zookeeper. "
+                          "Part data is {}", part.name, part.has_exclusive_blobs.value() ? "exclusive" : "shared");
+            return std::make_pair(part.has_exclusive_blobs.value(), NameSet{});
         }
-
-        /// The other cases (merge/mutate) is harder to distinguish. Fall back to unlock part in the keper.
     }
 
     if (has_metadata_in_zookeeper.has_value() && !has_metadata_in_zookeeper)
diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h
index 77ab976d0fd..ade4e4f0b4b 100644
--- a/src/Storages/StorageReplicatedMergeTree.h
+++ b/src/Storages/StorageReplicatedMergeTree.h
@@ -703,7 +703,7 @@ private:
 
     /// Required only to avoid races between executeLogEntry and fetchPartition
     std::unordered_set<String> currently_fetching_parts;
-    mutable std::mutex currently_fetching_parts_mutex;
+    std::mutex currently_fetching_parts_mutex;
 
     /// With the quorum being tracked, add a replica to the quorum for the part.
     void updateQuorum(const String & part_name, bool is_parallel);

From d6448b749e85476bedf0efeaa736ce00391cf951 Mon Sep 17 00:00:00 2001
From: merlllle <merle.xzhang@gmail.com>
Date: Sat, 6 May 2023 02:17:19 +0100
Subject: [PATCH 216/689] Fix missing backticks for inline code in
 architecture.md

---
 docs/en/development/architecture.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md
index da7f95ac207..ba81b31b8ef 100644
--- a/docs/en/development/architecture.md
+++ b/docs/en/development/architecture.md
@@ -98,7 +98,7 @@ A hand-written recursive descent parser parses a query. For example, `ParserSele
 
 ## Interpreters {#interpreters}
 
-Interpreters are responsible for creating the query execution pipeline from an `AST`. There are simple interpreters, such as `InterpreterExistsQuery` and `InterpreterDropQuery`, or the more sophisticated `InterpreterSelectQuery`. The query execution pipeline is a combination of block input or output streams. For example, the result of interpreting the `SELECT` query is the `IBlockInputStream` to read the result set from; the result of the INSERT query is the `IBlockOutputStream` to write data for insertion to, and the result of interpreting the `INSERT SELECT` query is the `IBlockInputStream` that returns an empty result set on the first read, but that copies data from `SELECT` to `INSERT` at the same time.
+Interpreters are responsible for creating the query execution pipeline from an `AST`. There are simple interpreters, such as `InterpreterExistsQuery` and `InterpreterDropQuery`, or the more sophisticated `InterpreterSelectQuery`. The query execution pipeline is a combination of block input or output streams. For example, the result of interpreting the `SELECT` query is the `IBlockInputStream` to read the result set from; the result of the `INSERT` query is the `IBlockOutputStream` to write data for insertion to, and the result of interpreting the `INSERT SELECT` query is the `IBlockInputStream` that returns an empty result set on the first read, but that copies data from `SELECT` to `INSERT` at the same time.
 
 `InterpreterSelectQuery` uses `ExpressionAnalyzer` and `ExpressionActions` machinery for query analysis and transformations. This is where most rule-based query optimizations are done. `ExpressionAnalyzer` is quite messy and should be rewritten: various query transformations and optimizations should be extracted to separate classes to allow modular transformations of query.
 

From 576fbb9c25da848650feba395bf1a3b49a97d887 Mon Sep 17 00:00:00 2001
From: jinjunzh <jasper.zhu@intel.com>
Date: Sat, 6 May 2023 10:38:11 -0400
Subject: [PATCH 217/689] fixed symbol conflict between contrib/isal and isal
 in qpl

---
 contrib/qpl-cmake/CMakeLists.txt               | 9 +++++++--
 src/Compression/CompressionCodecDeflateQpl.cpp | 2 +-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt
index 334731d105f..d2be8add3c7 100644
--- a/contrib/qpl-cmake/CMakeLists.txt
+++ b/contrib/qpl-cmake/CMakeLists.txt
@@ -187,7 +187,9 @@ target_include_directories(qplcore_avx512
 set_target_properties(qplcore_avx512 PROPERTIES
         $<$<C_COMPILER_ID:GNU>:C_STANDARD 17>)
 
-target_link_libraries(qplcore_avx512 ${CMAKE_DL_LIBS} isal)
+target_link_libraries(qplcore_avx512
+        PRIVATE isal
+        PRIVATE ${CMAKE_DL_LIBS})
 
 target_compile_options(qplcore_avx512
                         PRIVATE ${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}
@@ -217,7 +219,9 @@ target_include_directories(qplcore_px
 set_target_properties(qplcore_px PROPERTIES
         $<$<C_COMPILER_ID:GNU>:C_STANDARD 17>)
 
-target_link_libraries(qplcore_px isal ${CMAKE_DL_LIBS})
+target_link_libraries(qplcore_px
+        PRIVATE isal
+        PRIVATE ${CMAKE_DL_LIBS})
 
 target_compile_options(qplcore_px
                         PRIVATE ${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}
@@ -339,6 +343,7 @@ target_compile_definitions(_qpl
 
 target_link_libraries(_qpl
         PRIVATE accel-config
+        PRIVATE ch_contrib::isal
         PRIVATE ${CMAKE_DL_LIBS})
 
 add_library (ch_contrib::qpl ALIAS _qpl)
diff --git a/src/Compression/CompressionCodecDeflateQpl.cpp b/src/Compression/CompressionCodecDeflateQpl.cpp
index 70ae8e54d97..60efbd96f8f 100644
--- a/src/Compression/CompressionCodecDeflateQpl.cpp
+++ b/src/Compression/CompressionCodecDeflateQpl.cpp
@@ -96,7 +96,7 @@ qpl_job * DeflateQplJobHWPool::acquireJob(UInt32 & job_id)
     if (isJobPoolReady())
     {
         UInt32 retry = 0;
-        auto index = distribution(random_engine);
+        UInt32 index = distribution(random_engine);
         while (!tryLockJob(index))
         {
             index = distribution(random_engine);

From 04a829b554d927bbb599b987e3edfa07de8238ab Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Fri, 5 May 2023 22:41:44 -0400
Subject: [PATCH 218/689] Fix PR issues

---
 src/Access/UsersConfigAccessStorage.cpp | 12 +++++++-----
 src/Parsers/Access/ParserGrantQuery.h   |  2 +-
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp
index a1840b8451c..df0e4584709 100644
--- a/src/Access/UsersConfigAccessStorage.cpp
+++ b/src/Access/UsersConfigAccessStorage.cpp
@@ -231,8 +231,13 @@ namespace
             }
         }
 
-        if (grant_queries && databases)
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Both `grants` and `allow_databases` can't be specified");
+        bool access_management = config.getBool(user_config + ".access_management", false);
+        bool named_collection_control = config.getBool(user_config + ".named_collection_control", false);
+        bool show_named_collections_secrets = config.getBool(user_config + ".show_named_collections_secrets", false);
+
+        if (grant_queries)
+            if (databases || dictionaries || access_management || named_collection_control || show_named_collections_secrets)
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Any other access control settings can't be specified with `grants`");
 
         if (grant_queries)
         {
@@ -289,20 +294,17 @@ namespace
                     user->access.grantWithGrantOption(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG, dictionary);
             }
 
-            bool access_management = config.getBool(user_config + ".access_management", false);
             if (!access_management)
             {
                 user->access.revoke(AccessType::ACCESS_MANAGEMENT);
                 user->access.revokeGrantOption(AccessType::ALL);
             }
 
-            bool named_collection_control = config.getBool(user_config + ".named_collection_control", false);
             if (!named_collection_control)
             {
                 user->access.revoke(AccessType::NAMED_COLLECTION_CONTROL);
             }
 
-            bool show_named_collections_secrets = config.getBool(user_config + ".show_named_collections_secrets", false);
             if (!show_named_collections_secrets)
             {
                 user->access.revoke(AccessType::SHOW_NAMED_COLLECTIONS_SECRETS);
diff --git a/src/Parsers/Access/ParserGrantQuery.h b/src/Parsers/Access/ParserGrantQuery.h
index 97a8510bc0a..58c2be433d5 100644
--- a/src/Parsers/Access/ParserGrantQuery.h
+++ b/src/Parsers/Access/ParserGrantQuery.h
@@ -22,6 +22,6 @@ protected:
 
 private:
     bool attach_mode = false;
-    bool allow_no_grantees = false;
+    bool allow_no_grantees = false;  /// if set, the parser will allow grant queries without any grantees.
 };
 }

From e1fa147e512e04ce5dd8e374916a59d90421a4f5 Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Fri, 5 May 2023 22:57:48 -0400
Subject: [PATCH 219/689] docs

---
 docs/en/operations/settings/settings-users.md | 26 +++++++++++++++++++
 docs/ru/operations/settings/settings-users.md | 25 ++++++++++++++++++
 2 files changed, 51 insertions(+)

diff --git a/docs/en/operations/settings/settings-users.md b/docs/en/operations/settings/settings-users.md
index 9f6fe87ae5f..1f41eafd02e 100644
--- a/docs/en/operations/settings/settings-users.md
+++ b/docs/en/operations/settings/settings-users.md
@@ -38,6 +38,10 @@ Structure of the `users` section:
                 </table_name>
             </database_name>
         </databases>
+        
+        <grants>
+            <query>GRANT SELECT ON system.*</query>
+        </grants>
     </user_name>
     <!-- Other users settings -->
 </users>
@@ -86,6 +90,28 @@ Possible values:
 
 Default value: 0.
 
+### grants {#grants-user-setting}
+
+This setting allows to grant any rights to selected user.
+Each element of the list should be `GRANT` query without any grantees specified.
+
+Example:
+
+```xml
+<user1>
+    <grants>
+        <query>GRANT SHOW ON *.*</query>
+        <query>GRANT CREATE ON *.* WITH GRANT OPTION</query>
+        <query>GRANT SELECT ON system.*</query>
+    </grants>
+</user1>
+```
+
+This setting can't be specified at the same time with
+`dictionaries`, `access_management`, `named_collection_control`, `show_named_collections_secrets`
+and `allow_databases` settings.
+
+
 ### user_name/networks {#user-namenetworks}
 
 List of networks from which the user can connect to the ClickHouse server.
diff --git a/docs/ru/operations/settings/settings-users.md b/docs/ru/operations/settings/settings-users.md
index a8d5f0ec453..50f4eb5ae6b 100644
--- a/docs/ru/operations/settings/settings-users.md
+++ b/docs/ru/operations/settings/settings-users.md
@@ -37,6 +37,10 @@ sidebar_label: "Настройки пользователей"
                 <table_name>
             </database_name>
         </databases>
+        
+        <grants>
+            <query>GRANT SELECT ON system.*</query>
+        </grants>
     </user_name>
     <!-- Other users settings -->
 </users>
@@ -89,6 +93,27 @@ sidebar_label: "Настройки пользователей"
 
 Значение по умолчанию: 0.
 
+### grants {#grants-user-setting}
+
+Настройка позволяет указать набор прав для заданного пользователя.
+Каждый элемент списка должен представлять собой `GRANT` запрос без указания пользователей в самом запросе.
+
+Пример:
+
+```xml
+<user1>
+    <grants>
+        <query>GRANT SHOW ON *.*</query>
+        <query>GRANT CREATE ON *.* WITH GRANT OPTION</query>
+        <query>GRANT SELECT ON system.*</query>
+    </grants>
+</user1>
+```
+
+Настройка не может быть выставлена одновременно с
+`dictionaries`, `access_management`, `named_collection_control`, `show_named_collections_secrets`
+или `allow_databases`.
+
 ### user_name/networks {#user-namenetworks}
 
 Список сетей, из которых пользователь может подключиться к серверу ClickHouse.

From 8f5afffaa0a203ab5b689d34c3ce447cc2a6b7f2 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Sat, 6 May 2023 14:23:57 +0200
Subject: [PATCH 220/689] Outdated parts has to be unlocked in general case

---
 src/Storages/StorageReplicatedMergeTree.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index e7cee59bda8..d0911aefc6d 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -8263,7 +8263,7 @@ StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, co
         return std::make_pair(true, NameSet{});
     }
 
-    if (part.getState() == MergeTreeDataPartState::Temporary || part.getState() == MergeTreeDataPartState::Outdated)
+    if (part.getState() == MergeTreeDataPartState::Temporary && part.is_temp)
     {
         /// Part {} is in temporary state and has it_temp flag. it means that it is under construction.
         /// That path hasn't been added to active set, no commit procedure has begun.

From 24c468edd0ff12e0b670f49b918056ae0be1cc52 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <104093494+CheSema@users.noreply.github.com>
Date: Sat, 6 May 2023 14:31:04 +0200
Subject: [PATCH 221/689] Update 02723_zookeeper_name.sql

---
 tests/queries/0_stateless/02723_zookeeper_name.sql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/02723_zookeeper_name.sql b/tests/queries/0_stateless/02723_zookeeper_name.sql
index 7ddbf4edd47..c4c968d4587 100644
--- a/tests/queries/0_stateless/02723_zookeeper_name.sql
+++ b/tests/queries/0_stateless/02723_zookeeper_name.sql
@@ -15,6 +15,8 @@ SELECT
 FROM system.replicas
 INNER JOIN system.parts USING (database, table)
 WHERE database = currentDatabase()
+ORDER BY
+    table,zookeeper_name
 GROUP BY
     table,zookeeper_name
 FORMAT CSV;

From 0598bfd79e6518c9432e341af100a31649f457e0 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 24 Mar 2023 14:38:40 +0000
Subject: [PATCH 222/689] Mark query cache as non-experimental

---
 docs/en/operations/caches.md                  |  2 +-
 docs/en/operations/query-cache.md             | 19 ++++---------------
 src/Core/Settings.h                           |  1 -
 src/Interpreters/Cache/QueryCache.cpp         |  4 +---
 src/Interpreters/executeQuery.cpp             |  4 +---
 tests/ci/stress.py                            |  1 -
 ...494_query_cache_case_agnostic_matching.sql |  2 --
 .../02494_query_cache_compression.sql         |  2 --
 .../02494_query_cache_drop_cache.sql          |  2 --
 .../02494_query_cache_eligible_queries.sql    |  2 --
 .../0_stateless/02494_query_cache_events.sql  |  2 --
 .../02494_query_cache_exception_handling.sql  |  2 --
 .../0_stateless/02494_query_cache_explain.sql |  1 -
 .../02494_query_cache_min_query_duration.sql  |  2 --
 .../02494_query_cache_min_query_runs.sql      |  2 --
 ...query_cache_nondeterministic_functions.sql |  2 --
 .../02494_query_cache_normalize_ast.sql       |  2 --
 .../02494_query_cache_passive_usage.sql       |  2 --
 .../0_stateless/02494_query_cache_secrets.sql |  2 --
 .../02494_query_cache_sparse_columns.sql      |  1 -
 ...494_query_cache_squash_partial_results.sql |  2 --
 .../02494_query_cache_totals_extremes.sql     |  2 --
 .../02494_query_cache_ttl_long.sql            |  2 --
 .../02494_query_cache_user_quotas.sql         |  2 --
 24 files changed, 7 insertions(+), 58 deletions(-)

diff --git a/docs/en/operations/caches.md b/docs/en/operations/caches.md
index 86bf8065d94..5bc16b6f51d 100644
--- a/docs/en/operations/caches.md
+++ b/docs/en/operations/caches.md
@@ -23,6 +23,6 @@ Additional cache types:
 - [Dictionaries](../sql-reference/dictionaries/index.md) data cache.
 - Schema inference cache.
 - [Filesystem cache](storing-data.md) over S3, Azure, Local and other disks.
-- [(Experimental) Query cache](query-cache.md).
+- [Query cache](query-cache.md).
 
 To drop one of the caches, use [SYSTEM DROP ... CACHE](../sql-reference/statements/system.md#drop-mark-cache) statements.
diff --git a/docs/en/operations/query-cache.md b/docs/en/operations/query-cache.md
index 50f73e2c1bb..bfa51650cd8 100644
--- a/docs/en/operations/query-cache.md
+++ b/docs/en/operations/query-cache.md
@@ -1,10 +1,10 @@
 ---
 slug: /en/operations/query-cache
 sidebar_position: 65
-sidebar_label: Query Cache [experimental]
+sidebar_label: Query Cache
 ---
 
-# Query Cache [experimental]
+# Query Cache
 
 The query cache allows to compute `SELECT` queries just once and to serve further executions of the same query directly from the cache.
 Depending on the type of the queries, this can dramatically reduce latency and resource consumption of the ClickHouse server.
@@ -29,21 +29,10 @@ Transactionally inconsistent caching is traditionally provided by client tools o
 the same caching logic and configuration is often duplicated. With ClickHouse's query cache, the caching logic moves to the server side.
 This reduces maintenance effort and avoids redundancy.
 
-:::note
-The query cache is an experimental feature that should not be used in production. There are known cases (e.g. in distributed query
-processing) where wrong results are returned.
-:::
-
 ## Configuration Settings and Usage
 
-As long as the result cache is experimental it must be activated using the following configuration setting:
-
-```sql
-SET allow_experimental_query_cache = true;
-```
-
-Afterwards, setting [use_query_cache](settings/settings.md#use-query-cache) can be used to control whether a specific query or all queries
-of the current session should utilize the query cache. For example, the first execution of query
+Setting [use_query_cache](settings/settings.md#use-query-cache) can be used to control whether a specific query or all queries of the
+current session should utilize the query cache. For example, the first execution of query
 
 ```sql
 SELECT some_expensive_calculation(column_1, column_2)
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 27e2b1801b7..42ae645615a 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -720,7 +720,6 @@ class IColumn;
     M(Bool, allow_experimental_nlp_functions, false, "Enable experimental functions for natural language processing.", 0) \
     M(Bool, allow_experimental_hash_functions, false, "Enable experimental hash functions (hashid, etc)", 0) \
     M(Bool, allow_experimental_object_type, false, "Allow Object and JSON data types", 0) \
-    M(Bool, allow_experimental_query_cache, false, "Enable experimental query cache", 0) \
     M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \
     M(String, ann_index_select_query_params, "", "Parameters passed to ANN indexes in SELECT queries, the format is 'param1=x, param2=y, ...'", 0) \
     M(UInt64, max_limit_for_ann_queries, 1000000, "Maximum limit value for using ANN indexes is used to prevent memory overflow in search queries for indexes", 0) \
diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp
index 796be10ccc8..aab004de4e5 100644
--- a/src/Interpreters/Cache/QueryCache.cpp
+++ b/src/Interpreters/Cache/QueryCache.cpp
@@ -79,9 +79,7 @@ public:
 
             auto is_query_cache_related_setting = [](const auto & change)
             {
-                return change.name == "allow_experimental_query_cache"
-                    || change.name.starts_with("query_cache")
-                    || change.name.ends_with("query_cache");
+                return change.name.starts_with("query_cache_") || change.name.ends_with("_query_cache");
             };
 
             std::erase_if(set_clause->changes, is_query_cache_related_setting);
diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index 763cf7ffc85..3a6b1f2124f 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -643,9 +643,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
             }
         }
 
-        bool can_use_query_cache =
-            settings.allow_experimental_query_cache && settings.use_query_cache && !internal
-            && !ast->as<ASTExplainQuery>();
+        bool can_use_query_cache = settings.use_query_cache && !internal && !ast->as<ASTExplainQuery>();
 
         if (!async_insert)
         {
diff --git a/tests/ci/stress.py b/tests/ci/stress.py
index 953b55dbf63..b9044874071 100755
--- a/tests/ci/stress.py
+++ b/tests/ci/stress.py
@@ -45,7 +45,6 @@ def get_options(i, upgrade_check):
             client_options.append("max_rows_in_join=1000")
 
     if i > 0 and random.random() < 1 / 3:
-        client_options.append("allow_experimental_query_cache=1")
         client_options.append("use_query_cache=1")
 
     if i % 5 == 1:
diff --git a/tests/queries/0_stateless/02494_query_cache_case_agnostic_matching.sql b/tests/queries/0_stateless/02494_query_cache_case_agnostic_matching.sql
index 9440a1fd9c0..9f26d7759de 100644
--- a/tests/queries/0_stateless/02494_query_cache_case_agnostic_matching.sql
+++ b/tests/queries/0_stateless/02494_query_cache_case_agnostic_matching.sql
@@ -1,8 +1,6 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 -- Start with empty query cache (QC) and query log
 SYSTEM DROP QUERY CACHE;
 DROP TABLE system.query_log SYNC;
diff --git a/tests/queries/0_stateless/02494_query_cache_compression.sql b/tests/queries/0_stateless/02494_query_cache_compression.sql
index 619ae7d5cd8..3d17deebd94 100644
--- a/tests/queries/0_stateless/02494_query_cache_compression.sql
+++ b/tests/queries/0_stateless/02494_query_cache_compression.sql
@@ -1,8 +1,6 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 SYSTEM DROP QUERY CACHE;
 DROP TABLE IF EXISTS t;
 
diff --git a/tests/queries/0_stateless/02494_query_cache_drop_cache.sql b/tests/queries/0_stateless/02494_query_cache_drop_cache.sql
index 078057a834f..bc2e7f442fc 100644
--- a/tests/queries/0_stateless/02494_query_cache_drop_cache.sql
+++ b/tests/queries/0_stateless/02494_query_cache_drop_cache.sql
@@ -1,8 +1,6 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 -- (it's silly to use what will be tested below but we have to assume other tests cluttered the query cache)
 SYSTEM DROP QUERY CACHE;
 
diff --git a/tests/queries/0_stateless/02494_query_cache_eligible_queries.sql b/tests/queries/0_stateless/02494_query_cache_eligible_queries.sql
index b4bc9e2c258..5c45ee8aedd 100644
--- a/tests/queries/0_stateless/02494_query_cache_eligible_queries.sql
+++ b/tests/queries/0_stateless/02494_query_cache_eligible_queries.sql
@@ -1,8 +1,6 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 SYSTEM DROP QUERY CACHE;
 DROP TABLE IF EXISTS eligible_test;
 DROP TABLE IF EXISTS eligible_test2;
diff --git a/tests/queries/0_stateless/02494_query_cache_events.sql b/tests/queries/0_stateless/02494_query_cache_events.sql
index d775467d525..900b68f5eb2 100644
--- a/tests/queries/0_stateless/02494_query_cache_events.sql
+++ b/tests/queries/0_stateless/02494_query_cache_events.sql
@@ -1,8 +1,6 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 -- Start with empty query cache QC and query log
 SYSTEM DROP QUERY CACHE;
 DROP TABLE system.query_log SYNC;
diff --git a/tests/queries/0_stateless/02494_query_cache_exception_handling.sql b/tests/queries/0_stateless/02494_query_cache_exception_handling.sql
index 4d686d81ed3..70a443cc75c 100644
--- a/tests/queries/0_stateless/02494_query_cache_exception_handling.sql
+++ b/tests/queries/0_stateless/02494_query_cache_exception_handling.sql
@@ -1,8 +1,6 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 SYSTEM DROP QUERY CACHE;
 
 -- If an exception is thrown during query execution, no entry must be created in the query cache
diff --git a/tests/queries/0_stateless/02494_query_cache_explain.sql b/tests/queries/0_stateless/02494_query_cache_explain.sql
index 68b7e0005f8..d12938181c2 100644
--- a/tests/queries/0_stateless/02494_query_cache_explain.sql
+++ b/tests/queries/0_stateless/02494_query_cache_explain.sql
@@ -2,7 +2,6 @@
 -- Tag no-parallel: Messes with internal cache
 
 SET allow_experimental_analyzer = 1;
-SET allow_experimental_query_cache = true;
 
 SYSTEM DROP QUERY CACHE;
 
diff --git a/tests/queries/0_stateless/02494_query_cache_min_query_duration.sql b/tests/queries/0_stateless/02494_query_cache_min_query_duration.sql
index 7d759c86130..ee0b73f43e2 100644
--- a/tests/queries/0_stateless/02494_query_cache_min_query_duration.sql
+++ b/tests/queries/0_stateless/02494_query_cache_min_query_duration.sql
@@ -1,8 +1,6 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 SYSTEM DROP QUERY CACHE;
 
 -- This creates an entry in the query cache ...
diff --git a/tests/queries/0_stateless/02494_query_cache_min_query_runs.sql b/tests/queries/0_stateless/02494_query_cache_min_query_runs.sql
index 2401beeab13..ffdf12169e3 100644
--- a/tests/queries/0_stateless/02494_query_cache_min_query_runs.sql
+++ b/tests/queries/0_stateless/02494_query_cache_min_query_runs.sql
@@ -1,8 +1,6 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 SYSTEM DROP QUERY CACHE;
 
 -- Cache the query after the 1st query invocation
diff --git a/tests/queries/0_stateless/02494_query_cache_nondeterministic_functions.sql b/tests/queries/0_stateless/02494_query_cache_nondeterministic_functions.sql
index 534d63aa427..045b7258a34 100644
--- a/tests/queries/0_stateless/02494_query_cache_nondeterministic_functions.sql
+++ b/tests/queries/0_stateless/02494_query_cache_nondeterministic_functions.sql
@@ -1,8 +1,6 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 SYSTEM DROP QUERY CACHE;
 
 -- rand() is non-deterministic, with default settings no entry in the query cache should be created
diff --git a/tests/queries/0_stateless/02494_query_cache_normalize_ast.sql b/tests/queries/0_stateless/02494_query_cache_normalize_ast.sql
index 7e3cd273312..5fd09eb935b 100644
--- a/tests/queries/0_stateless/02494_query_cache_normalize_ast.sql
+++ b/tests/queries/0_stateless/02494_query_cache_normalize_ast.sql
@@ -1,8 +1,6 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 -- Start with empty query cache (QC) and query log.
 SYSTEM DROP QUERY CACHE;
 DROP TABLE system.query_log SYNC;
diff --git a/tests/queries/0_stateless/02494_query_cache_passive_usage.sql b/tests/queries/0_stateless/02494_query_cache_passive_usage.sql
index 8f1e3972b6d..6143b5f7083 100644
--- a/tests/queries/0_stateless/02494_query_cache_passive_usage.sql
+++ b/tests/queries/0_stateless/02494_query_cache_passive_usage.sql
@@ -1,8 +1,6 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 -- Start with empty query cache (QC).
 SYSTEM DROP QUERY CACHE;
 
diff --git a/tests/queries/0_stateless/02494_query_cache_secrets.sql b/tests/queries/0_stateless/02494_query_cache_secrets.sql
index 99a972b003c..66427df8f10 100644
--- a/tests/queries/0_stateless/02494_query_cache_secrets.sql
+++ b/tests/queries/0_stateless/02494_query_cache_secrets.sql
@@ -2,8 +2,6 @@
 -- Tag no-fasttest: Depends on OpenSSL
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 SYSTEM DROP QUERY CACHE;
 
 -- Cache a result of a query with secret in the query cache
diff --git a/tests/queries/0_stateless/02494_query_cache_sparse_columns.sql b/tests/queries/0_stateless/02494_query_cache_sparse_columns.sql
index 952c3227852..4344d139d60 100644
--- a/tests/queries/0_stateless/02494_query_cache_sparse_columns.sql
+++ b/tests/queries/0_stateless/02494_query_cache_sparse_columns.sql
@@ -12,7 +12,6 @@ SYSTEM STOP MERGES t_cache_sparse;
 INSERT INTO t_cache_sparse SELECT number, number FROM numbers(10000);
 INSERT INTO t_cache_sparse SELECT number, 0 FROM numbers(10000);
 
-SET allow_experimental_query_cache = 1;
 SET use_query_cache = 1;
 SET max_threads = 1;
 
diff --git a/tests/queries/0_stateless/02494_query_cache_squash_partial_results.sql b/tests/queries/0_stateless/02494_query_cache_squash_partial_results.sql
index 0dd5fcb2977..391cc7f7e1a 100644
--- a/tests/queries/0_stateless/02494_query_cache_squash_partial_results.sql
+++ b/tests/queries/0_stateless/02494_query_cache_squash_partial_results.sql
@@ -1,8 +1,6 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 SYSTEM DROP QUERY CACHE;
 DROP TABLE IF EXISTS t;
 
diff --git a/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql b/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql
index 0d05adca6c8..8fde4a668c8 100644
--- a/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql
+++ b/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql
@@ -1,8 +1,6 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 SYSTEM DROP QUERY CACHE;
 DROP TABLE IF EXISTS tbl;
 
diff --git a/tests/queries/0_stateless/02494_query_cache_ttl_long.sql b/tests/queries/0_stateless/02494_query_cache_ttl_long.sql
index 135ddf2195c..acaf34ee81d 100644
--- a/tests/queries/0_stateless/02494_query_cache_ttl_long.sql
+++ b/tests/queries/0_stateless/02494_query_cache_ttl_long.sql
@@ -3,8 +3,6 @@
 -- Tag long: Test runtime is > 6 sec
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_query_cache = true;
-
 SYSTEM DROP QUERY CACHE;
 
 -- Cache query result into query cache with a TTL of 3 sec
diff --git a/tests/queries/0_stateless/02494_query_cache_user_quotas.sql b/tests/queries/0_stateless/02494_query_cache_user_quotas.sql
index 9890cdf7933..123c9d21101 100644
--- a/tests/queries/0_stateless/02494_query_cache_user_quotas.sql
+++ b/tests/queries/0_stateless/02494_query_cache_user_quotas.sql
@@ -5,8 +5,6 @@
 -- be used in a settings profile, together with a readonly constraint. For simplicity, test both settings stand-alone in a stateless test
 -- instead of an integration test - the relevant logic will still be covered by that.
 
-SET allow_experimental_query_cache = true;
-
 SYSTEM DROP QUERY CACHE;
 
 SET query_cache_max_size_in_bytes = 1;

From 883e0abea1570a7dee6e10b6579a94758338f3f5 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 6 May 2023 14:22:23 +0000
Subject: [PATCH 223/689] Query Cache: Safeguard against empty chunks

Reading from the query cache is implemented using processor
"SourceFromChunks" which inherits from ISource. The latter has logic
which finishes processing (= calls .finish() on the output port +
returns Status::Finished) when the derived class returns an empty chunk.
If this empty chunk is not the last chunk, i.e. if it is followed by
non-empty chunks, the query result will be incorrect. This situation
should theoretically never occur in practice but who knows ... To be on
the safe side, writing into the query cache now rejects empty chunks and
thereby avoids this scenario.
---
 docs/en/operations/settings/settings.md | 4 ++--
 src/Interpreters/Cache/QueryCache.cpp   | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 39bf111bef6..f21dff9fbb7 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -1514,7 +1514,7 @@ Default value: `0`.
 
 ## query_cache_max_size_in_bytes {#query-cache-max-size-in-bytes}
 
-The maximum amount of memory (in bytes) the current user may allocate in the query cache. 0 means unlimited.
+The maximum amount of memory (in bytes) the current user may allocate in the [query cache](../query-cache.md). 0 means unlimited.
 
 Possible values:
 
@@ -1524,7 +1524,7 @@ Default value: 0 (no restriction).
 
 ## query_cache_max_entries {#query-cache-max-entries}
 
-The maximum number of query results the current user may store in the query cache. 0 means unlimited.
+The maximum number of query results the current user may store in the [query cache](../query-cache.md). 0 means unlimited.
 
 Possible values:
 
diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp
index 796be10ccc8..6afcac5172b 100644
--- a/src/Interpreters/Cache/QueryCache.cpp
+++ b/src/Interpreters/Cache/QueryCache.cpp
@@ -208,6 +208,9 @@ void QueryCache::Writer::buffer(Chunk && chunk, ChunkType chunk_type)
     if (skip_insert)
         return;
 
+    if (chunk.empty())
+        return;
+
     std::lock_guard lock(mutex);
 
     switch (chunk_type)

From d353cdcb091ceb40af906637cde4413701ab7fd1 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 6 May 2023 14:46:53 +0000
Subject: [PATCH 224/689] 02723_zookeeper_name: Force a deterministic result
 order

---
 tests/queries/0_stateless/02723_zookeeper_name.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/02723_zookeeper_name.sql b/tests/queries/0_stateless/02723_zookeeper_name.sql
index 7ddbf4edd47..f0d01166368 100644
--- a/tests/queries/0_stateless/02723_zookeeper_name.sql
+++ b/tests/queries/0_stateless/02723_zookeeper_name.sql
@@ -17,6 +17,7 @@ INNER JOIN system.parts USING (database, table)
 WHERE database = currentDatabase()
 GROUP BY
     table,zookeeper_name
+ORDER BY table
 FORMAT CSV;
 
 DROP TABLE t1;

From 452951179962fcffb2e97cec92e0f6a6ad6f9b4a Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 6 May 2023 15:01:24 +0000
Subject: [PATCH 225/689] Fix table function docs

---
 src/TableFunctions/TableFunctionDeltaLake.cpp |  7 ++++-
 src/TableFunctions/TableFunctionExplain.cpp   | 11 +++++++-
 src/TableFunctions/TableFunctionZeros.cpp     | 26 +++++++++++++++++--
 3 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/src/TableFunctions/TableFunctionDeltaLake.cpp b/src/TableFunctions/TableFunctionDeltaLake.cpp
index a77acce775c..b8bf810f6fa 100644
--- a/src/TableFunctions/TableFunctionDeltaLake.cpp
+++ b/src/TableFunctions/TableFunctionDeltaLake.cpp
@@ -20,7 +20,12 @@ using TableFunctionDeltaLake = ITableFunctionDataLake<TableFunctionDeltaLakeName
 
 void registerTableFunctionDeltaLake(TableFunctionFactory & factory)
 {
-    factory.registerFunction<TableFunctionDeltaLake>({.allow_readonly = false});
+    factory.registerFunction<TableFunctionDeltaLake>(
+        {.documentation = {
+            .description=R"(The table function can be used to read the DeltaLake table stored on object store.)",
+            .examples{{"deltaLake", "SELECT * FROM deltaLake(url, access_key_id, secret_access_key)", ""}},
+            .categories{"DataLake"}},
+         .allow_readonly = false});
 }
 
 }
diff --git a/src/TableFunctions/TableFunctionExplain.cpp b/src/TableFunctions/TableFunctionExplain.cpp
index 335ddb6f0ff..aae93c8b596 100644
--- a/src/TableFunctions/TableFunctionExplain.cpp
+++ b/src/TableFunctions/TableFunctionExplain.cpp
@@ -147,7 +147,16 @@ InterpreterExplainQuery TableFunctionExplain::getInterpreter(ContextPtr context)
 
 void registerTableFunctionExplain(TableFunctionFactory & factory)
 {
-    factory.registerFunction<TableFunctionExplain>();
+    factory.registerFunction<TableFunctionExplain>({.documentation = {
+            .description=R"(
+                Returns result of EXPLAIN query.
+                The function should not be called directly but can be invoked via `SELECT * FROM (EXPLAIN <query>)`.
+                You can use this query to process the result of EXPLAIN further using SQL (e.g., in tests).
+                Example:
+                [example:1]
+                )",
+            .examples={{"1", "SELECT explain FROM (EXPLAIN AST SELECT * FROM system.numbers) WHERE explain LIKE '%Asterisk%'", ""}}
+        }});
 }
 
 }
diff --git a/src/TableFunctions/TableFunctionZeros.cpp b/src/TableFunctions/TableFunctionZeros.cpp
index c3dc819981b..3c487362e1f 100644
--- a/src/TableFunctions/TableFunctionZeros.cpp
+++ b/src/TableFunctions/TableFunctionZeros.cpp
@@ -48,8 +48,30 @@ StoragePtr TableFunctionZeros<multithreaded>::executeImpl(const ASTPtr & ast_fun
 
 void registerTableFunctionZeros(TableFunctionFactory & factory)
 {
-    factory.registerFunction<TableFunctionZeros<true>>();
-    factory.registerFunction<TableFunctionZeros<false>>();
+    factory.registerFunction<TableFunctionZeros<true>>({.documentation = {
+            .description=R"(
+                Generates a stream of zeros (a table with one column 'zero' of type 'UInt8') of specified size.
+                This table function is used in performance tests, where you want to spend as little time as possible to data generation while testing some other parts of queries.
+                In contrast to the `zeros_mt`, this table function is using single thread for data generation.
+                Example:
+                [example:1]
+                This query will test the speed of `randomPrintableASCII` function using single thread.
+                See also the `system.zeros` table.)",
+            .examples={{"1", "SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomPrintableASCII(10))", ""}}
+    }});
+
+    factory.registerFunction<TableFunctionZeros<false>>({.documentation = {
+            .description=R"(
+                Generates a stream of zeros (a table with one column 'zero' of type 'UInt8') of specified size.
+                This table function is used in performance tests, where you want to spend as little time as possible to data generation while testing some other parts of queries.
+                In contrast to the `zeros`, this table function is using multiple threads for data generation, according to the `max_threads` setting.
+                Example:
+                [example:1]
+                This query will test the speed of `randomPrintableASCII` function using multiple threads.
+                See also the `system.zeros` table.
+                )",
+            .examples={{"1", "SELECT count() FROM zeros_mt(1000000000) WHERE NOT ignore(randomPrintableASCII(10))", ""}}
+}});
 }
 
 template <bool multithreaded>

From 8f1123dfb853398a57a85ecdf855da7c173bb9ff Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 18:23:54 +0300
Subject: [PATCH 226/689] Update test_in_memory_parts_still_read.py

---
 .../test_in_memory_parts_still_read.py                          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py b/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
index d32a35bf95c..0595be54879 100644
--- a/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
+++ b/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
@@ -27,7 +27,7 @@ def start_cluster():
 
 def test_in_memory_parts_still_read(start_cluster):
     node.query(
-        "CREATE TABLE t (x UInt64, s String, a Array(Tuple(Map(String, LowCardinality(String)), Date32, DateTime64(3))) ENGINE = MergeTree ORDER BY s SETTINGS min_rows_for_compact_part = 1000000, min_bytes_for_compact_part = '1G', in_memory_parts_enable_wal = 1"
+        "CREATE TABLE t (x UInt64, s String, a Array(Tuple(Map(String, LowCardinality(String)))), Date32, DateTime64(3))) ENGINE = MergeTree ORDER BY s SETTINGS min_rows_for_compact_part = 1000000, min_bytes_for_compact_part = '1G', in_memory_parts_enable_wal = 1"
     )
     node.query("INSERT INTO t SELECT generateRandom() LIMIT 100")
 

From 62d867777392c06b9921e461606df8a754871fe6 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 18:25:43 +0300
Subject: [PATCH 227/689] Update QueryCache.cpp

---
 src/Interpreters/Cache/QueryCache.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp
index 6afcac5172b..2f8ce7e9981 100644
--- a/src/Interpreters/Cache/QueryCache.cpp
+++ b/src/Interpreters/Cache/QueryCache.cpp
@@ -208,6 +208,12 @@ void QueryCache::Writer::buffer(Chunk && chunk, ChunkType chunk_type)
     if (skip_insert)
         return;
 
+    /// Reading from the query cache is implemented using processor `SourceFromChunks` which inherits from `ISource`.
+    /// The latter has logic which finishes processing (= calls `.finish()` on the output port + returns `Status::Finished`)
+    /// when the derived class returns an empty chunk. If this empty chunk is not the last chunk,
+    /// i.e. if it is followed by non-empty chunks, the query result will be incorrect.
+    /// This situation should theoretically never occur in practice but who knows...
+    /// To be on the safe side, writing into the query cache now rejects empty chunks and thereby avoids this scenario.
     if (chunk.empty())
         return;
 

From 56f291771873961b5cf0d512622db632e411a99b Mon Sep 17 00:00:00 2001
From: Sema Checherinda <104093494+CheSema@users.noreply.github.com>
Date: Sat, 6 May 2023 17:59:07 +0200
Subject: [PATCH 228/689] Update StorageReplicatedMergeTree.cpp

---
 src/Storages/StorageReplicatedMergeTree.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index d0911aefc6d..1729e0f111f 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -8275,7 +8275,7 @@ StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, co
         /// In opposition, when the part has been mutated in generally it hardlinks the files.
         /// Therefore remote data is shared, it has to be unlocked in the keper.
         /// In order to track all that cases has_exclusive_blobs optional value is used
-        /// If has_exclusive_blobs is determined, then CH traked the parts origin from the creation
+        /// If has_exclusive_blobs is determined, then CH tracked the parts origin from the creation
 
         if (part.has_exclusive_blobs.has_value())
         {

From 7897a6cf92da9b08d67fb89c97be3390430df97d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 18:14:01 +0200
Subject: [PATCH 229/689] Remove dangerous code (stringstream)

---
 src/Common/StackTrace.cpp | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp
index 3c0bcd2f808..033581dfc86 100644
--- a/src/Common/StackTrace.cpp
+++ b/src/Common/StackTrace.cpp
@@ -9,6 +9,10 @@
 #include <Common/MemorySanitizer.h>
 #include <Common/SymbolIndex.h>
 
+#include <IO/WriteBufferFromString.h>
+#include <IO/WriteHelpers.h>
+#include <IO/Operators.h>
+
 #include <atomic>
 #include <filesystem>
 #include <map>
@@ -340,8 +344,7 @@ toStringEveryLineImpl([[maybe_unused]] bool fatal, const StackTraceRefTriple & s
         return callback("<Empty trace>");
 
 #if defined(__ELF__) && !defined(OS_FREEBSD)
-    std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
-    out.exceptions(std::ios::failbit);
+    DB::WriteBufferFromOwnString out;
 
     using enum DB::Dwarf::LocationInfoMode;
     const auto mode = fatal ? FULL_WITH_INLINE : FAST;
@@ -376,7 +379,10 @@ toStringEveryLineImpl([[maybe_unused]] bool fatal, const StackTraceRefTriple & s
             out << "?";
 
         if (shouldShowAddress(physical_addr))
-            out << " @ " << physical_addr;
+        {
+            out << " @ ";
+            DB::writePointerHex(physical_addr, out);
+        }
 
         out << " in " << (object ? object->name : "?");
 
@@ -393,7 +399,6 @@ toStringEveryLineImpl([[maybe_unused]] bool fatal, const StackTraceRefTriple & s
         }
 
         callback(out.str());
-        out.str({});
     }
 #else
     for (size_t i = stack_trace.offset; i < stack_trace.size; ++i)
@@ -431,8 +436,7 @@ String toStringCached(const StackTrace::FramePointers & pointers, size_t offset,
         return it->second;
     else
     {
-        std::ostringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
-        out.exceptions(std::ios::failbit);
+        DB::WriteBufferFromOwnString out;
         toStringEveryLineImpl(false, key, [&](std::string_view str) { out << str << '\n'; });
 
         return cache.emplace(StackTraceTriple{pointers, offset, size}, out.str()).first->second;

From b1a92a57122322e1b29d58139b78633ca6ab21dd Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 18:30:18 +0200
Subject: [PATCH 230/689] Remove dangerous code

---
 src/Columns/tests/gtest_weak_hash_32.cpp | 54 ++----------------------
 src/Common/StudentTTest.cpp              | 17 ++++----
 src/Daemon/BaseDaemon.cpp                |  9 ++--
 3 files changed, 19 insertions(+), 61 deletions(-)

diff --git a/src/Columns/tests/gtest_weak_hash_32.cpp b/src/Columns/tests/gtest_weak_hash_32.cpp
index cbf47790b9f..44337e5d3f0 100644
--- a/src/Columns/tests/gtest_weak_hash_32.cpp
+++ b/src/Columns/tests/gtest_weak_hash_32.cpp
@@ -26,9 +26,7 @@ using namespace DB;
 template <typename T>
 void checkColumn(
     const WeakHash32::Container & hash,
-    const PaddedPODArray<T> & eq_class,
-    size_t allowed_collisions = 0,
-    size_t max_collisions_to_print = 10)
+    const PaddedPODArray<T> & eq_class)
 {
     ASSERT_EQ(hash.size(), eq_class.size());
 
@@ -52,41 +50,6 @@ void checkColumn(
             }
         }
     }
-
-    /// Check have not many collisions.
-    {
-        std::unordered_map<UInt32, T> map;
-        size_t num_collisions = 0;
-
-        std::stringstream collisions_str;       // STYLE_CHECK_ALLOW_STD_STRING_STREAM
-        collisions_str.exceptions(std::ios::failbit);
-
-        for (size_t i = 0; i < eq_class.size(); ++i)
-        {
-            auto & val = eq_class[i];
-            auto it = map.find(hash[i]);
-
-            if (it == map.end())
-                map[hash[i]] = val;
-            else if (it->second != val)
-            {
-                ++num_collisions;
-
-                if (num_collisions <= max_collisions_to_print)
-                {
-                    collisions_str << "Collision:\n";
-                }
-
-                if (num_collisions > allowed_collisions)
-                {
-                    std::cerr << collisions_str.rdbuf();
-                    break;
-                }
-            }
-        }
-
-        ASSERT_LE(num_collisions, allowed_collisions);
-    }
 }
 
 TEST(WeakHash32, ColumnVectorU8)
@@ -374,10 +337,7 @@ TEST(WeakHash32, ColumnString2)
     WeakHash32 hash(col->size());
     col->updateWeakHash32(hash);
 
-    /// Now there is single collision between 'k' * 544 and 'q' * 2512 (which is calculated twice)
-    size_t allowed_collisions = 4;
-
-    checkColumn(hash.getData(), data, allowed_collisions);
+    checkColumn(hash.getData(), data);
 }
 
 TEST(WeakHash32, ColumnString3)
@@ -717,8 +677,7 @@ TEST(WeakHash32, ColumnTupleUInt64String)
     WeakHash32 hash(col_tuple->size());
     col_tuple->updateWeakHash32(hash);
 
-    size_t allowed_collisions = 8;
-    checkColumn(hash.getData(), eq, allowed_collisions);
+    checkColumn(hash.getData(), eq);
 }
 
 TEST(WeakHash32, ColumnTupleUInt64FixedString)
@@ -803,10 +762,5 @@ TEST(WeakHash32, ColumnTupleUInt64Array)
     WeakHash32 hash(col_tuple->size());
     col_tuple->updateWeakHash32(hash);
 
-    /// There are 2 collisions right now (repeated 2 times each):
-    /// (0, [array of size 1212 with values 7]) vs (0, [array of size 2265 with values 17])
-    /// (0, [array of size 558 with values 5]) vs (1, [array of size 879 with values 21])
-
-    size_t allowed_collisions = 8;
-    checkColumn(hash.getData(), eq_data, allowed_collisions);
+    checkColumn(hash.getData(), eq_data);
 }
diff --git a/src/Common/StudentTTest.cpp b/src/Common/StudentTTest.cpp
index 63f48f319d2..59842488fd0 100644
--- a/src/Common/StudentTTest.cpp
+++ b/src/Common/StudentTTest.cpp
@@ -6,6 +6,9 @@
 #include <sstream>
 #include <stdexcept>
 
+#include <IO/WriteBufferFromString.h>
+#include <IO/Operators.h>
+
 
 namespace
 {
@@ -153,19 +156,17 @@ std::pair<bool, std::string> StudentTTest::compareAndReport(size_t confidence_le
 
     double mean_confidence_interval = table_value * t_statistic;
 
-    std::stringstream ss;       // STYLE_CHECK_ALLOW_STD_STRING_STREAM
-    ss.exceptions(std::ios::failbit);
+    DB::WriteBufferFromOwnString out;
 
     if (mean_difference > mean_confidence_interval && (mean_difference - mean_confidence_interval > 0.0001)) /// difference must be more than 0.0001, to take into account connection latency.
     {
-        ss << "Difference at " << confidence_level[confidence_level_index] <<  "% confidence: ";
-        ss << std::fixed << std::setprecision(8) << "mean difference is " << mean_difference << ", but confidence interval is " << mean_confidence_interval;
-        return {false, ss.str()};
+        out << "Difference at " << confidence_level[confidence_level_index] <<  "% confidence: ";
+        out << "mean difference is " << mean_difference << ", but confidence interval is " << mean_confidence_interval;
+        return {false, out.str()};
     }
     else
     {
-        ss << "No difference proven at " << confidence_level[confidence_level_index] <<  "% confidence";
-        return {true, ss.str()};
+        out << "No difference proven at " << confidence_level[confidence_level_index] <<  "% confidence";
+        return {true, out.str()};
     }
 }
-
diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp
index 077fa55e8d3..4780dfed4b2 100644
--- a/src/Daemon/BaseDaemon.cpp
+++ b/src/Daemon/BaseDaemon.cpp
@@ -358,10 +358,13 @@ private:
             /// NOTE: This still require memory allocations and mutex lock inside logger.
             ///       BTW we can also print it to stderr using write syscalls.
 
-            std::stringstream bare_stacktrace; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
-            bare_stacktrace << "Stack trace:";
+            DB::WriteBufferFromOwnString bare_stacktrace;
+            DB::writeString("Stack trace:", bare_stacktrace);
             for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i)
-                bare_stacktrace << ' ' << stack_trace.getFramePointers()[i];
+            {
+                DB::writeChar(' ', bare_stacktrace);
+                DB::writePointerHex(stack_trace.getFramePointers()[i], bare_stacktrace);
+            }
 
             LOG_FATAL(log, fmt::runtime(bare_stacktrace.str()));
         }

From 355d5af78692db46b49a06829f4a2b22c081710a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 18:33:45 +0200
Subject: [PATCH 231/689] Remove some code

---
 CMakeLists.txt                                    |  4 ++--
 base/poco/Foundation/include/Poco/Config.h        |  2 +-
 base/poco/Foundation/include/Poco/NumericString.h | 11 -----------
 base/poco/Foundation/src/Format.cpp               |  6 ------
 base/poco/Foundation/src/NumberFormatter.cpp      |  3 ---
 base/poco/Foundation/src/NumberParser.cpp         |  3 ---
 6 files changed, 3 insertions(+), 26 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 263b202049b..5defe27a272 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -57,7 +57,7 @@ if (ENABLE_CHECK_HEAVY_BUILDS)
     # set CPU time limit to 1000 seconds
     set (RLIMIT_CPU 1000)
 
-    # gcc10/gcc10/clang -fsanitize=memory is too heavy
+    # -fsanitize=memory is too heavy
     if (SANITIZE STREQUAL "memory")
        set (RLIMIT_DATA 10000000000) # 10G
     endif()
@@ -280,7 +280,7 @@ set (CMAKE_C_STANDARD 11)
 set (CMAKE_C_EXTENSIONS ON) # required by most contribs written in C
 set (CMAKE_C_STANDARD_REQUIRED ON)
 
-# Compiler-specific coverage flags e.g. -fcoverage-mapping for gcc
+# Compiler-specific coverage flags e.g. -fcoverage-mapping
 option(WITH_COVERAGE "Profile the resulting binary/binaries" OFF)
 
 if (COMPILER_CLANG)
diff --git a/base/poco/Foundation/include/Poco/Config.h b/base/poco/Foundation/include/Poco/Config.h
index e3fcb9d91cd..83d2c89553a 100644
--- a/base/poco/Foundation/include/Poco/Config.h
+++ b/base/poco/Foundation/include/Poco/Config.h
@@ -53,7 +53,7 @@
 
 
 // Define if no <locale> header is available (such as on WinCE)
-// #define POCO_NO_LOCALE
+#define POCO_NO_LOCALE
 
 
 // Define to desired default thread stack size
diff --git a/base/poco/Foundation/include/Poco/NumericString.h b/base/poco/Foundation/include/Poco/NumericString.h
index 7c9628c7f16..cc5c011e6ed 100644
--- a/base/poco/Foundation/include/Poco/NumericString.h
+++ b/base/poco/Foundation/include/Poco/NumericString.h
@@ -30,9 +30,6 @@
 #include <cctype>
 #include <cmath>
 #include <limits>
-#if !defined(POCO_NO_LOCALE)
-#    include <locale>
-#endif
 
 
 // binary numbers are supported, thus 64 (bits) + 1 (string terminating zero)
@@ -53,11 +50,7 @@ inline char decimalSeparator()
 /// Returns decimal separator from global locale or
 /// default '.' for platforms where locale is unavailable.
 {
-#if !defined(POCO_NO_LOCALE)
-    return std::use_facet<std::numpunct<char>>(std::locale()).decimal_point();
-#else
     return '.';
-#endif
 }
 
 
@@ -65,11 +58,7 @@ inline char thousandSeparator()
 /// Returns thousand separator from global locale or
 /// default ',' for platforms where locale is unavailable.
 {
-#if !defined(POCO_NO_LOCALE)
-    return std::use_facet<std::numpunct<char>>(std::locale()).thousands_sep();
-#else
     return ',';
-#endif
 }
 
 
diff --git a/base/poco/Foundation/src/Format.cpp b/base/poco/Foundation/src/Format.cpp
index ed5db5c21b2..9872ddff042 100644
--- a/base/poco/Foundation/src/Format.cpp
+++ b/base/poco/Foundation/src/Format.cpp
@@ -16,9 +16,6 @@
 #include "Poco/Exception.h"
 #include "Poco/Ascii.h"
 #include <sstream>
-#if !defined(POCO_NO_LOCALE)
-#include <locale>
-#endif
 #include <cstddef>
 
 
@@ -147,9 +144,6 @@ namespace
 	void formatOne(std::string& result, std::string::const_iterator& itFmt, const std::string::const_iterator& endFmt, std::vector<Any>::const_iterator& itVal)
 	{
 		std::ostringstream str;
-#if !defined(POCO_NO_LOCALE)
-		str.imbue(std::locale::classic());
-#endif
 		try
 		{
 			parseFlags(str, itFmt, endFmt);
diff --git a/base/poco/Foundation/src/NumberFormatter.cpp b/base/poco/Foundation/src/NumberFormatter.cpp
index 0a9334059a9..328164911bf 100644
--- a/base/poco/Foundation/src/NumberFormatter.cpp
+++ b/base/poco/Foundation/src/NumberFormatter.cpp
@@ -15,9 +15,6 @@
 #include "Poco/NumberFormatter.h"
 #include "Poco/MemoryStream.h"
 #include <iomanip>
-#if !defined(POCO_NO_LOCALE)
-#include <locale>
-#endif
 #include <cstdio>
 
 
diff --git a/base/poco/Foundation/src/NumberParser.cpp b/base/poco/Foundation/src/NumberParser.cpp
index 8d32e1a722c..b4faa455922 100644
--- a/base/poco/Foundation/src/NumberParser.cpp
+++ b/base/poco/Foundation/src/NumberParser.cpp
@@ -19,9 +19,6 @@
 #include <cstdio>
 #include <cctype>
 #include <stdlib.h>
-#if !defined(POCO_NO_LOCALE)
-	#include <locale>
-#endif
 
 
 #if defined(POCO_LONG_IS_64_BIT)

From 13ed528e0bb8e6d309075bd51dc1df3c93feb13b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 18:44:10 +0200
Subject: [PATCH 232/689] Remove "locale"

---
 contrib/llvm-project | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/llvm-project b/contrib/llvm-project
index 2aedf7598a4..d857c707fcc 160000
--- a/contrib/llvm-project
+++ b/contrib/llvm-project
@@ -1 +1 @@
-Subproject commit 2aedf7598a4040b23881dbe05b6afaca25a337ef
+Subproject commit d857c707fccd50423bea1c4710dc469cf89607a9

From 6848eafb399834dfddd53b9f69fd8408e73659fa Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 6 May 2023 17:06:44 +0000
Subject: [PATCH 233/689] Cleanup utils build

---
 utils/CMakeLists.txt                    | 11 ++---------
 utils/keeper-bench/CMakeLists.txt       |  2 +-
 utils/keeper-data-dumper/CMakeLists.txt |  5 +++++
 utils/memcpy-bench/CMakeLists.txt       |  6 ++++++
 4 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt
index bb9d4c88fa1..5519b9bb983 100644
--- a/utils/CMakeLists.txt
+++ b/utils/CMakeLists.txt
@@ -26,13 +26,6 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
     add_subdirectory (wal-dump)
     add_subdirectory (check-mysql-binlog)
     add_subdirectory (keeper-bench)
-
-    if (TARGET ch_contrib::nuraft)
-        add_subdirectory (keeper-data-dumper)
-    endif ()
-
-    # memcpy_jart.S contains position dependent code
-    if (NOT CMAKE_POSITION_INDEPENDENT_CODE AND OS_LINUX AND ARCH_AMD64)
-        add_subdirectory (memcpy-bench)
-    endif ()
+    add_subdirectory (keeper-data-dumper)
+    add_subdirectory (memcpy-bench)
 endif ()
diff --git a/utils/keeper-bench/CMakeLists.txt b/utils/keeper-bench/CMakeLists.txt
index 87fa64b1761..49ce2068246 100644
--- a/utils/keeper-bench/CMakeLists.txt
+++ b/utils/keeper-bench/CMakeLists.txt
@@ -1,7 +1,7 @@
 if (NOT TARGET ch_contrib::rapidjson)
     message (${RECONFIGURE_MESSAGE_LEVEL} "Not building keeper-bench due to rapidjson is disabled")
     return()
-endif()
+endif ()
 
 clickhouse_add_executable(keeper-bench Generator.cpp Runner.cpp Stats.cpp main.cpp)
 target_link_libraries(keeper-bench PRIVATE clickhouse_common_config_no_zookeeper_log ch_contrib::rapidjson)
diff --git a/utils/keeper-data-dumper/CMakeLists.txt b/utils/keeper-data-dumper/CMakeLists.txt
index 1f55e50e68e..8c260c2122e 100644
--- a/utils/keeper-data-dumper/CMakeLists.txt
+++ b/utils/keeper-data-dumper/CMakeLists.txt
@@ -1,2 +1,7 @@
+if (NOT TARGET ch_contrib::nuraft)
+    message (${RECONFIGURE_MESSAGE_LEVEL} "Not building keeper-data-dumper due to nuraft is disabled")
+    return ()
+endif ()
+
 clickhouse_add_executable(keeper-data-dumper main.cpp)
 target_link_libraries(keeper-data-dumper PRIVATE dbms)
diff --git a/utils/memcpy-bench/CMakeLists.txt b/utils/memcpy-bench/CMakeLists.txt
index 460a06ba851..7227728e849 100644
--- a/utils/memcpy-bench/CMakeLists.txt
+++ b/utils/memcpy-bench/CMakeLists.txt
@@ -1,3 +1,9 @@
+# memcpy_jart.S contains position dependent code
+if (CMAKE_POSITION_INDEPENDENT_CODE OR NOT OS_LINUX OR NOT ARCH_AMD64)
+    message (${RECONFIGURE_MESSAGE_LEVEL} "Not building memcpy-bench due to PIC enabled or not Linux or not x86")
+    return ()
+endif ()
+
 enable_language(ASM)
 
 clickhouse_add_executable (memcpy-bench

From 3745783a224634fd0e37e0420f42e5a164ef4aa4 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Wed, 3 May 2023 03:09:50 +0000
Subject: [PATCH 234/689] Make 'exceptions shorter than 30' test less noisy

---
 tests/clickhouse-test                         | 47 ++++-------------
 ..._log_and_exception_messages_formatting.sql | 52 ++++++-------------
 2 files changed, 27 insertions(+), 72 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index acc8688cc4a..87a02d8c192 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -2084,40 +2084,14 @@ def reportLogStats(args):
     print("\n")
 
     query = """
-        WITH ('', '({}) Keys: {}', '({}) {}', 'Aggregating', 'Became leader', 'Cleaning queue', 'Creating set.',
-              'Cyclic aliases', 'Detaching {}', 'Executing {}', 'Fire events: {}', 'Found part {}', 'Loaded queue',
-              'No sharding key', 'No tables', 'Query: {}', 'Removed', 'Removed part {}', 'Removing parts.',
-              'Request URI: {}', 'Sending part {}', 'Sent handshake', 'Starting {}', 'Will mimic {}', 'Writing to {}',
-              'dropIfEmpty', 'loadAll {}', '{} ({}:{})', '{} -> {}', '{} {}', '{}: {}', 'Query was cancelled',
-              'Table {} already exists.', '{}%', 'Cancelled merging parts', 'All replicas are lost',
-              'Cancelled mutating parts', 'Read object: {}', 'New segment: {}', 'Unknown geometry type {}',
-              'Table {} is not replicated', '{} {}.{} already exists', 'Attempt to read after eof',
-              'Replica {} already exists', 'Convert overflow', 'key must be a tuple', 'Division by zero',
-              'No part {} in committed state', 'Files set to {}', 'Bytes set to {}', 'Sharding key {} is not used',
-              'Cannot parse datetime', 'Bad get: has {}, requested {}', 'There is no {} in {}', 'Numeric overflow',
-              'Polygon is not valid: {}', 'Decimal math overflow', '{} only accepts maps', 'Dictionary ({}) not found',
-              'Unknown format {}', 'Invalid IPv4 value', 'Invalid IPv6 value', 'Unknown setting {}',
-              'Unknown table function {}', 'Database {} already exists.', 'Table {} doesn''t exist',
-              'Invalid credentials', 'Part {} already exists', 'Invalid mode: {}', 'Log pulling is cancelled',
-              'JOIN {} cannot get JOIN keys', 'Unknown function {}{}', 'Cannot parse IPv6 {}',
-              'Not found address of host: {}', '{} must contain a tuple', 'Unknown codec family: {}',
-              'Expected const String column', 'Invalid partition format: {}', 'Cannot parse IPv4 {}',
-              'AST is too deep. Maximum: {}', 'Array sizes are too large: {}', 'Unable to connect to HDFS: {}',
-              'Shutdown is called for table', 'File is not inside {}',
-              'Table {} doesn''t exist', 'Database {} doesn''t exist', 'Table {}.{} doesn''t exist',
-              'File {} doesn''t exist', 'No such attribute ''{}''', 'User name ''{}'' is reserved',
-              'Could not find table: {}', 'Detached part "{}" not found', 'Unknown data type family: {}',
-              'Unknown input format {}', 'Cannot UPDATE key column {}', 'Substitution {} is not set',
-              'Cannot OPTIMIZE table: {}', 'User name is empty', 'Table name is empty', 'AST is too big. Maximum: {}',
-              'Unsupported cipher mode', 'Unknown explain kind ''{}''', 'Table {} was suddenly removed',
-              'No cache found by path: {}', 'No such column {} in table {}', 'There is no port named {}',
-              'Function {} cannot resize {}', 'Function {} is not parametric', 'Unknown key attribute ''{}''',
-              'Transaction was cancelled', 'Unknown parent id {}', 'Session {} not found', 'Mutation {} was killed',
-              'Table {}.{} doesn''t exist.', 'Table is not initialized yet', '{} is not an identifier',
-              'Column ''{}'' already exists', 'No macro {} in config', 'Invalid origin H3 index: {}',
-              'Invalid session timeout: ''{}''', 'Tuple cannot be empty', 'Database name is empty',
-              'Table {} is not a Dictionary', 'Expected function, got: {}', 'Unknown identifier: ''{}''',
-              'Failed to {} input ''{}''', '{}.{} is not a VIEW', 'Cannot convert NULL to {}', 'Dictionary {} doesn''t exist',
+        WITH ('', '({}) Keys: {}', '({}) {}', 'Aggregating', 'Became leader', 'Cleaning queue',
+              'Creating set.', 'Cyclic aliases', 'Detaching {}', 'Executing {}', 'Fire events: {}',
+              'Found part {}', 'Loaded queue', 'No sharding key', 'No tables', 'Query: {}',
+              'Removed', 'Removed part {}', 'Removing parts.', 'Request URI: {}', 'Sending part {}',
+              'Sent handshake', 'Starting {}', 'Will mimic {}', 'Writing to {}', 'dropIfEmpty',
+              'loadAll {}', '{} ({}:{})', '{} -> {}', '{} {}', '{}: {}', '{}%', 'Read object: {}',
+              'New segment: {}', 'Convert overflow', 'Division by zero', 'Files set to {}',
+              'Bytes set to {}', 'Numeric overflow', 'Invalid mode: {}',
               'Write file: {}', 'Unable to parse JSONPath', 'Host is empty in S3 URI.', 'Expected end of line',
               'inflate failed: {}{}', 'Center is not valid', 'Column ''{}'' is ambiguous', 'Cannot parse object', 'Invalid date: {}',
               'There is no cache by name: {}', 'No part {} in table', '`{}` should be a String', 'There are duplicate id {}',
@@ -2126,11 +2100,12 @@ def reportLogStats(args):
               'brotli decode error{}', 'Invalid H3 index: {}', 'Too large node state size', 'No additional keys found.',
               'Attempt to read after EOF.', 'Replication was stopped', '{}	building file infos', 'Cannot parse uuid {}'
         ) AS known_short_messages
-        SELECT count() AS c, message_format_string, substr(any(message), 1, 120)
+        SELECT count() AS c, message_format_string, substr(any(message), 1, 120),
+            min(if(length(regexpExtract(message, '(.*)\\([A-Z0-9_]+\\)')) as pref > 0, pref, length(message)) - 26 AS length_without_exception_boilerplate) AS min_length_without_exception_boilerplate
         FROM system.text_log
         WHERE (now() - toIntervalMinute(120)) < event_time
             AND (length(message_format_string) < 16
-                OR (length(message_format_string) < 30 AND message ilike '%DB::Exception%'))
+                OR (message ilike '%DB::Exception%' AND length_without_exception_boilerplate < 30))
             AND message_format_string NOT IN known_short_messages
         GROUP BY message_format_string ORDER BY c DESC LIMIT 50 FORMAT TSVWithNamesAndTypes
     """
diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
index 7796785afb5..8e625766f49 100644
--- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
+++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
@@ -16,40 +16,14 @@ select 'runtime exceptions', max2(coalesce(sum(length(message_format_string) = 0
 
 -- FIXME some of the following messages are not informative and it has to be fixed
 create temporary table known_short_messages (s String) as select * from (select
-['', '({}) Keys: {}', '({}) {}', 'Aggregating', 'Became leader', 'Cleaning queue', 'Creating set.',
-'Cyclic aliases', 'Detaching {}', 'Executing {}', 'Fire events: {}', 'Found part {}', 'Loaded queue',
-'No sharding key', 'No tables', 'Query: {}', 'Removed', 'Removed part {}', 'Removing parts.',
-'Request URI: {}', 'Sending part {}', 'Sent handshake', 'Starting {}', 'Will mimic {}', 'Writing to {}',
-'dropIfEmpty', 'loadAll {}', '{} ({}:{})', '{} -> {}', '{} {}', '{}: {}', 'Query was cancelled',
-'Table {} already exists.', '{}%', 'Cancelled merging parts', 'All replicas are lost',
-'Cancelled mutating parts', 'Read object: {}', 'New segment: {}', 'Unknown geometry type {}',
-'Table {} is not replicated', '{} {}.{} already exists', 'Attempt to read after eof',
-'Replica {} already exists', 'Convert overflow', 'key must be a tuple', 'Division by zero',
-'No part {} in committed state', 'Files set to {}', 'Bytes set to {}', 'Sharding key {} is not used',
-'Cannot parse datetime', 'Bad get: has {}, requested {}', 'There is no {} in {}', 'Numeric overflow',
-'Polygon is not valid: {}', 'Decimal math overflow', '{} only accepts maps', 'Dictionary ({}) not found',
-'Unknown format {}', 'Invalid IPv4 value', 'Invalid IPv6 value', 'Unknown setting {}',
-'Unknown table function {}', 'Database {} already exists.', 'Table {} doesn''t exist',
-'Invalid credentials', 'Part {} already exists', 'Invalid mode: {}', 'Log pulling is cancelled',
-'JOIN {} cannot get JOIN keys', 'Unknown function {}{}', 'Cannot parse IPv6 {}',
-'Not found address of host: {}', '{} must contain a tuple', 'Unknown codec family: {}',
-'Expected const String column', 'Invalid partition format: {}', 'Cannot parse IPv4 {}',
-'AST is too deep. Maximum: {}', 'Array sizes are too large: {}', 'Unable to connect to HDFS: {}',
-'Shutdown is called for table', 'File is not inside {}',
-'Table {} doesn''t exist', 'Database {} doesn''t exist', 'Table {}.{} doesn''t exist',
-'File {} doesn''t exist', 'No such attribute ''{}''', 'User name ''{}'' is reserved',
-'Could not find table: {}', 'Detached part "{}" not found', 'Unknown data type family: {}',
-'Unknown input format {}', 'Cannot UPDATE key column {}', 'Substitution {} is not set',
-'Cannot OPTIMIZE table: {}', 'User name is empty', 'Table name is empty', 'AST is too big. Maximum: {}',
-'Unsupported cipher mode', 'Unknown explain kind ''{}''', 'Table {} was suddenly removed',
-'No cache found by path: {}', 'No such column {} in table {}', 'There is no port named {}',
-'Function {} cannot resize {}', 'Function {} is not parametric', 'Unknown key attribute ''{}''',
-'Transaction was cancelled', 'Unknown parent id {}', 'Session {} not found', 'Mutation {} was killed',
-'Table {}.{} doesn''t exist.', 'Table is not initialized yet', '{} is not an identifier',
-'Column ''{}'' already exists', 'No macro {} in config', 'Invalid origin H3 index: {}',
-'Invalid session timeout: ''{}''', 'Tuple cannot be empty', 'Database name is empty',
-'Table {} is not a Dictionary', 'Expected function, got: {}', 'Unknown identifier: ''{}''',
-'Failed to {} input ''{}''', '{}.{} is not a VIEW', 'Cannot convert NULL to {}', 'Dictionary {} doesn''t exist',
+['', '({}) Keys: {}', '({}) {}', 'Aggregating', 'Became leader', 'Cleaning queue',
+'Creating set.', 'Cyclic aliases', 'Detaching {}', 'Executing {}', 'Fire events: {}',
+'Found part {}', 'Loaded queue', 'No sharding key', 'No tables', 'Query: {}', 'Removed',
+'Removed part {}', 'Removing parts.', 'Request URI: {}', 'Sending part {}',
+'Sent handshake', 'Starting {}', 'Will mimic {}', 'Writing to {}', 'dropIfEmpty',
+'loadAll {}', '{} ({}:{})', '{} -> {}', '{} {}', '{}: {}', '{}%', 'Read object: {}',
+'New segment: {}', 'Convert overflow', 'Division by zero', 'Files set to {}',
+'Bytes set to {}', 'Numeric overflow', 'Invalid mode: {}',
 'Write file: {}', 'Unable to parse JSONPath', 'Host is empty in S3 URI.', 'Expected end of line',
 'inflate failed: {}{}', 'Center is not valid', 'Column ''{}'' is ambiguous', 'Cannot parse object', 'Invalid date: {}',
 'There is no cache by name: {}', 'No part {} in table', '`{}` should be a String', 'There are duplicate id {}',
@@ -65,8 +39,14 @@ select 'messages shorter than 10', max2(countDistinctOrDefault(message_format_st
 -- Same as above. Feel free to update the threshold or remove this query if really necessary
 select 'messages shorter than 16', max2(countDistinctOrDefault(message_format_string), 3) from logs where length(message_format_string) < 16 and message_format_string not in known_short_messages;
 
--- Same as above, but exceptions must be more informative. Feel free to update the threshold or remove this query if really necessary
-select 'exceptions shorter than 30', max2(countDistinctOrDefault(message_format_string), 3) from logs where length(message_format_string) < 30 and message ilike '%DB::Exception%' and message_format_string not in known_short_messages;
+-- Unlike above, here we look at length of the formatted message, not format string. Most short format strings are fine because they end up decorated with context from outer or inner exceptions, e.g.:
+-- "Expected end of line" -> "Code: 117. DB::Exception: Expected end of line: (in file/uri /var/lib/clickhouse/user_files/data_02118): (at row 1)"
+-- But we have to cut out the boilerplate, e.g.:
+-- "Code: 60. DB::Exception: Table default.a doesn't exist. (UNKNOWN_TABLE), Stack trace" -> "Table default.a doesn't exist."
+-- This table currently doesn't have enough information to do this reliably, so we just regex search for " (ERROR_NAME_IN_CAPS)" and hope that's good enough.
+-- For the "Code: 123. DB::Exception: " part, we just subtract 26 instead of searching for it. Because sometimes it's not at the start, e.g.:
+-- "Unexpected error, will try to restart main thread: Code: 341. DB::Exception: Unexpected error: Code: 57. DB::Exception:[...]"
+select 'exceptions shorter than 30', max2(countDistinctOrDefault(message_format_string), 3) from logs where message ilike '%DB::Exception%' and if(length(regexpExtract(message, '(.*)\\([A-Z0-9_]+\\)')) as pref > 0, pref, length(message)) < 30 + 26 and message_format_string not in known_short_messages;
 
 
 -- Avoid too noisy messages: top 1 message frequency must be less than 30%. We should reduce the threshold

From 49394a097e091cd298d3a046c032a9dde783179a Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Wed, 3 May 2023 22:19:59 +0000
Subject: [PATCH 235/689] Fix 'noisy Warning messages' failing when there are
 no Warning messages

---
 src/Dictionaries/RegExpTreeDictionary.cpp      |  2 +-
 tests/clickhouse-test                          | 18 ++++++++++++------
 ...2_log_and_exception_messages_formatting.sql | 14 ++++++++++----
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp
index 8263c87d0e6..9edc06cd4ce 100644
--- a/src/Dictionaries/RegExpTreeDictionary.cpp
+++ b/src/Dictionaries/RegExpTreeDictionary.cpp
@@ -264,7 +264,7 @@ void RegExpTreeDictionary::initGraph()
         if (regex_nodes.contains(pid))
             regex_nodes[pid]->children.push_back(id);
         else
-            throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "Unknown parent id {}", pid);
+            throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "Unknown parent id {} in regexp tree dictionary", pid);
     }
     std::set<UInt64> visited;
     UInt64 topology_id = 0;
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 87a02d8c192..e61749e5377 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -2022,7 +2022,7 @@ def reportLogStats(args):
 
     query = """
         WITH
-            120 AS mins,
+            240 AS mins,
             (
                 SELECT (count(), sum(length(message)))
                 FROM system.text_log
@@ -2052,7 +2052,7 @@ def reportLogStats(args):
 
     query = """
         WITH
-            120 AS mins
+            240 AS mins
         SELECT
             count() AS count,
             substr(replaceRegexpAll(message, '[^A-Za-z]+', ''), 1, 32) AS pattern,
@@ -2073,7 +2073,7 @@ def reportLogStats(args):
     query = """
         SELECT message_format_string, count(), substr(any(message), 1, 120) AS any_message
         FROM system.text_log
-        WHERE (now() - toIntervalMinute(120)) < event_time
+        WHERE (now() - toIntervalMinute(240)) < event_time
         AND (message NOT LIKE (replaceRegexpAll(message_format_string, '{[:.0-9dfx]*}', '%') AS s))
         AND (message NOT LIKE concat('%Exception: ', s, '%'))
         GROUP BY message_format_string ORDER BY count() DESC LIMIT 20 FORMAT TSVWithNamesAndTypes
@@ -2098,12 +2098,18 @@ def reportLogStats(args):
               'Invalid replica name: {}', 'Unexpected value {} in enum', 'Unknown BSON type: {}', 'Point is not valid',
               'Invalid qualified name: {}', 'INTO OUTFILE is not allowed', 'Arguments must not be NaN', 'Cell is not valid',
               'brotli decode error{}', 'Invalid H3 index: {}', 'Too large node state size', 'No additional keys found.',
-              'Attempt to read after EOF.', 'Replication was stopped', '{}	building file infos', 'Cannot parse uuid {}'
+              'Attempt to read after EOF.', 'Replication was stopped', '{}	building file infos', 'Cannot parse uuid {}',
+              'Query was cancelled', 'Cancelled merging parts', 'Cancelled mutating parts', 'Log pulling is cancelled',
+              'Transaction was cancelled', 'Could not find table: {}', 'Table {} doesn''t exist',
+              'Database {} doesn''t exist', 'Dictionary ({}) not found', 'Unknown table function {}',
+              'Unknown format {}', 'Unknown explain kind ''{}''', 'Unknown setting {}', 'Unknown input format {}',
+              'Unknown identifier: ''{}''', 'User name is empty', 'Expected function, got: {}',
+              'Attempt to read after eof', 'String size is too big ({}), maximum: {}'
         ) AS known_short_messages
         SELECT count() AS c, message_format_string, substr(any(message), 1, 120),
             min(if(length(regexpExtract(message, '(.*)\\([A-Z0-9_]+\\)')) as pref > 0, pref, length(message)) - 26 AS length_without_exception_boilerplate) AS min_length_without_exception_boilerplate
         FROM system.text_log
-        WHERE (now() - toIntervalMinute(120)) < event_time
+        WHERE (now() - toIntervalMinute(240)) < event_time
             AND (length(message_format_string) < 16
                 OR (message ilike '%DB::Exception%' AND length_without_exception_boilerplate < 30))
             AND message_format_string NOT IN known_short_messages
@@ -2117,7 +2123,7 @@ def reportLogStats(args):
     query = """
         SELECT max((freq, message_format_string)), level
         FROM (SELECT count() / (SELECT count() FROM system.text_log
-              WHERE (now() - toIntervalMinute(120)) < event_time) AS freq,
+              WHERE (now() - toIntervalMinute(240)) < event_time) AS freq,
               min(level) AS level, message_format_string FROM system.text_log
               WHERE (now() - toIntervalMinute(120)) < event_time
               GROUP BY message_format_string ORDER BY freq DESC)
diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
index 8e625766f49..acb6117f937 100644
--- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
+++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
@@ -30,7 +30,13 @@ create temporary table known_short_messages (s String) as select * from (select
 'Invalid replica name: {}', 'Unexpected value {} in enum', 'Unknown BSON type: {}', 'Point is not valid',
 'Invalid qualified name: {}', 'INTO OUTFILE is not allowed', 'Arguments must not be NaN', 'Cell is not valid',
 'brotli decode error{}', 'Invalid H3 index: {}', 'Too large node state size', 'No additional keys found.',
-'Attempt to read after EOF.', 'Replication was stopped', '{}	building file infos', 'Cannot parse uuid {}'
+'Attempt to read after EOF.', 'Replication was stopped', '{}	building file infos', 'Cannot parse uuid {}',
+'Query was cancelled', 'Cancelled merging parts', 'Cancelled mutating parts', 'Log pulling is cancelled',
+'Transaction was cancelled', 'Could not find table: {}', 'Table {} doesn''t exist',
+'Database {} doesn''t exist', 'Dictionary ({}) not found', 'Unknown table function {}',
+'Unknown format {}', 'Unknown explain kind ''{}''', 'Unknown setting {}', 'Unknown input format {}',
+'Unknown identifier: ''{}''', 'User name is empty', 'Expected function, got: {}',
+'Attempt to read after eof', 'String size is too big ({}), maximum: {}'
 ] as arr) array join arr;
 
 -- Check that we don't have too many short meaningless message patterns.
@@ -65,11 +71,11 @@ select 'noisy Info messages', max2((select count() from logs where level <= 'Inf
 
 -- Same as above for Warning
 with ('Not enabled four letter command {}') as frequent_in_tests
-select 'noisy Warning messages', max2((select countOrDefault() from logs where level = 'Warning' and message_format_string not in frequent_in_tests
-    group by message_format_string order by count() desc limit 1) / (select count() from logs), 0.01);
+select 'noisy Warning messages', max2(coalesce((select countOrDefault() from logs where level = 'Warning' and message_format_string not in frequent_in_tests
+    group by message_format_string order by count() desc limit 1), 0) / (select count() from logs), 0.01);
 
 -- Same as above for Error
-select 'noisy Error messages', max2((select countOrDefault() from logs where level = 'Error' group by message_format_string order by count() desc limit 1) / (select count() from logs), 0.02);
+select 'noisy Error messages', max2(coalesce((select countOrDefault() from logs where level = 'Error' group by message_format_string order by count() desc limit 1), 0) / (select count() from logs), 0.02);
 
 select 'no Fatal messages', count() from logs where level = 'Fatal';
 

From 2986c28761ad0d5d1d4b3aed3764f1f2f26c48a7 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 6 May 2023 18:10:17 +0000
Subject: [PATCH 236/689] Small fixes

---
 .../sql-reference/functions/date-time-functions.md   |  6 ++++++
 src/Functions/makeDate.cpp                           | 12 +++++-------
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md
index 0e899045c7c..5a90200d60b 100644
--- a/docs/en/sql-reference/functions/date-time-functions.md
+++ b/docs/en/sql-reference/functions/date-time-functions.md
@@ -108,6 +108,12 @@ Result:
 
 Like [makeDateTime](#makedatetime) but produces a [DateTime64](../../sql-reference/data-types/datetime64.md).
 
+**Syntax**
+
+``` sql
+makeDateTime32(year, month, day, hour, minute, second[, fraction[, precision[, timezone]]])
+```
+
 ## timeZone
 
 Returns the timezone of the server.
diff --git a/src/Functions/makeDate.cpp b/src/Functions/makeDate.cpp
index 97271386286..2fd76c995b3 100644
--- a/src/Functions/makeDate.cpp
+++ b/src/Functions/makeDate.cpp
@@ -51,7 +51,7 @@ public:
 
 protected:
     template <class ArgumentNames>
-    void checkRequiredArguments(const ColumnsWithTypeAndName & arguments, const ArgumentNames & argument_names, const size_t optional_argument_count) const
+    void checkRequiredArguments(const ColumnsWithTypeAndName & arguments, const ArgumentNames & argument_names, size_t optional_argument_count) const
     {
         if (arguments.size() < argument_names.size() || arguments.size() > argument_names.size() + optional_argument_count)
             throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
@@ -63,7 +63,7 @@ protected:
             DataTypePtr argument_type = arguments[i].type;
             if (!isNumber(argument_type))
                 throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "Argument '{}' for function {} must be a number", std::string(argument_names[i]), getName());
+                    "Argument '{}' for function {} must be a number", argument_names[i], getName());
         }
     }
 
@@ -145,7 +145,6 @@ public:
     }
 };
 
-/// makeDate(year, month, day)
 struct MakeDateTraits
 {
     static constexpr auto name = "makeDate";
@@ -157,7 +156,6 @@ struct MakeDateTraits
     static constexpr auto MAX_DATE = YearMonthDayToSingleInt(MAX_YEAR, 6, 6);
 };
 
-/// makeDate32(year, month, day)
 struct MakeDate32Traits
 {
     static constexpr auto name = "makeDate32";
@@ -180,7 +178,7 @@ public:
     size_t getNumberOfArguments() const override { return 0; }
 
 protected:
-    void checkRequiredArguments(const ColumnsWithTypeAndName & arguments, const size_t optional_argument_count) const
+    void checkRequiredArguments(const ColumnsWithTypeAndName & arguments, size_t optional_argument_count) const
     {
         FunctionWithNumericParamsBase::checkRequiredArguments(arguments, argument_names, optional_argument_count);
     }
@@ -235,7 +233,7 @@ protected:
 class FunctionMakeDateTime : public FunctionMakeDateTimeBase
 {
 private:
-    static constexpr std::array<const char*, 1> optional_argument_names = {"timezone"};
+    static constexpr std::array optional_argument_names = {"timezone"};
 
 public:
     static constexpr auto name = "makeDateTime";
@@ -304,7 +302,7 @@ public:
 class FunctionMakeDateTime64 : public FunctionMakeDateTimeBase
 {
 private:
-    static constexpr std::array<const char*, 3> optional_argument_names = {"fraction", "precision", "timezone"};
+    static constexpr std::array optional_argument_names = {"fraction", "precision", "timezone"};
     static constexpr UInt8 DEFAULT_PRECISION = 3;
 
 public:

From 2228107134a141f44aab6e4cc6bff3c6f3564ead Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Sat, 6 May 2023 20:12:26 +0200
Subject: [PATCH 237/689] Better name

---
 src/Disks/IO/getThreadPoolReader.cpp | 8 ++++----
 src/Disks/IO/getThreadPoolReader.h   | 2 +-
 src/Interpreters/Context.cpp         | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/Disks/IO/getThreadPoolReader.cpp b/src/Disks/IO/getThreadPoolReader.cpp
index d604c934e5c..a28501f4dbd 100644
--- a/src/Disks/IO/getThreadPoolReader.cpp
+++ b/src/Disks/IO/getThreadPoolReader.cpp
@@ -32,24 +32,24 @@ IAsynchronousReader & getThreadPoolReader(FilesystemReaderType type)
     {
         case FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER:
         {
-            static auto asynchronous_remote_fs_reader = getThreadPoolReaderImpl(type, config);
+            static auto asynchronous_remote_fs_reader = createThreadPoolReader(type, config);
             return *asynchronous_remote_fs_reader;
         }
         case FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER:
         {
-            static auto asynchronous_local_fs_reader = getThreadPoolReaderImpl(type, config);
+            static auto asynchronous_local_fs_reader = createThreadPoolReader(type, config);
             return *asynchronous_local_fs_reader;
         }
         case FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER:
         {
-            static auto synchronous_local_fs_reader = getThreadPoolReaderImpl(type, config);
+            static auto synchronous_local_fs_reader = createThreadPoolReader(type, config);
             return *synchronous_local_fs_reader;
         }
     }
 #endif
 }
 
-std::unique_ptr<IAsynchronousReader> getThreadPoolReaderImpl(
+std::unique_ptr<IAsynchronousReader> createThreadPoolReader(
     FilesystemReaderType type, const Poco::Util::AbstractConfiguration & config)
 {
     switch (type)
diff --git a/src/Disks/IO/getThreadPoolReader.h b/src/Disks/IO/getThreadPoolReader.h
index 54dbca7585d..243e0e5478f 100644
--- a/src/Disks/IO/getThreadPoolReader.h
+++ b/src/Disks/IO/getThreadPoolReader.h
@@ -16,7 +16,7 @@ enum class FilesystemReaderType
 
 IAsynchronousReader & getThreadPoolReader(FilesystemReaderType type);
 
-std::unique_ptr<IAsynchronousReader> getThreadPoolReaderImpl(
+std::unique_ptr<IAsynchronousReader> createThreadPoolReader(
     FilesystemReaderType type,
     const Poco::Util::AbstractConfiguration & config);
 
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index aae5544ba90..99cac230859 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -4178,20 +4178,20 @@ IAsynchronousReader & Context::getThreadPoolReader(FilesystemReaderType type) co
         case FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER:
         {
             if (!shared->asynchronous_remote_fs_reader)
-                shared->asynchronous_remote_fs_reader = getThreadPoolReaderImpl(type, getConfigRef());
+                shared->asynchronous_remote_fs_reader = createThreadPoolReader(type, getConfigRef());
             return *shared->asynchronous_remote_fs_reader;
         }
         case FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER:
         {
             if (!shared->asynchronous_local_fs_reader)
-                shared->asynchronous_local_fs_reader = getThreadPoolReaderImpl(type, getConfigRef());
+                shared->asynchronous_local_fs_reader = createThreadPoolReader(type, getConfigRef());
 
             return *shared->asynchronous_local_fs_reader;
         }
         case FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER:
         {
             if (!shared->synchronous_local_fs_reader)
-                shared->synchronous_local_fs_reader = getThreadPoolReaderImpl(type, getConfigRef());
+                shared->synchronous_local_fs_reader = createThreadPoolReader(type, getConfigRef());
 
             return *shared->synchronous_local_fs_reader;
         }

From 61f1861d7070fb37b276c69c31959901edc94139 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 20:54:52 +0200
Subject: [PATCH 238/689] Fix error

---
 src/Common/StackTrace.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp
index 033581dfc86..e6b7839c4e2 100644
--- a/src/Common/StackTrace.cpp
+++ b/src/Common/StackTrace.cpp
@@ -344,7 +344,6 @@ toStringEveryLineImpl([[maybe_unused]] bool fatal, const StackTraceRefTriple & s
         return callback("<Empty trace>");
 
 #if defined(__ELF__) && !defined(OS_FREEBSD)
-    DB::WriteBufferFromOwnString out;
 
     using enum DB::Dwarf::LocationInfoMode;
     const auto mode = fatal ? FULL_WITH_INLINE : FAST;
@@ -361,6 +360,7 @@ toStringEveryLineImpl([[maybe_unused]] bool fatal, const StackTraceRefTriple & s
         uintptr_t virtual_offset = object ? uintptr_t(object->address_begin) : 0;
         const void * physical_addr = reinterpret_cast<const void *>(uintptr_t(virtual_addr) - virtual_offset);
 
+        DB::WriteBufferFromOwnString out;
         out << i << ". ";
 
         if (std::error_code ec; object && std::filesystem::exists(object->name, ec) && !ec)

From c893302a08d4cdd818e78917b8c35a5fb35c560f Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 6 May 2023 18:50:25 +0000
Subject: [PATCH 239/689] Implement a MySQL-compatible variant of makeDate()

Fixes #49143
---
 .../functions/date-time-functions.md          |  27 +-
 src/Functions/makeDate.cpp                    | 267 ++++++++++--------
 .../0_stateless/02242_make_date.reference     |   2 +
 tests/queries/0_stateless/02242_make_date.sql |   4 +-
 .../02242_make_date_mysql.reference           |  25 ++
 .../0_stateless/02242_make_date_mysql.sql     |  41 +++
 .../queries/0_stateless/02243_make_date32.sql |   1 -
 .../02243_make_date32_mysql.reference         |  26 ++
 .../0_stateless/02243_make_date32_mysql.sql   |  42 +++
 9 files changed, 311 insertions(+), 124 deletions(-)
 create mode 100644 tests/queries/0_stateless/02242_make_date_mysql.reference
 create mode 100644 tests/queries/0_stateless/02242_make_date_mysql.sql
 create mode 100644 tests/queries/0_stateless/02243_make_date32_mysql.reference
 create mode 100644 tests/queries/0_stateless/02243_make_date32_mysql.sql

diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md
index 5a90200d60b..599b64ef43f 100644
--- a/docs/en/sql-reference/functions/date-time-functions.md
+++ b/docs/en/sql-reference/functions/date-time-functions.md
@@ -26,19 +26,27 @@ SELECT
 
 ## makeDate
 
-Creates a [Date](../../sql-reference/data-types/date.md) from a year, month and day argument.
+Creates a [Date](../../sql-reference/data-types/date.md)
+- from a year, month and day argument, or
+- from a year and day of year argument.
 
 **Syntax**
 
 ``` sql
-makeDate(year, month, day)
+makeDate(year, month, day);
+makeDate(year, day_of_year);
 ```
 
+Alias:
+- `MAKEDATE(year, month, day);`
+- `MAKEDATE(year, day_of_year);`
+
 **Arguments**
 
 - `year` — Year. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
 - `month` — Month. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
 - `day` — Day. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
+- `day_of_year` — Day of the year. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
 
 **Returned value**
 
@@ -48,6 +56,8 @@ Type: [Date](../../sql-reference/data-types/date.md).
 
 **Example**
 
+Create a Date from a year, month and day:
+
 ``` sql
 SELECT makeDate(2023, 2, 28) AS Date;
 ```
@@ -60,6 +70,19 @@ Result:
 └────────────┘
 ```
 
+Create a Date from a year and day of year argument:
+
+``` sql
+SELECT makeDate(2023, 42) AS Date;
+```
+
+Result:
+
+``` text
+┌───────date─┐
+│ 2023-02-11 │
+└────────────┘
+```
 ## makeDate32
 
 Like [makeDate](#makeDate) but produces a [Date32](../../sql-reference/data-types/date32.md).
diff --git a/src/Functions/makeDate.cpp b/src/Functions/makeDate.cpp
index 2fd76c995b3..ae12659c8de 100644
--- a/src/Functions/makeDate.cpp
+++ b/src/Functions/makeDate.cpp
@@ -28,13 +28,7 @@ namespace ErrorCodes
 namespace
 {
 
-/// A helper function to simplify comparisons of valid YYYY-MM-DD values for <,>,=
-inline constexpr Int64 YearMonthDayToSingleInt(Int64 year, Int64 month, Int64 day)
-{
-    return year * 512 + month * 32 + day;
-}
-
-/// Common logic to handle numeric arguments like year, month, day, hour, minute, second
+/// Functions common to makeDate, makeDate32, makeDateTime, makeDateTime64
 class FunctionWithNumericParamsBase : public IFunction
 {
 public:
@@ -49,9 +43,151 @@ public:
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
+    bool isVariadic() const override { return true; }
+
+    size_t getNumberOfArguments() const override { return 0; }
+
 protected:
-    template <class ArgumentNames>
-    void checkRequiredArguments(const ColumnsWithTypeAndName & arguments, const ArgumentNames & argument_names, size_t optional_argument_count) const
+};
+
+/// Common implementation for makeDate, makeDate32
+template <typename Traits>
+class FunctionMakeDate : public FunctionWithNumericParamsBase
+{
+private:
+    static constexpr std::array argument_names_month_day = {"year", "month", "day"};
+    static constexpr std::array argument_names_dayofyear = {"year", "dayofyear"};
+
+public:
+    static constexpr auto name = Traits::name;
+
+    static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionMakeDate>(); }
+
+    String getName() const override { return name; }
+
+    DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
+    {
+        if (arguments.size() != argument_names_month_day.size() && arguments.size() != argument_names_dayofyear.size())
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+                "Function {} requires {} or {} arguments, but {} given",
+                getName(), argument_names_month_day.size(), argument_names_dayofyear.size(), arguments.size());
+
+        for (size_t i = 0; i < arguments.size(); ++i)
+        {
+            DataTypePtr argument_type = arguments[i].type;
+            if (!isNumber(argument_type))
+                    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                        "Argument '{}' for function {} must be a number",
+                        (arguments.size() == argument_names_month_day.size()) ? argument_names_month_day[i] : argument_names_dayofyear[i], getName());
+
+        }
+
+        return std::make_shared<typename Traits::ReturnDataType>();
+    }
+
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
+    {
+        Columns converted_arguments;
+        const DataTypePtr converted_argument_type = std::make_shared<DataTypeFloat32>();
+        converted_arguments.clear();
+        for (const auto & argument : arguments)
+        {
+            ColumnPtr argument_column = castColumn(argument, converted_argument_type);
+            argument_column = argument_column->convertToFullColumnIfConst();
+            converted_arguments.push_back(argument_column);
+        }
+
+        auto res_column = Traits::ReturnDataType::ColumnType::create(input_rows_count);
+        auto & result_data = res_column->getData();
+
+        const auto & date_lut = DateLUT::instance();
+        const Int32 max_days_since_epoch = date_lut.makeDayNum(Traits::MAX_DATE[0], Traits::MAX_DATE[1], Traits::MAX_DATE[2]);
+
+        if (converted_arguments.size() == argument_names_month_day.size())
+        {
+            const auto & year_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[0]).getData();
+            const auto & month_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[1]).getData();
+            const auto & day_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[2]).getData();
+
+            for (size_t i = 0; i < input_rows_count; ++i)
+            {
+                const auto year = year_data[i];
+                const auto month = month_data[i];
+                const auto day = day_data[i];
+
+                Int32 day_num = 0;
+
+                if (year >= Traits::MIN_YEAR &&
+                    year <= Traits::MAX_YEAR &&
+                    month >= 1 && month <= 12 &&
+                    day >= 1 && day <= 31)
+                {
+                    Int32 days_since_epoch = date_lut.makeDayNum(static_cast<Int16>(year), static_cast<UInt8>(month), static_cast<UInt8>(day));
+                    if (days_since_epoch <= max_days_since_epoch)
+                        day_num = days_since_epoch;
+                }
+
+                result_data[i] = day_num;
+            }
+        }
+        else
+        {
+            /// case argument_names_dayofyear:
+            const auto & year_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[0]).getData();
+            const auto & dayofyear_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[1]).getData();
+
+            for (size_t i = 0; i < input_rows_count; ++i)
+            {
+                const auto year = year_data[i];
+                const auto dayofyear = dayofyear_data[i];
+
+                Int32 day_num = 0;
+
+                if (year >= Traits::MIN_YEAR &&
+                    year <= Traits::MAX_YEAR &&
+                    dayofyear >= 1 && dayofyear <= 365)
+                {
+                    Int32 days_since_epoch = date_lut.makeDayNum(static_cast<Int16>(year), 1, 1) + static_cast<Int32>(dayofyear) - 1;
+                    if (days_since_epoch <= max_days_since_epoch)
+                        day_num = days_since_epoch;
+                }
+
+                result_data[i] = day_num;
+            }
+        }
+
+        return res_column;
+    }
+};
+
+struct MakeDateTraits
+{
+    static constexpr auto name = "makeDate";
+    using ReturnDataType = DataTypeDate;
+
+    static constexpr auto MIN_YEAR = 1970;
+    static constexpr auto MAX_YEAR = 2149;
+    /// This date has the maximum day number that fits in 16-bit uint
+    static constexpr std::array MAX_DATE = {MAX_YEAR, 6, 6};
+};
+
+struct MakeDate32Traits
+{
+    static constexpr auto name = "makeDate32";
+    using ReturnDataType = DataTypeDate32;
+
+    static constexpr auto MIN_YEAR = 1900;
+    static constexpr auto MAX_YEAR = 2299;
+    static constexpr std::array MAX_DATE = {MAX_YEAR, 12, 31};
+};
+
+/// Common implementation for makeDateTime, makeDateTime64
+class FunctionMakeDateTimeBase : public FunctionWithNumericParamsBase
+{
+protected:
+    static constexpr std::array argument_names = {"year", "month", "day", "hour", "minute", "second"};
+
+    void checkRequiredArguments(const ColumnsWithTypeAndName & arguments, size_t optional_argument_count) const
     {
         if (arguments.size() < argument_names.size() || arguments.size() > argument_names.size() + optional_argument_count)
             throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
@@ -67,12 +203,10 @@ protected:
         }
     }
 
-    template <class ArgumentNames>
-    void convertRequiredArguments(const ColumnsWithTypeAndName & arguments, const ArgumentNames & argument_names, Columns & converted_arguments) const
+    void convertRequiredArguments(const ColumnsWithTypeAndName & arguments, Columns & converted_arguments) const
     {
         const DataTypePtr converted_argument_type = std::make_shared<DataTypeFloat32>();
         converted_arguments.clear();
-        converted_arguments.reserve(arguments.size());
         for (size_t i = 0; i < argument_names.size(); ++i)
         {
             ColumnPtr argument_column = castColumn(arguments[i], converted_argument_type);
@@ -80,113 +214,6 @@ protected:
             converted_arguments.push_back(argument_column);
         }
     }
-};
-
-/// Common implementation for makeDate, makeDate32
-template <typename Traits>
-class FunctionMakeDate : public FunctionWithNumericParamsBase
-{
-private:
-    static constexpr std::array argument_names = {"year", "month", "day"};
-
-public:
-    static constexpr auto name = Traits::name;
-
-    static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionMakeDate>(); }
-
-    String getName() const override { return name; }
-
-    bool isVariadic() const override { return false; }
-
-    size_t getNumberOfArguments() const override { return argument_names.size(); }
-
-    DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
-    {
-        checkRequiredArguments(arguments, argument_names, 0);
-
-        return std::make_shared<typename Traits::ReturnDataType>();
-    }
-
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
-    {
-        Columns converted_arguments;
-        convertRequiredArguments(arguments, argument_names, converted_arguments);
-
-        auto res_column = Traits::ReturnDataType::ColumnType::create(input_rows_count);
-        auto & result_data = res_column->getData();
-
-        const auto & year_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[0]).getData();
-        const auto & month_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[1]).getData();
-        const auto & day_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[2]).getData();
-
-        const auto & date_lut = DateLUT::instance();
-
-        for (size_t i = 0; i < input_rows_count; ++i)
-        {
-            const auto year = year_data[i];
-            const auto month = month_data[i];
-            const auto day = day_data[i];
-
-            Int32 day_num = 0;
-
-            if (year >= Traits::MIN_YEAR &&
-                year <= Traits::MAX_YEAR &&
-                month >= 1 && month <= 12 &&
-                day >= 1 && day <= 31 &&
-                YearMonthDayToSingleInt(static_cast<Int64>(year), static_cast<Int64>(month), static_cast<Int64>(day)) <= Traits::MAX_DATE)
-            {
-                day_num = date_lut.makeDayNum(static_cast<Int16>(year), static_cast<UInt8>(month), static_cast<UInt8>(day));
-            }
-
-            result_data[i] = day_num;
-        }
-
-        return res_column;
-    }
-};
-
-struct MakeDateTraits
-{
-    static constexpr auto name = "makeDate";
-    using ReturnDataType = DataTypeDate;
-
-    static constexpr auto MIN_YEAR = 1970;
-    static constexpr auto MAX_YEAR = 2149;
-    /// This date has the maximum day number that fits in 16-bit uint
-    static constexpr auto MAX_DATE = YearMonthDayToSingleInt(MAX_YEAR, 6, 6);
-};
-
-struct MakeDate32Traits
-{
-    static constexpr auto name = "makeDate32";
-    using ReturnDataType = DataTypeDate32;
-
-    static constexpr auto MIN_YEAR = 1900;
-    static constexpr auto MAX_YEAR = 2299;
-    static constexpr auto MAX_DATE = YearMonthDayToSingleInt(MAX_YEAR, 12, 31);
-};
-
-/// Common implementation for makeDateTime, makeDateTime64
-class FunctionMakeDateTimeBase : public FunctionWithNumericParamsBase
-{
-protected:
-    static constexpr std::array argument_names = {"year", "month", "day", "hour", "minute", "second"};
-
-public:
-    bool isVariadic() const override { return true; }
-
-    size_t getNumberOfArguments() const override { return 0; }
-
-protected:
-    void checkRequiredArguments(const ColumnsWithTypeAndName & arguments, size_t optional_argument_count) const
-    {
-        FunctionWithNumericParamsBase::checkRequiredArguments(arguments, argument_names, optional_argument_count);
-    }
-
-    void convertRequiredArguments(const ColumnsWithTypeAndName & arguments, Columns & converted_arguments) const
-    {
-        FunctionWithNumericParamsBase::convertRequiredArguments(arguments, argument_names, converted_arguments);
-    }
 
     template <typename T>
     static Int64 dateTime(T year, T month, T day_of_month, T hour, T minute, T second, const DateLUTImpl & lut)
@@ -437,7 +464,7 @@ private:
 
 REGISTER_FUNCTION(MakeDate)
 {
-    factory.registerFunction<FunctionMakeDate<MakeDateTraits>>();
+    factory.registerFunction<FunctionMakeDate<MakeDateTraits>>({}, FunctionFactory::CaseInsensitive);
     factory.registerFunction<FunctionMakeDate<MakeDate32Traits>>();
     factory.registerFunction<FunctionMakeDateTime>();
     factory.registerFunction<FunctionMakeDateTime64>();
diff --git a/tests/queries/0_stateless/02242_make_date.reference b/tests/queries/0_stateless/02242_make_date.reference
index 0e8500baeb7..dc85d872607 100644
--- a/tests/queries/0_stateless/02242_make_date.reference
+++ b/tests/queries/0_stateless/02242_make_date.reference
@@ -56,6 +56,8 @@ Nullable(Date)
 \N
 \N
 \N
+1980-01-01
+1980-01-01
 1984-02-03
 \N
 \N
diff --git a/tests/queries/0_stateless/02242_make_date.sql b/tests/queries/0_stateless/02242_make_date.sql
index 9e66daa2778..78feabfffb3 100644
--- a/tests/queries/0_stateless/02242_make_date.sql
+++ b/tests/queries/0_stateless/02242_make_date.sql
@@ -79,9 +79,11 @@ select makeDate(1980, NULL, 4);
 select makeDate(1980, 3, NULL);
 
 select makeDate(1980); -- { serverError 42 }
-select makeDate(1980, 1); -- { serverError 42 }
 select makeDate(1980, 1, 1, 1); -- { serverError 42 }
 
+select MAKEDATE(1980, 1, 1);
+select MAKEDATE(1980, 1);
+
 select makeDate(year, month, day) from (select NULL as year, 2 as month, 3 as day union all select 1984 as year, 2 as month, 3 as day) order by year, month, day;
 
 select makeDate(year, month, day) from (select NULL as year, 2 as month, 3 as day union all select NULL as year, 2 as month, 3 as day) order by year, month, day;
diff --git a/tests/queries/0_stateless/02242_make_date_mysql.reference b/tests/queries/0_stateless/02242_make_date_mysql.reference
new file mode 100644
index 00000000000..0fc51594a8c
--- /dev/null
+++ b/tests/queries/0_stateless/02242_make_date_mysql.reference
@@ -0,0 +1,25 @@
+Date
+Nullable(Date)
+Nullable(Date)
+1970-01-01
+2020-01-08
+1970-01-01
+1970-01-01
+1980-01-09
+1970-01-01
+1980-01-09
+1970-01-01
+1980-01-09
+1970-01-01
+1970-01-01
+1970-01-01
+2149-06-06
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-02
+\N
+\N
diff --git a/tests/queries/0_stateless/02242_make_date_mysql.sql b/tests/queries/0_stateless/02242_make_date_mysql.sql
new file mode 100644
index 00000000000..82d80579788
--- /dev/null
+++ b/tests/queries/0_stateless/02242_make_date_mysql.sql
@@ -0,0 +1,41 @@
+select toTypeName(makeDate(1991, 8));
+select toTypeName(makeDate(cast(1991 as Nullable(UInt64)), 8));
+select toTypeName(makeDate(1991, cast(8 as Nullable(UInt64))));
+
+select makeDate(1970, 01);
+select makeDate(2020, 08);
+select makeDate(-1980, 10);
+select makeDate(1980, -10);
+select makeDate(1980.0, 9);
+select makeDate(-1980.0, 9);
+select makeDate(cast(1980.1 as Decimal(20,5)), 9);
+select makeDate(cast('-1980.1' as Decimal(20,5)), 9);
+select makeDate(cast(1980.1 as Float32), 9);
+select makeDate(cast(-1980.1 as Float32), 9);
+
+select makeDate(cast(1980 as Date), 10); -- { serverError 43 }
+select makeDate(cast(-1980 as Date), 10); -- { serverError 43 }
+select makeDate(cast(1980 as Date32), 10); -- { serverError 43 }
+select makeDate(cast(-1980 as Date32), 10); -- { serverError 43 }
+select makeDate(cast(1980 as DateTime), 10); -- { serverError 43 }
+select makeDate(cast(-1980 as DateTime), 10); -- { serverError 43 }
+select makeDate(cast(1980 as DateTime64), 10); -- { serverError 43 }
+select makeDate(cast(-1980 as DateTime64), 10); -- { serverError 43 }
+select makeDate('1980', '10'); -- { serverError 43 }
+select makeDate('-1980', 3); -- { serverError 43 }
+select makeDate('aa', 3); -- { serverError 43 }
+select makeDate(1994, 'aa'); -- { serverError 43 }
+
+select makeDate(0, 1);
+select makeDate(19800, 12);
+select makeDate(2149, 157);
+select makeDate(2149, 158);
+select makeDate(1969,355);
+select makeDate(1969,356);
+select makeDate(1969,357);
+select makeDate(1970,0);
+select makeDate(1970,1);
+select makeDate(1970,2);
+
+select makeDate(NULL, 3);
+select makeDate(1980, NULL);
diff --git a/tests/queries/0_stateless/02243_make_date32.sql b/tests/queries/0_stateless/02243_make_date32.sql
index 2cf4ac6b358..9b0009b33a2 100644
--- a/tests/queries/0_stateless/02243_make_date32.sql
+++ b/tests/queries/0_stateless/02243_make_date32.sql
@@ -78,7 +78,6 @@ select makeDate32(1980, NULL, 4);
 select makeDate32(1980, 3, NULL);
 
 select makeDate32(1980); -- { serverError 42 }
-select makeDate32(1980, 1); -- { serverError 42 }
 select makeDate32(1980, 1, 1, 1); -- { serverError 42 }
 
 select makeDate32(year, month, day) from (select NULL as year, 2 as month, 3 as day union all select 1984 as year, 2 as month, 3 as day) order by year, month, day;
diff --git a/tests/queries/0_stateless/02243_make_date32_mysql.reference b/tests/queries/0_stateless/02243_make_date32_mysql.reference
new file mode 100644
index 00000000000..07c08a2ec70
--- /dev/null
+++ b/tests/queries/0_stateless/02243_make_date32_mysql.reference
@@ -0,0 +1,26 @@
+Date32
+Nullable(Date32)
+Nullable(Date32)
+1900-01-01
+2020-01-08
+1970-01-01
+1970-01-01
+1980-01-09
+1970-01-01
+1980-01-09
+1970-01-01
+1980-01-09
+1970-01-01
+1970-01-01
+1970-01-01
+2299-12-31
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1900-01-01
+1900-01-02
+\N
+\N
diff --git a/tests/queries/0_stateless/02243_make_date32_mysql.sql b/tests/queries/0_stateless/02243_make_date32_mysql.sql
new file mode 100644
index 00000000000..4a67dcd80de
--- /dev/null
+++ b/tests/queries/0_stateless/02243_make_date32_mysql.sql
@@ -0,0 +1,42 @@
+select toTypeName(makeDate32(1991, 8));
+select toTypeName(makeDate32(cast(1991 as Nullable(UInt64)), 8));
+select toTypeName(makeDate32(1991, cast(8 as Nullable(UInt64))));
+
+select makeDate32(1900, 01);
+select makeDate32(2020, 08);
+select makeDate32(-1980, 10);
+select makeDate32(1980, -10);
+select makeDate32(1980.0, 9);
+select makeDate32(-1980.0, 9);
+select makeDate32(cast(1980.1 as Decimal(20,5)), 9);
+select makeDate32(cast('-1980.1' as Decimal(20,5)), 9);
+select makeDate32(cast(1980.1 as Float32), 9);
+select makeDate32(cast(-1980.1 as Float32), 9);
+
+select makeDate32(cast(1980 as Date), 10); -- { serverError 43 }
+select makeDate32(cast(-1980 as Date), 10); -- { serverError 43 }
+select makeDate32(cast(1980 as Date32), 10); -- { serverError 43 }
+select makeDate32(cast(-1980 as Date32), 10); -- { serverError 43 }
+select makeDate32(cast(1980 as DateTime), 10); -- { serverError 43 }
+select makeDate32(cast(-1980 as DateTime), 10); -- { serverError 43 }
+select makeDate32(cast(1980 as DateTime64), 10); -- { serverError 43 }
+select makeDate32(cast(-1980 as DateTime64), 10); -- { serverError 43 }
+select makeDate32('1980', '10'); -- { serverError 43 }
+select makeDate32('-1980', 3); -- { serverError 43 }
+select makeDate32('aa', 3); -- { serverError 43 }
+select makeDate32(1994, 'aa'); -- { serverError 43 }
+
+select makeDate32(0, 1);
+select makeDate32(19800, 12);
+select makeDate32(2299, 365);
+select makeDate32(2299, 366);
+select makeDate32(2300, 1);
+select makeDate32(1899, 365);
+select makeDate32(1899, 366);
+select makeDate32(1899, 367);
+select makeDate32(1900, 0);
+select makeDate32(1900, 1);
+select makeDate32(1900, 2);
+
+select makeDate32(NULL, 3);
+select makeDate32(1980, NULL);

From 76cf206efe9011cdfdeba1a2d075d827587c0eb3 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 6 May 2023 20:26:27 +0000
Subject: [PATCH 240/689] Fix build when memcpy-bench isn't available

---
 utils/memcpy-bench/CMakeLists.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/utils/memcpy-bench/CMakeLists.txt b/utils/memcpy-bench/CMakeLists.txt
index 7227728e849..f2cc4bdc8c5 100644
--- a/utils/memcpy-bench/CMakeLists.txt
+++ b/utils/memcpy-bench/CMakeLists.txt
@@ -1,6 +1,5 @@
 # memcpy_jart.S contains position dependent code
 if (CMAKE_POSITION_INDEPENDENT_CODE OR NOT OS_LINUX OR NOT ARCH_AMD64)
-    message (${RECONFIGURE_MESSAGE_LEVEL} "Not building memcpy-bench due to PIC enabled or not Linux or not x86")
     return ()
 endif ()
 

From 1107988a82a84f3f68d8db835ac5168d1b36e8b4 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 6 May 2023 18:54:56 +0200
Subject: [PATCH 241/689] Improve performance of BLAKE3 by 11% by enabling LTO
 for Rust

LTO in Rust produces multiple definition of `rust_eh_personality' (and
few others), and to overcome this --allow-multiple-definition has been
added.

Query for benchmark:

    SELECT ignore(BLAKE3(materialize('Lorem ipsum dolor sit amet, consectetur adipiscing elit'))) FROM numbers(1000000000) FORMAT `Null`

  upstream           : Elapsed: 2.494 sec. Processed 31.13 million rows, 249.08 MB (12.48 million rows/s., 99.86 MB/s.)
  upstream + rust lto: Elapsed: 13.56 sec. Processed 191.9 million rows, 1.5400 GB (14.15 million rows/s., 113.22 MB/s.)
  llvm BLAKE3        : Elapsed: 3.053 sec. Processed 43.24 million rows, 345.88 MB (14.16 million rows/s., 113.28 MB/s.)

Note, I thought about simply replacing it with BLAKE3 from LLVM, but:
- this will not solve LTO issues for Rust (and in future more libraries
  could be added)
- it makes integrating_rust_libraries.md useless (and there is even blog
  post)

So instead I've decided to add this quirk (--allow-multiple-definition)
to fix builds.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 CMakeLists.txt         | 20 ++++++++++++++++++++
 rust/BLAKE3/Cargo.toml |  7 +++++++
 rust/CMakeLists.txt    | 13 +++++++++++--
 rust/skim/Cargo.toml   |  5 +++++
 4 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 263b202049b..31f58b399db 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -522,6 +522,26 @@ include (cmake/print_flags.cmake)
 
 if (ENABLE_RUST)
     add_subdirectory (rust)
+
+    # With LTO Rust adds few symbols with global visiblity, the most common is
+    # rust_eh_personality. And this leads to linking errors because multiple
+    # Rust libraries contains the same symbol.
+    #
+    # If it was shared library, that we could use version script for linker to
+    # hide this symbols, but libraries are static.
+    #
+    # we could in theory compile everything to one library but this will be a
+    # mess
+    #
+    # But this should be OK since CI has lots of other builds that are done
+    # without LTO and it will find multiple definitions if there will be any.
+    #
+    # More information about this behaviour in Rust can be found here
+    # - https://github.com/rust-lang/rust/issues/44322
+    # - https://alanwu.space/post/symbol-hygiene/
+    if (ENABLE_THINLTO)
+        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--allow-multiple-definition")
+    endif()
 endif()
 
 add_subdirectory (base)
diff --git a/rust/BLAKE3/Cargo.toml b/rust/BLAKE3/Cargo.toml
index eb8f3467424..ed414fa54c1 100644
--- a/rust/BLAKE3/Cargo.toml
+++ b/rust/BLAKE3/Cargo.toml
@@ -11,3 +11,10 @@ libc = "0.2.132"
 [lib]
 crate-type = ["staticlib"]
 
+[profile.release]
+debug = true
+
+[profile.release-thinlto]
+inherits = "release"
+# BLAKE3 module requires "full" LTO (not "thin") to get additional 10% performance benefit
+lto = true
diff --git a/rust/CMakeLists.txt b/rust/CMakeLists.txt
index ec2377fce71..d229894791a 100644
--- a/rust/CMakeLists.txt
+++ b/rust/CMakeLists.txt
@@ -34,9 +34,18 @@ function(clickhouse_import_crate)
     else()
         set(CMAKE_CONFIGURATION_TYPES "${CMAKE_BUILD_TYPE};debug")
     endif()
-    # NOTE: we may use LTO for rust too
 
-    corrosion_import_crate(NO_STD ${ARGN})
+    if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
+        set(profile "")
+    else()
+        if (ENABLE_THINLTO)
+            set(profile "release-thinlto")
+        else()
+            set(profile "release")
+        endif()
+    endif()
+
+    corrosion_import_crate(NO_STD ${ARGN} PROFILE ${profile})
 endfunction()
 
 # Add crate from the build directory.
diff --git a/rust/skim/Cargo.toml b/rust/skim/Cargo.toml
index ce137221065..e5801a26f77 100644
--- a/rust/skim/Cargo.toml
+++ b/rust/skim/Cargo.toml
@@ -18,3 +18,8 @@ crate-type = ["staticlib"]
 
 [profile.release]
 debug = true
+
+[profile.release-thinlto]
+inherits = "release"
+# We use LTO here as well to slightly decrease binary size
+lto = true

From 1c517119e33a8362c51844cf3d7d1c0423efa887 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 22:34:05 +0200
Subject: [PATCH 242/689] Follow-up for #49580

---
 .../01056_create_table_as.reference           |   1 +
 .../0_stateless/01056_create_table_as.sql     |  51 +++++++
 .../01602_show_create_view.reference          |   6 +
 .../0_stateless/01602_show_create_view.sql    |  47 ++++++
 ...query_views_log_current_database.reference |  77 ++++++++++
 ...01927_query_views_log_current_database.sql | 137 ++++++++++++++++++
 6 files changed, 319 insertions(+)
 create mode 100644 tests/queries/0_stateless/01056_create_table_as.reference
 create mode 100644 tests/queries/0_stateless/01056_create_table_as.sql
 create mode 100644 tests/queries/0_stateless/01602_show_create_view.reference
 create mode 100644 tests/queries/0_stateless/01602_show_create_view.sql
 create mode 100644 tests/queries/0_stateless/01927_query_views_log_current_database.reference
 create mode 100644 tests/queries/0_stateless/01927_query_views_log_current_database.sql

diff --git a/tests/queries/0_stateless/01056_create_table_as.reference b/tests/queries/0_stateless/01056_create_table_as.reference
new file mode 100644
index 00000000000..c6e14cb675e
--- /dev/null
+++ b/tests/queries/0_stateless/01056_create_table_as.reference
@@ -0,0 +1 @@
+1	String
diff --git a/tests/queries/0_stateless/01056_create_table_as.sql b/tests/queries/0_stateless/01056_create_table_as.sql
new file mode 100644
index 00000000000..6df660dba61
--- /dev/null
+++ b/tests/queries/0_stateless/01056_create_table_as.sql
@@ -0,0 +1,51 @@
+-- Tags: no-parallel
+
+DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t2;
+DROP TABLE IF EXISTS t3;
+DROP TABLE IF EXISTS v;
+DROP TABLE IF EXISTS lv;
+
+CREATE TABLE t1 (key Int) Engine=Memory;
+CREATE TABLE t2 AS t1;
+DROP TABLE t2;
+CREATE TABLE t2 Engine=Memory AS t1;
+DROP TABLE t2;
+CREATE TABLE t2 AS t1 Engine=Memory;
+DROP TABLE t2;
+CREATE TABLE t3 AS numbers(10);
+DROP TABLE t3;
+
+-- view
+CREATE VIEW v AS SELECT * FROM t1;
+CREATE TABLE t3 AS v; -- { serverError 80 }
+DROP TABLE v;
+
+-- dictionary
+DROP DICTIONARY IF EXISTS dict;
+DROP DATABASE if exists test_01056_dict_data;
+CREATE DATABASE test_01056_dict_data;
+CREATE TABLE test_01056_dict_data.dict_data (key Int, value UInt16) Engine=Memory();
+CREATE DICTIONARY dict
+(
+    `key` UInt64,
+    `value` UInt16
+)
+PRIMARY KEY key
+SOURCE(CLICKHOUSE(
+    HOST '127.0.0.1' PORT tcpPort()
+    TABLE 'dict_data' DB 'test_01056_dict_data' USER 'default' PASSWORD ''))
+LIFETIME(MIN 0 MAX 0)
+LAYOUT(SPARSE_HASHED());
+CREATE TABLE t3 AS dict; -- { serverError 80 }
+
+DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t3;
+DROP DICTIONARY dict;
+DROP TABLE test_01056_dict_data.dict_data;
+
+DROP DATABASE test_01056_dict_data;
+
+CREATE TABLE t1 (x String) ENGINE = Memory AS SELECT 1;
+SELECT x, toTypeName(x) FROM t1;
+DROP TABLE t1;
diff --git a/tests/queries/0_stateless/01602_show_create_view.reference b/tests/queries/0_stateless/01602_show_create_view.reference
new file mode 100644
index 00000000000..5fe11a38db3
--- /dev/null
+++ b/tests/queries/0_stateless/01602_show_create_view.reference
@@ -0,0 +1,6 @@
+CREATE VIEW test_1602.v\n(\n    `EventDate` DateTime,\n    `CounterID` UInt32,\n    `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl
+CREATE MATERIALIZED VIEW test_1602.vv\n(\n    `EventDate` DateTime,\n    `CounterID` UInt32,\n    `UserID` UInt32\n)\nENGINE = MergeTree\nPARTITION BY toYYYYMM(EventDate)\nORDER BY (CounterID, EventDate, intHash32(UserID))\nSETTINGS index_granularity = 8192 AS\nSELECT *\nFROM test_1602.tbl
+CREATE VIEW test_1602.VIEW\n(\n    `EventDate` DateTime,\n    `CounterID` UInt32,\n    `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl
+CREATE VIEW test_1602.DATABASE\n(\n    `EventDate` DateTime,\n    `CounterID` UInt32,\n    `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl
+CREATE VIEW test_1602.DICTIONARY\n(\n    `EventDate` DateTime,\n    `CounterID` UInt32,\n    `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl
+CREATE VIEW test_1602.TABLE\n(\n    `EventDate` DateTime,\n    `CounterID` UInt32,\n    `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl
diff --git a/tests/queries/0_stateless/01602_show_create_view.sql b/tests/queries/0_stateless/01602_show_create_view.sql
new file mode 100644
index 00000000000..1d4dd54b1c1
--- /dev/null
+++ b/tests/queries/0_stateless/01602_show_create_view.sql
@@ -0,0 +1,47 @@
+-- Tags: no-parallel
+
+DROP DATABASE IF EXISTS test_1602;
+
+CREATE DATABASE test_1602;
+
+CREATE TABLE test_1602.tbl (`EventDate` DateTime, `CounterID` UInt32, `UserID` UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SETTINGS index_granularity = 8192;
+
+CREATE VIEW test_1602.v AS SELECT * FROM test_1602.tbl; 
+
+CREATE VIEW test_1602.DATABASE AS SELECT * FROM test_1602.tbl; 
+
+CREATE VIEW test_1602.DICTIONARY AS SELECT * FROM test_1602.tbl; 
+
+CREATE VIEW test_1602.TABLE AS SELECT * FROM test_1602.tbl; 
+
+CREATE MATERIALIZED VIEW test_1602.vv (`EventDate` DateTime, `CounterID` UInt32, `UserID` UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SETTINGS index_granularity = 8192 AS SELECT * FROM test_1602.tbl;
+
+CREATE VIEW test_1602.VIEW AS SELECT * FROM test_1602.tbl; 
+
+SHOW CREATE VIEW test_1602.v;
+
+SHOW CREATE VIEW test_1602.vv;
+
+SHOW CREATE VIEW test_1602.not_exist_view; -- { serverError 390 }
+
+SHOW CREATE VIEW test_1602.tbl; -- { serverError 36 }
+
+SHOW CREATE TEMPORARY VIEW; -- { serverError 60 }
+
+SHOW CREATE VIEW; -- { clientError 62 }
+
+SHOW CREATE DATABASE; -- { clientError 62 }
+
+SHOW CREATE DICTIONARY; -- { clientError 62 }
+
+SHOW CREATE TABLE; -- { clientError 62 }
+
+SHOW CREATE test_1602.VIEW;
+
+SHOW CREATE test_1602.DATABASE;
+
+SHOW CREATE test_1602.DICTIONARY;
+
+SHOW CREATE test_1602.TABLE;
+
+DROP DATABASE IF EXISTS test_1602;
diff --git a/tests/queries/0_stateless/01927_query_views_log_current_database.reference b/tests/queries/0_stateless/01927_query_views_log_current_database.reference
new file mode 100644
index 00000000000..64bf1e501f3
--- /dev/null
+++ b/tests/queries/0_stateless/01927_query_views_log_current_database.reference
@@ -0,0 +1,77 @@
+Row 1:
+──────
+stage:                Query log rows
+read_rows:            300
+written_rows:         201
+databases:            ['_table_function','default']
+tables:               ['_table_function.numbers','default.table_a','default.table_b','default.table_c']
+views:                ['default.matview_a_to_b','default.matview_b_to_c']
+sleep_calls:          200
+sleep_us:             298
+profile_select_rows:  300
+profile_select_bytes: 3600
+profile_insert_rows:  201
+profile_insert_bytes: 2808
+Row 1:
+──────
+stage:                Depending views
+view_name:            default.matview_a_to_b
+view_type:            Materialized
+status:               QueryFinish
+view_target:          default.table_b
+view_query:           SELECT toFloat64(a) AS a, b + sleepEachRow(0.000001) AS count FROM default.table_a
+read_rows:            100
+written_rows:         100
+sleep_calls:          100
+sleep_us:             99
+profile_select_rows:  100
+profile_select_bytes: 2000
+profile_insert_rows:  100
+profile_insert_bytes: 800
+
+Row 2:
+──────
+stage:                Depending views
+view_name:            default.matview_b_to_c
+view_type:            Materialized
+status:               QueryFinish
+view_target:          default.table_c
+view_query:           SELECT sum(a + sleepEachRow(0.000002)) AS a FROM default.table_b
+read_rows:            100
+written_rows:         1
+sleep_calls:          100
+sleep_us:             199
+profile_select_rows:  100
+profile_select_bytes: 800
+profile_insert_rows:  1
+profile_insert_bytes: 8
+Row 1:
+──────
+stage:                Query log rows 2
+read_rows:            100
+written_rows:         100
+databases:            ['_table_function','default']
+tables:               ['_table_function.numbers','default.table_d','default.table_e','default.table_f']
+views:                ['default.matview_join_d_e']
+sleep_calls:          50
+sleep_us:             150
+profile_select_rows:  100
+profile_select_bytes: 800
+profile_insert_rows:  100
+profile_insert_bytes: 1600
+Row 1:
+──────
+stage:                Depending views 2
+view_name:            default.matview_join_d_e
+view_type:            Materialized
+status:               QueryFinish
+view_target:          default.table_f
+view_query:           SELECT table_d.a AS a, table_e.count + sleepEachRow(0.000003) AS count FROM default.table_d LEFT JOIN default.table_e ON table_d.a = table_e.a
+read_rows:            50
+written_rows:         50
+sleep_calls:          50
+sleep_us:             150
+profile_select_rows:  50
+profile_select_bytes: 400
+profile_insert_rows:  50
+profile_insert_bytes: 800
diff --git a/tests/queries/0_stateless/01927_query_views_log_current_database.sql b/tests/queries/0_stateless/01927_query_views_log_current_database.sql
new file mode 100644
index 00000000000..ba42795333c
--- /dev/null
+++ b/tests/queries/0_stateless/01927_query_views_log_current_database.sql
@@ -0,0 +1,137 @@
+SET log_queries=0;
+SET log_query_threads=0;
+
+-- SETUP TABLES
+CREATE TABLE table_a (a String, b Int64) ENGINE = MergeTree ORDER BY b;
+CREATE TABLE table_b (a Float64,  b Int64) ENGINE = MergeTree ORDER BY tuple();
+CREATE TABLE table_c (a Float64) ENGINE = MergeTree ORDER BY a;
+
+CREATE TABLE table_d (a Float64, count Int64) ENGINE MergeTree ORDER BY a;
+CREATE TABLE table_e (a Float64, count Int64) ENGINE MergeTree ORDER BY a;
+CREATE TABLE table_f (a Float64, count Int64) ENGINE MergeTree ORDER BY a;
+
+-- SETUP MATERIALIZED VIEWS
+CREATE MATERIALIZED VIEW matview_a_to_b TO table_b AS SELECT toFloat64(a) AS a, b + sleepEachRow(0.000001) AS count FROM table_a;
+CREATE MATERIALIZED VIEW matview_b_to_c TO table_c AS SELECT SUM(a + sleepEachRow(0.000002)) as a FROM table_b;
+CREATE MATERIALIZED VIEW matview_join_d_e TO table_f AS SELECT table_d.a as a, table_e.count + sleepEachRow(0.000003) as count FROM table_d LEFT JOIN table_e ON table_d.a = table_e.a;
+
+-- ENABLE LOGS
+SET log_query_views=1;
+SET log_queries_min_type='QUERY_FINISH';
+SET log_queries=1;
+
+-- INSERT 1
+INSERT INTO table_a SELECT '111', * FROM numbers(100);
+
+-- INSERT 2
+INSERT INTO table_d SELECT 0.5, * FROM numbers(50);
+
+SYSTEM FLUSH LOGS;
+
+
+-- CHECK LOGS OF INSERT 1
+SELECT
+    'Query log rows' as stage,
+    read_rows,
+    written_rows,
+    arraySort(databases) as databases,
+    arraySort(tables) as tables,
+    arraySort(views) as views,
+    ProfileEvents['SleepFunctionCalls'] as sleep_calls,
+    ProfileEvents['SleepFunctionMicroseconds'] as sleep_us,
+    ProfileEvents['SelectedRows'] as profile_select_rows,
+    ProfileEvents['SelectedBytes'] as profile_select_bytes,
+    ProfileEvents['InsertedRows'] as profile_insert_rows,
+    ProfileEvents['InsertedBytes'] as profile_insert_bytes
+FROM system.query_log
+WHERE query like '-- INSERT 1%INSERT INTO table_a%'
+  AND current_database = currentDatabase()
+  AND event_date >= yesterday()
+FORMAT Vertical;
+
+SELECT
+    'Depending views' as stage,
+    view_name,
+    view_type,
+    status,
+    view_target,
+    view_query,
+    read_rows,
+    written_rows,
+    ProfileEvents['SleepFunctionCalls'] as sleep_calls,
+    ProfileEvents['SleepFunctionMicroseconds'] as sleep_us,
+    ProfileEvents['SelectedRows'] as profile_select_rows,
+    ProfileEvents['SelectedBytes'] as profile_select_bytes,
+    ProfileEvents['InsertedRows'] as profile_insert_rows,
+    ProfileEvents['InsertedBytes'] as profile_insert_bytes
+FROM system.query_views_log
+WHERE initial_query_id =
+      (
+          SELECT initial_query_id
+          FROM system.query_log
+          WHERE query like '-- INSERT 1%INSERT INTO table_a%'
+            AND current_database = currentDatabase()
+            AND event_date >= yesterday()
+          LIMIT 1
+      )
+ORDER BY view_name
+FORMAT Vertical;
+
+-- CHECK LOGS OF INSERT 2
+SELECT
+    'Query log rows 2' as stage,
+    read_rows,
+    written_rows,
+    arraySort(databases) as databases,
+    arraySort(tables) as tables,
+    arraySort(views) as views,
+    ProfileEvents['SleepFunctionCalls'] as sleep_calls,
+    ProfileEvents['SleepFunctionMicroseconds'] as sleep_us,
+    ProfileEvents['SelectedRows'] as profile_select_rows,
+    ProfileEvents['SelectedBytes'] as profile_select_bytes,
+    ProfileEvents['InsertedRows'] as profile_insert_rows,
+    ProfileEvents['InsertedBytes'] as profile_insert_bytes
+FROM system.query_log
+WHERE query like '-- INSERT 2%INSERT INTO table_d%'
+  AND current_database = currentDatabase()
+  AND event_date >= yesterday()
+FORMAT Vertical;
+
+SELECT
+    'Depending views 2' as stage,
+    view_name,
+    view_type,
+    status,
+    view_target,
+    view_query,
+    read_rows,
+    written_rows,
+    ProfileEvents['SleepFunctionCalls'] as sleep_calls,
+    ProfileEvents['SleepFunctionMicroseconds'] as sleep_us,
+    ProfileEvents['SelectedRows'] as profile_select_rows,
+    ProfileEvents['SelectedBytes'] as profile_select_bytes,
+    ProfileEvents['InsertedRows'] as profile_insert_rows,
+    ProfileEvents['InsertedBytes'] as profile_insert_bytes
+FROM system.query_views_log
+WHERE initial_query_id =
+      (
+          SELECT initial_query_id
+          FROM system.query_log
+          WHERE query like '-- INSERT 2%INSERT INTO table_d%'
+            AND current_database = currentDatabase()
+            AND event_date >= yesterday()
+          LIMIT 1
+      )
+ORDER BY view_name
+FORMAT Vertical;
+
+-- TEARDOWN
+DROP TABLE matview_a_to_b;
+DROP TABLE matview_b_to_c;
+DROP TABLE matview_join_d_e;
+DROP TABLE table_f;
+DROP TABLE table_e;
+DROP TABLE table_d;
+DROP TABLE table_c;
+DROP TABLE table_b;
+DROP TABLE table_a;

From 2fe2e6fb4bdd4c281b8be2bde16c2faf8397eba4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 23:13:32 +0200
Subject: [PATCH 243/689] There is no point in parallel formatting of Pretty
 formats

---
 .../0_stateless/02122_parallel_formatting_Pretty.reference   | 2 --
 .../queries/0_stateless/02122_parallel_formatting_Pretty.sh  | 5 -----
 .../02122_parallel_formatting_PrettyCompact.reference        | 2 --
 .../0_stateless/02122_parallel_formatting_PrettyCompact.sh   | 5 -----
 ...2122_parallel_formatting_PrettyCompactNoEscapes.reference | 2 --
 .../02122_parallel_formatting_PrettyCompactNoEscapes.sh      | 5 -----
 .../02122_parallel_formatting_PrettyNoEscapes.reference      | 2 --
 .../0_stateless/02122_parallel_formatting_PrettyNoEscapes.sh | 5 -----
 .../02122_parallel_formatting_PrettySpace.reference          | 2 --
 .../0_stateless/02122_parallel_formatting_PrettySpace.sh     | 5 -----
 .../02122_parallel_formatting_PrettySpaceNoEscapes.reference | 2 --
 .../02122_parallel_formatting_PrettySpaceNoEscapes.sh        | 5 -----
 12 files changed, 42 deletions(-)
 delete mode 100644 tests/queries/0_stateless/02122_parallel_formatting_Pretty.reference
 delete mode 100755 tests/queries/0_stateless/02122_parallel_formatting_Pretty.sh
 delete mode 100644 tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.reference
 delete mode 100755 tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.sh
 delete mode 100644 tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.reference
 delete mode 100755 tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.sh
 delete mode 100644 tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.reference
 delete mode 100755 tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.sh
 delete mode 100644 tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.reference
 delete mode 100755 tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.sh
 delete mode 100644 tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.reference
 delete mode 100755 tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.sh

diff --git a/tests/queries/0_stateless/02122_parallel_formatting_Pretty.reference b/tests/queries/0_stateless/02122_parallel_formatting_Pretty.reference
deleted file mode 100644
index 72d346f82f7..00000000000
--- a/tests/queries/0_stateless/02122_parallel_formatting_Pretty.reference
+++ /dev/null
@@ -1,2 +0,0 @@
-Pretty-1
-Pretty-2
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_Pretty.sh b/tests/queries/0_stateless/02122_parallel_formatting_Pretty.sh
deleted file mode 100755
index a72510b0614..00000000000
--- a/tests/queries/0_stateless/02122_parallel_formatting_Pretty.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-fasttest
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-${CUR_DIR}/02122_parallel_formatting.lib Pretty
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.reference b/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.reference
deleted file mode 100644
index 35761b294db..00000000000
--- a/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.reference
+++ /dev/null
@@ -1,2 +0,0 @@
-PrettyCompact-1
-PrettyCompact-2
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.sh b/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.sh
deleted file mode 100755
index 351f7ce0cd6..00000000000
--- a/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-fasttest
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-${CUR_DIR}/02122_parallel_formatting.lib PrettyCompact
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.reference b/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.reference
deleted file mode 100644
index 8b70c25b168..00000000000
--- a/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.reference
+++ /dev/null
@@ -1,2 +0,0 @@
-PrettyCompactNoEscapes-1
-PrettyCompactNoEscapes-2
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.sh b/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.sh
deleted file mode 100755
index fd1e7a83ead..00000000000
--- a/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-fasttest
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-${CUR_DIR}/02122_parallel_formatting.lib PrettyCompactNoEscapes
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.reference b/tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.reference
deleted file mode 100644
index e2bfec67b7c..00000000000
--- a/tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.reference
+++ /dev/null
@@ -1,2 +0,0 @@
-PrettyNoEscapes-1
-PrettyNoEscapes-2
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.sh b/tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.sh
deleted file mode 100755
index 20e6d9c6973..00000000000
--- a/tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-fasttest
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-${CUR_DIR}/02122_parallel_formatting.lib PrettyNoEscapes
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.reference b/tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.reference
deleted file mode 100644
index fab8c8bde76..00000000000
--- a/tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.reference
+++ /dev/null
@@ -1,2 +0,0 @@
-PrettySpace-1
-PrettySpace-2
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.sh b/tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.sh
deleted file mode 100755
index 8af990d9113..00000000000
--- a/tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-fasttest
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-${CUR_DIR}/02122_parallel_formatting.lib PrettySpace
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.reference b/tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.reference
deleted file mode 100644
index 154ee87e82a..00000000000
--- a/tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.reference
+++ /dev/null
@@ -1,2 +0,0 @@
-PrettySpaceNoEscapes-1
-PrettySpaceNoEscapes-2
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.sh b/tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.sh
deleted file mode 100755
index 8fcbfbc16cf..00000000000
--- a/tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-fasttest
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-${CUR_DIR}/02122_parallel_formatting.lib PrettySpaceNoEscapes

From 763c98bd70b9aa06363fb3f49000eb9720bdc435 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 May 2023 23:40:19 +0200
Subject: [PATCH 244/689] Fix typo

---
 docker/bare/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/bare/README.md b/docker/bare/README.md
index 0a610b1de90..4baf308f1e1 100644
--- a/docker/bare/README.md
+++ b/docker/bare/README.md
@@ -1,6 +1,6 @@
 ## The bare minimum ClickHouse Docker image.
 
-It is intented as a showcase to check the amount of implicit dependencies of ClickHouse from the OS in addition to the OS kernel.
+It is intended as a showcase to check the amount of implicit dependencies of ClickHouse from the OS in addition to the OS kernel.
 
 Example usage:
 

From 07027be872305d220aa28445b004a51e1e7ba6ee Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 02:09:37 +0300
Subject: [PATCH 245/689] Update test_in_memory_parts_still_read.py

---
 .../test_in_memory_parts_still_read.py                          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py b/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
index 0595be54879..839a698f274 100644
--- a/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
+++ b/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
@@ -27,7 +27,7 @@ def start_cluster():
 
 def test_in_memory_parts_still_read(start_cluster):
     node.query(
-        "CREATE TABLE t (x UInt64, s String, a Array(Tuple(Map(String, LowCardinality(String)))), Date32, DateTime64(3))) ENGINE = MergeTree ORDER BY s SETTINGS min_rows_for_compact_part = 1000000, min_bytes_for_compact_part = '1G', in_memory_parts_enable_wal = 1"
+        "CREATE TABLE t (x UInt64, s String, a Array(Tuple(Map(String, LowCardinality(String)))), Date32, DateTime64(3)) ENGINE = MergeTree ORDER BY s SETTINGS min_rows_for_compact_part = 1000000, min_bytes_for_compact_part = '1G', in_memory_parts_enable_wal = 1"
     )
     node.query("INSERT INTO t SELECT generateRandom() LIMIT 100")
 

From 08bfc8f58d8e3823df215e1c8239a028a04a9a26 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Sun, 7 May 2023 01:51:43 +0200
Subject: [PATCH 246/689] inserted parts has exlusive blobs

---
 src/Storages/MergeTree/MergeTreeDataWriter.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index adb7505a8ba..28dab4c9068 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -469,6 +469,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl(
     new_data_part->partition = std::move(partition);
     new_data_part->minmax_idx = std::move(minmax_idx);
     new_data_part->is_temp = true;
+    new_data_part->has_exclusive_blobs = true;
 
     SyncGuardPtr sync_guard;
     if (new_data_part->isStoredOnDisk())

From 15c648a6159bd124065d4867d4361cca8eb2f5f3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 02:33:45 +0200
Subject: [PATCH 247/689] Fix bad test 01660_system_parts_smoke

---
 tests/queries/0_stateless/01660_system_parts_smoke.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01660_system_parts_smoke.sql b/tests/queries/0_stateless/01660_system_parts_smoke.sql
index 64cba86b8f6..15162af095a 100644
--- a/tests/queries/0_stateless/01660_system_parts_smoke.sql
+++ b/tests/queries/0_stateless/01660_system_parts_smoke.sql
@@ -35,7 +35,7 @@ SELECT count(), _state FROM system.parts WHERE database = currentDatabase() AND
 -- Inactive parts are clearing by async process also
 SELECT '# truncate';
 TRUNCATE data_01660;
-SELECT if (count() > 0, 'HAVE PARTS', 'NO PARTS'), _state FROM system.parts WHERE database = currentDatabase() AND table = 'data_01660' GROUP BY _state;
+SELECT if (count() > 0, 'HAVE PARTS', 'NO PARTS'), _state FROM system.parts WHERE database = currentDatabase() AND table = 'data_01660' GROUP BY _state ORDER BY _state;
 
 -- But DROP does
 SELECT '# drop';

From cb90c7e670e33c92df7aa966d6f90704eab0575e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 02:39:21 +0200
Subject: [PATCH 248/689] Minor changes

---
 src/AggregateFunctions/AggregateFunctionNothing.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/AggregateFunctions/AggregateFunctionNothing.h b/src/AggregateFunctions/AggregateFunctionNothing.h
index 8280d72c085..8c1b95c26b0 100644
--- a/src/AggregateFunctions/AggregateFunctionNothing.h
+++ b/src/AggregateFunctions/AggregateFunctionNothing.h
@@ -75,7 +75,7 @@ public:
         [[maybe_unused]] char symbol;
         readChar(symbol, buf);
         if (symbol != '\0')
-            throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect state of aggregate function 'nothing', it should contain exactly one zero byte.");
+            throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect state of aggregate function 'nothing', it should contain exactly one zero byte, while it is {}.", static_cast<UInt32>(symbol));
     }
 
     void insertResultInto(AggregateDataPtr __restrict, IColumn & to, Arena *) const override

From 3b6edbacabc989c99666942647b1549fae255a74 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 03:45:17 +0200
Subject: [PATCH 249/689] Update tests

---
 .../MergeTree/registerStorageMergeTree.cpp    |  2 +-
 tests/integration/ci-runner.py                | 13 +++---
 .../test_async_insert_memory/test.py          |  2 +-
 .../test_backward_compatibility/test.py       |  4 +-
 ...test_vertical_merges_from_compact_parts.py | 13 ++++--
 tests/integration/test_check_table/test.py    | 22 +++++----
 .../test_cluster_discovery/test.py            |  2 +-
 tests/integration/test_mysql_protocol/test.py | 16 +++----
 tests/integration/test_partition/test.py      |  4 +-
 .../test_replicated_database/test.py          |  9 ++--
 .../s3_mocks/unstable_server.py               |  2 +-
 tests/integration/test_storage_s3/test.py     | 17 +++----
 tests/integration/test_storage_url/test.py    | 22 +++++----
 tests/integration/test_system_merges/test.py  | 45 +++++++++----------
 14 files changed, 89 insertions(+), 84 deletions(-)

diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp
index 5209be0fd3b..027cd1af7c9 100644
--- a/src/Storages/MergeTree/registerStorageMergeTree.cpp
+++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp
@@ -535,7 +535,7 @@ static StoragePtr create(const StorageFactory::Arguments & args)
         if (!args.storage_def->order_by)
             throw Exception(ErrorCodes::BAD_ARGUMENTS,
                             "You must provide an ORDER BY or PRIMARY KEY expression in the table definition. "
-                            "If you don't want this table to be sorted, use ORDER BY/PRIMARY KEY tuple()");
+                            "If you don't want this table to be sorted, use ORDER BY/PRIMARY KEY ()");
 
         /// Get sorting key from engine arguments.
         ///
diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py
index d6d17abe725..be438dad492 100755
--- a/tests/integration/ci-runner.py
+++ b/tests/integration/ci-runner.py
@@ -295,14 +295,11 @@ class ClickhouseIntegrationTestsRunner:
     def _pre_pull_images(self, repo_path):
         image_cmd = self._get_runner_image_cmd(repo_path)
 
-        cmd = (
-            "cd {repo_path}/tests/integration && "
-            "timeout -s 9 1h ./runner {runner_opts} {image_cmd} --pre-pull --command '{command}' ".format(
-                repo_path=repo_path,
-                runner_opts=self._get_runner_opts(),
-                image_cmd=image_cmd,
-                command=r""" echo Pre Pull finished """,
-            )
+        cmd = "cd {repo_path}/tests/integration && " "timeout -s 9 1h ./runner {runner_opts} {image_cmd} --pre-pull --command '{command}' ".format(
+            repo_path=repo_path,
+            runner_opts=self._get_runner_opts(),
+            image_cmd=image_cmd,
+            command=r""" echo Pre Pull finished """,
         )
 
         for i in range(5):
diff --git a/tests/integration/test_async_insert_memory/test.py b/tests/integration/test_async_insert_memory/test.py
index 5d2e5503680..f897007f7bb 100644
--- a/tests/integration/test_async_insert_memory/test.py
+++ b/tests/integration/test_async_insert_memory/test.py
@@ -43,7 +43,7 @@ def test_memory_usage():
 
     response = node.get_query_request(
         "SELECT groupArray(number) FROM numbers(1000000) SETTINGS max_memory_usage_for_user={}".format(
-            30 * (2**23)
+            30 * (2 ** 23)
         ),
         user="A",
     )
diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py
index e4ee6aa2de9..ea1d3ab9c07 100644
--- a/tests/integration/test_backward_compatibility/test.py
+++ b/tests/integration/test_backward_compatibility/test.py
@@ -12,7 +12,9 @@ node1 = cluster.add_instance(
     with_installed_binary=True,
 )
 node2 = cluster.add_instance(
-    "node2", main_configs=["configs/wide_parts_only.xml", "configs/no_compress_marks.xml"], with_zookeeper=True
+    "node2",
+    main_configs=["configs/wide_parts_only.xml", "configs/no_compress_marks.xml"],
+    with_zookeeper=True,
 )
 
 
diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
index 4b144a37ce9..e16f74571cb 100644
--- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
+++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
@@ -30,7 +30,7 @@ def start_cluster():
         cluster.shutdown()
 
 
-def test_vertical_merges_from_comapact_parts(start_cluster):
+def test_vertical_merges_from_compact_parts(start_cluster):
     for i, node in enumerate([node_old, node_new]):
         node.query(
             """
@@ -42,7 +42,7 @@ def test_vertical_merges_from_comapact_parts(start_cluster):
                 vertical_merge_algorithm_min_rows_to_activate = 1,
                 vertical_merge_algorithm_min_columns_to_activate = 1,
                 min_bytes_for_wide_part = 0,
-                min_rows_for_wide_part = 100;
+                min_rows_for_wide_part = 100
         """.format(
                 i
             )
@@ -105,8 +105,13 @@ def test_vertical_merges_from_comapact_parts(start_cluster):
     node_old.query("SYSTEM FLUSH LOGS")
 
     assert not (
-        node_old.contains_in_log("CHECKSUM_DOESNT_MATCH")
-        or node_new.contains_in_log("CHECKSUM_DOESNT_MATCH")
+        # Now the old node is restarted as a new, and its config allows compressed indices, and it merged the data into compressed indices,
+        # that's why the error about different number of compressed files is expected and ok.
+        (
+            node_old.contains_in_log("CHECKSUM_DOESNT_MATCH")
+            and not node_old.contains_in_log("Different number of files: 7 compressed")
+        )
+        or (node_new.contains_in_log("CHECKSUM_DOESNT_MATCH"))
     )
 
     assert node_new.query(check_query.format("all_0_3_3")) == "Vertical\tWide\n"
diff --git a/tests/integration/test_check_table/test.py b/tests/integration/test_check_table/test.py
index 613ac3fb35f..60492141e69 100644
--- a/tests/integration/test_check_table/test.py
+++ b/tests/integration/test_check_table/test.py
@@ -244,10 +244,13 @@ def test_check_replicated_table_corruption(started_cluster):
     )
 
     node1.query_with_retry("SYSTEM SYNC REPLICA replicated_mt_1")
-    assert node1.query(
-        "CHECK TABLE replicated_mt_1 PARTITION 201901",
-        settings={"check_query_single_value_result": 0},
-    ) == "{}\t1\t\n".format(part_name)
+    assert (
+        node1.query(
+            "CHECK TABLE replicated_mt_1 PARTITION 201901",
+            settings={"check_query_single_value_result": 0},
+        )
+        == "{}\t1\t\n".format(part_name)
+    )
     assert node1.query("SELECT count() from replicated_mt_1") == "4\n"
 
     remove_part_from_disk(node2, "replicated_mt_1", part_name)
@@ -259,8 +262,11 @@ def test_check_replicated_table_corruption(started_cluster):
     )
 
     node1.query("SYSTEM SYNC REPLICA replicated_mt_1")
-    assert node1.query(
-        "CHECK TABLE replicated_mt_1 PARTITION 201901",
-        settings={"check_query_single_value_result": 0},
-    ) == "{}\t1\t\n".format(part_name)
+    assert (
+        node1.query(
+            "CHECK TABLE replicated_mt_1 PARTITION 201901",
+            settings={"check_query_single_value_result": 0},
+        )
+        == "{}\t1\t\n".format(part_name)
+    )
     assert node1.query("SELECT count() from replicated_mt_1") == "4\n"
diff --git a/tests/integration/test_cluster_discovery/test.py b/tests/integration/test_cluster_discovery/test.py
index ad3deb5b142..a2e7e15b956 100644
--- a/tests/integration/test_cluster_discovery/test.py
+++ b/tests/integration/test_cluster_discovery/test.py
@@ -61,7 +61,7 @@ def check_on_cluster(
         print(f"Retry {retry}/{retries} unsuccessful, result: {node_results}")
 
         if retry != retries:
-            time.sleep(2**retry)
+            time.sleep(2 ** retry)
     else:
         msg = msg or f"Wrong '{what}' result"
         raise Exception(
diff --git a/tests/integration/test_mysql_protocol/test.py b/tests/integration/test_mysql_protocol/test.py
index 5e03c4883b4..e224853da32 100644
--- a/tests/integration/test_mysql_protocol/test.py
+++ b/tests/integration/test_mysql_protocol/test.py
@@ -828,14 +828,14 @@ def test_types(started_cluster):
 
     result = cursor.fetchall()[0]
     expected = [
-        ("Int8_column", -(2**7)),
-        ("UInt8_column", 2**8 - 1),
-        ("Int16_column", -(2**15)),
-        ("UInt16_column", 2**16 - 1),
-        ("Int32_column", -(2**31)),
-        ("UInt32_column", 2**32 - 1),
-        ("Int64_column", -(2**63)),
-        ("UInt64_column", 2**64 - 1),
+        ("Int8_column", -(2 ** 7)),
+        ("UInt8_column", 2 ** 8 - 1),
+        ("Int16_column", -(2 ** 15)),
+        ("UInt16_column", 2 ** 16 - 1),
+        ("Int32_column", -(2 ** 31)),
+        ("UInt32_column", 2 ** 32 - 1),
+        ("Int64_column", -(2 ** 63)),
+        ("UInt64_column", 2 ** 64 - 1),
         ("String_column", "тест"),
         ("FixedString_column", "тест"),
         ("Float32_column", 1.5),
diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py
index 93f03f4420e..066d0bcf800 100644
--- a/tests/integration/test_partition/test.py
+++ b/tests/integration/test_partition/test.py
@@ -561,9 +561,7 @@ def test_make_clone_in_detached(started_cluster):
         ["cp", "-r", path + "all_0_0_0", path + "detached/broken_all_0_0_0"]
     )
     assert_eq_with_retry(instance, "select * from clone_in_detached", "\n")
-    assert [
-        "broken_all_0_0_0",
-    ] == sorted(
+    assert ["broken_all_0_0_0",] == sorted(
         instance.exec_in_container(["ls", path + "detached/"]).strip().split("\n")
     )
 
diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py
index 2ab2fe499ff..e17170f630c 100644
--- a/tests/integration/test_replicated_database/test.py
+++ b/tests/integration/test_replicated_database/test.py
@@ -492,12 +492,9 @@ def test_alters_from_different_replicas(started_cluster):
     dummy_node.stop_clickhouse(kill=True)
 
     settings = {"distributed_ddl_task_timeout": 5}
-    assert (
-        "There are 1 unfinished hosts (0 of them are currently active)"
-        in competing_node.query_and_get_error(
-            "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added0 UInt32;",
-            settings=settings,
-        )
+    assert "There are 1 unfinished hosts (0 of them are currently active)" in competing_node.query_and_get_error(
+        "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added0 UInt32;",
+        settings=settings,
     )
     settings = {
         "distributed_ddl_task_timeout": 5,
diff --git a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py
index 103dd30340c..16697596306 100644
--- a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py
+++ b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py
@@ -8,7 +8,7 @@ import sys
 
 def gen_n_digit_number(n):
     assert 0 < n < 19
-    return random.randint(10 ** (n - 1), 10**n - 1)
+    return random.randint(10 ** (n - 1), 10 ** n - 1)
 
 
 sum_in_4_column = 0
diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py
index d9ac70f51ad..78e33534022 100644
--- a/tests/integration/test_storage_s3/test.py
+++ b/tests/integration/test_storage_s3/test.py
@@ -538,16 +538,13 @@ def test_multipart(started_cluster, maybe_auth, positive):
         assert csv_data == get_s3_file_content(started_cluster, bucket, filename)
 
     # select uploaded data from many threads
-    select_query = (
-        "select sum(column1), sum(column2), sum(column3) "
-        "from s3('http://{host}:{port}/{bucket}/{filename}', {auth}'CSV', '{table_format}')".format(
-            host=started_cluster.minio_redirect_host,
-            port=started_cluster.minio_redirect_port,
-            bucket=bucket,
-            filename=filename,
-            auth=maybe_auth,
-            table_format=table_format,
-        )
+    select_query = "select sum(column1), sum(column2), sum(column3) " "from s3('http://{host}:{port}/{bucket}/{filename}', {auth}'CSV', '{table_format}')".format(
+        host=started_cluster.minio_redirect_host,
+        port=started_cluster.minio_redirect_port,
+        bucket=bucket,
+        filename=filename,
+        auth=maybe_auth,
+        table_format=table_format,
     )
     try:
         select_result = run_query(
diff --git a/tests/integration/test_storage_url/test.py b/tests/integration/test_storage_url/test.py
index f3ed087d3d0..2ef04c444f8 100644
--- a/tests/integration/test_storage_url/test.py
+++ b/tests/integration/test_storage_url/test.py
@@ -49,15 +49,21 @@ def test_table_function_url_access_rights():
         f"SELECT * FROM url('http://nginx:80/test_1', 'TSV')", user="u1"
     )
 
-    assert node1.query(
-        f"DESCRIBE TABLE url('http://nginx:80/test_1', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')",
-        user="u1",
-    ) == TSV([["column1", "UInt32"], ["column2", "UInt32"], ["column3", "UInt32"]])
+    assert (
+        node1.query(
+            f"DESCRIBE TABLE url('http://nginx:80/test_1', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')",
+            user="u1",
+        )
+        == TSV([["column1", "UInt32"], ["column2", "UInt32"], ["column3", "UInt32"]])
+    )
 
-    assert node1.query(
-        f"DESCRIBE TABLE url('http://nginx:80/not-exist', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')",
-        user="u1",
-    ) == TSV([["column1", "UInt32"], ["column2", "UInt32"], ["column3", "UInt32"]])
+    assert (
+        node1.query(
+            f"DESCRIBE TABLE url('http://nginx:80/not-exist', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')",
+            user="u1",
+        )
+        == TSV([["column1", "UInt32"], ["column2", "UInt32"], ["column3", "UInt32"]])
+    )
 
     expected_error = "necessary to have grant URL ON *.*"
     assert expected_error in node1.query_and_get_error(
diff --git a/tests/integration/test_system_merges/test.py b/tests/integration/test_system_merges/test.py
index ff303afe19e..d4b9b403871 100644
--- a/tests/integration/test_system_merges/test.py
+++ b/tests/integration/test_system_merges/test.py
@@ -202,36 +202,33 @@ def test_mutation_simple(started_cluster, replicated):
             sleep_time=0.1,
         )
 
-        assert (
-            split_tsv(
-                node_check.query(
-                    """
+        assert split_tsv(
+            node_check.query(
+                """
             SELECT database, table, num_parts, source_part_names, source_part_paths, result_part_name, result_part_path, partition_id, is_mutation
                 FROM system.merges
                 WHERE table = '{name}'
         """.format(
-                        name=table_name
-                    )
+                    name=table_name
                 )
             )
-            == [
-                [
-                    db_name,
-                    table_name,
-                    "1",
-                    "['{}']".format(part),
-                    "['{clickhouse}/{table_path}/{}/']".format(
-                        part, clickhouse=clickhouse_path, table_path=table_path
-                    ),
-                    result_part,
-                    "{clickhouse}/{table_path}/{}/".format(
-                        result_part, clickhouse=clickhouse_path, table_path=table_path
-                    ),
-                    "all",
-                    "1",
-                ],
-            ]
-        )
+        ) == [
+            [
+                db_name,
+                table_name,
+                "1",
+                "['{}']".format(part),
+                "['{clickhouse}/{table_path}/{}/']".format(
+                    part, clickhouse=clickhouse_path, table_path=table_path
+                ),
+                result_part,
+                "{clickhouse}/{table_path}/{}/".format(
+                    result_part, clickhouse=clickhouse_path, table_path=table_path
+                ),
+                "all",
+                "1",
+            ],
+        ]
         t.join()
 
         assert (

From 6e46a4e0ead455422a4a26bee9977e131bc057ad Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 04:56:09 +0300
Subject: [PATCH 250/689] Update test_in_memory_parts_still_read.py

---
 .../test_in_memory_parts_still_read.py                          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py b/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
index 839a698f274..94baf03c83a 100644
--- a/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
+++ b/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
@@ -27,7 +27,7 @@ def start_cluster():
 
 def test_in_memory_parts_still_read(start_cluster):
     node.query(
-        "CREATE TABLE t (x UInt64, s String, a Array(Tuple(Map(String, LowCardinality(String)))), Date32, DateTime64(3)) ENGINE = MergeTree ORDER BY s SETTINGS min_rows_for_compact_part = 1000000, min_bytes_for_compact_part = '1G', in_memory_parts_enable_wal = 1"
+        "CREATE TABLE t (x UInt64, s String, a Array(Tuple(Map(String, LowCardinality(String)), Date32, DateTime64(3)))) ENGINE = MergeTree ORDER BY s SETTINGS min_rows_for_compact_part = 1000000, min_bytes_for_compact_part = '1G', in_memory_parts_enable_wal = 1"
     )
     node.query("INSERT INTO t SELECT generateRandom() LIMIT 100")
 

From d8f5667ba6f3192fd1eba65371b29e5170079790 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 03:59:35 +0200
Subject: [PATCH 251/689] Follow-up for #49576

---
 .../__init__.py                               |   0
 .../configs/config.d/clusters.xml             |  39 ---
 .../configs/config.d/storage_conf.xml         |  35 ---
 .../config.d/storage_conf_another_bucket.xml  |  34 ---
 .../storage_conf_without_zero_copy.xml        |  35 ---
 .../test.py                                   | 263 ------------------
 6 files changed, 406 deletions(-)
 delete mode 100644 tests/integration/test_replicated_merge_tree_s3_restore/__init__.py
 delete mode 100644 tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/clusters.xml
 delete mode 100644 tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf.xml
 delete mode 100644 tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml
 delete mode 100644 tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_without_zero_copy.xml
 delete mode 100644 tests/integration/test_replicated_merge_tree_s3_restore/test.py

diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/__init__.py b/tests/integration/test_replicated_merge_tree_s3_restore/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/clusters.xml b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/clusters.xml
deleted file mode 100644
index e0b94f46d83..00000000000
--- a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/clusters.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<clickhouse>
-    <remote_servers>
-        <node_zero_copy>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>node1z</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>node2z</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </node_zero_copy>
-        <node_no_zero_copy>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>node1n</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>node2n</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </node_no_zero_copy>
-        <node_another_bucket>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>node_another_bucket</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </node_another_bucket>
-    </remote_servers>
-</clickhouse>
diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf.xml
deleted file mode 100644
index 63fe7ac1769..00000000000
--- a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<clickhouse>
-    <storage_configuration>
-        <disks>
-            <s3>
-                <type>s3</type>
-                <endpoint>http://minio1:9001/root/data/</endpoint>
-                <access_key_id>minio</access_key_id>
-                <secret_access_key>minio123</secret_access_key>
-                <send_metadata>true</send_metadata>
-                <list_object_keys_size>1</list_object_keys_size> <!-- To effectively test restore parallelism -->
-            </s3>
-            <hdd>
-                <type>local</type>
-                <path>/</path>
-            </hdd>
-        </disks>
-        <policies>
-            <s3>
-                <volumes>
-                    <main>
-                        <disk>s3</disk>
-                    </main>
-                    <external>
-                        <disk>hdd</disk>
-                    </external>
-                </volumes>
-            </s3>
-        </policies>
-    </storage_configuration>
-
-    <merge_tree>
-        <min_bytes_for_wide_part>0</min_bytes_for_wide_part>
-        <allow_remote_fs_zero_copy_replication>1</allow_remote_fs_zero_copy_replication>
-    </merge_tree>
-</clickhouse>
diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml
deleted file mode 100644
index 920db2c9edd..00000000000
--- a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<clickhouse>
-    <storage_configuration>
-        <disks>
-            <s3>
-                <type>s3</type>
-                <endpoint>http://minio1:9001/root2/data/</endpoint>
-                <access_key_id>minio</access_key_id>
-                <secret_access_key>minio123</secret_access_key>
-                <send_metadata>true</send_metadata>
-                <list_object_keys_size>1</list_object_keys_size> <!-- To effectively test restore parallelism -->
-            </s3>
-            <hdd>
-                <type>local</type>
-                <path>/</path>
-            </hdd>
-        </disks>
-        <policies>
-            <s3>
-                <volumes>
-                    <main>
-                        <disk>s3</disk>
-                    </main>
-                    <external>
-                        <disk>hdd</disk>
-                    </external>
-                </volumes>
-            </s3>
-        </policies>
-    </storage_configuration>
-
-    <merge_tree>
-        <min_bytes_for_wide_part>0</min_bytes_for_wide_part>
-    </merge_tree>
-</clickhouse>
diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_without_zero_copy.xml b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_without_zero_copy.xml
deleted file mode 100644
index 9194d779d16..00000000000
--- a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_without_zero_copy.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<clickhouse>
-    <storage_configuration>
-        <disks>
-            <s3>
-                <type>s3</type>
-                <endpoint>http://minio1:9001/root/data/</endpoint>
-                <access_key_id>minio</access_key_id>
-                <secret_access_key>minio123</secret_access_key>
-                <send_metadata>true</send_metadata>
-                <list_object_keys_size>1</list_object_keys_size> <!-- To effectively test restore parallelism -->
-            </s3>
-            <hdd>
-                <type>local</type>
-                <path>/</path>
-            </hdd>
-        </disks>
-        <policies>
-            <s3>
-                <volumes>
-                    <main>
-                        <disk>s3</disk>
-                    </main>
-                    <external>
-                        <disk>hdd</disk>
-                    </external>
-                </volumes>
-            </s3>
-        </policies>
-    </storage_configuration>
-
-    <merge_tree>
-        <min_bytes_for_wide_part>0</min_bytes_for_wide_part>
-        <allow_remote_fs_zero_copy_replication>0</allow_remote_fs_zero_copy_replication>
-    </merge_tree>
-</clickhouse>
diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/test.py b/tests/integration/test_replicated_merge_tree_s3_restore/test.py
deleted file mode 100644
index 20a7668982b..00000000000
--- a/tests/integration/test_replicated_merge_tree_s3_restore/test.py
+++ /dev/null
@@ -1,263 +0,0 @@
-import os
-import logging
-import random
-import string
-import time
-
-import pytest
-from helpers.cluster import ClickHouseCluster
-
-
-COMMON_CONFIGS = ["configs/config.d/clusters.xml"]
-
-
-@pytest.fixture(scope="module")
-def cluster():
-    try:
-        cluster = ClickHouseCluster(__file__)
-
-        cluster.add_instance(
-            "node1z",
-            main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf.xml"],
-            macros={"cluster": "node_zero_copy", "replica": "0"},
-            with_minio=True,
-            with_zookeeper=True,
-            stay_alive=True,
-        )
-        cluster.add_instance(
-            "node2z",
-            main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf.xml"],
-            macros={"cluster": "node_zero_copy", "replica": "1"},
-            with_zookeeper=True,
-            stay_alive=True,
-        )
-        cluster.add_instance(
-            "node1n",
-            main_configs=COMMON_CONFIGS
-            + ["configs/config.d/storage_conf_without_zero_copy.xml"],
-            macros={"cluster": "node_no_zero_copy", "replica": "2"},
-            with_minio=True,
-            with_zookeeper=True,
-            stay_alive=True,
-        )
-        cluster.add_instance(
-            "node2n",
-            main_configs=COMMON_CONFIGS
-            + ["configs/config.d/storage_conf_without_zero_copy.xml"],
-            macros={"cluster": "node_no_zero_copy", "replica": "3"},
-            with_zookeeper=True,
-            stay_alive=True,
-        )
-        cluster.add_instance(
-            "node_another_bucket",
-            main_configs=COMMON_CONFIGS
-            + ["configs/config.d/storage_conf_another_bucket.xml"],
-            macros={"cluster": "node_another_bucket", "replica": "0"},
-            with_zookeeper=True,
-            stay_alive=True,
-        )
-
-        logging.info("Starting cluster...")
-        cluster.start()
-        logging.info("Cluster started")
-
-        yield cluster
-    finally:
-        cluster.shutdown()
-
-
-def random_string(length):
-    letters = string.ascii_letters
-    return "".join(random.choice(letters) for i in range(length))
-
-
-def create_table(node, table_name, schema, attach=False, db_atomic=False, uuid=""):
-    node.query(
-        "CREATE DATABASE IF NOT EXISTS s3 {on_cluster} ENGINE = {engine}".format(
-            engine="Atomic" if db_atomic else "Ordinary",
-            on_cluster="ON CLUSTER '{cluster}'",
-        ),
-        settings={"allow_deprecated_database_ordinary": 1},
-    )
-
-    create_table_statement = """
-        {create} TABLE s3.{table_name} {uuid} {on_cluster} (
-            key UInt32,
-            {schema}
-        ) ENGINE={engine}
-        PARTITION BY key
-        ORDER BY key
-        SETTINGS
-            storage_policy='s3',
-            old_parts_lifetime=600,
-            index_granularity=512
-        """.format(
-        create="ATTACH" if attach else "CREATE",
-        table_name=table_name,
-        uuid="UUID '{uuid}'".format(uuid=uuid) if db_atomic and uuid else "",
-        on_cluster="ON CLUSTER '{cluster}'",
-        schema=schema,
-        engine="ReplicatedMergeTree('/clickhouse/tables/{cluster}/test', '{replica}')",
-    )
-
-    node.query(create_table_statement)
-
-
-def purge_s3(cluster, bucket):
-    minio = cluster.minio_client
-    for obj in list(minio.list_objects(bucket, recursive=True)):
-        if str(obj.object_name).find(".SCHEMA_VERSION") != -1:
-            continue
-        minio.remove_object(bucket, obj.object_name)
-
-
-def drop_s3_metadata(node):
-    node.exec_in_container(
-        ["bash", "-c", "rm -rf /var/lib/clickhouse/disks/s3/*"], user="root"
-    )
-
-
-def drop_shadow_information(node):
-    node.exec_in_container(
-        ["bash", "-c", "rm -rf /var/lib/clickhouse/shadow/*"], user="root"
-    )
-
-
-def create_restore_file(node, revision=None, bucket=None, path=None, detached=None):
-    node.exec_in_container(
-        ["bash", "-c", "mkdir -p /var/lib/clickhouse/disks/s3/"], user="root"
-    )
-    node.exec_in_container(
-        ["bash", "-c", "touch /var/lib/clickhouse/disks/s3/restore"], user="root"
-    )
-
-    num_restore_options = 0
-
-    def add_restore_option(key, value):
-        nonlocal num_restore_options
-        to = ">>" if num_restore_options else ">"
-        node.exec_in_container(
-            [
-                "bash",
-                "-c",
-                f'echo -en "{key}={value}\n" {to} /var/lib/clickhouse/disks/s3/restore',
-            ],
-            user="root",
-        )
-        num_restore_options += 1
-
-    if revision:
-        add_restore_option("revision", revision)
-    if bucket:
-        add_restore_option("source_bucket", bucket)
-    if path:
-        add_restore_option("source_path", path)
-    if detached:
-        add_restore_option("detached", "true")
-
-
-def get_revision_counter(node, backup_number):
-    return int(
-        node.exec_in_container(
-            [
-                "bash",
-                "-c",
-                "cat /var/lib/clickhouse/disks/s3/shadow/{}/revision.txt".format(
-                    backup_number
-                ),
-            ],
-            user="root",
-        )
-    )
-
-
-def get_table_uuid(node, db_atomic, table):
-    uuid = ""
-    if db_atomic:
-        uuid = node.query(
-            "SELECT uuid FROM system.tables WHERE database='s3' AND table='{}' FORMAT TabSeparated".format(
-                table
-            )
-        ).strip()
-    return uuid
-
-
-def drop_table(cluster):
-    node_names = ["node1z", "node2z", "node1n", "node2n", "node_another_bucket"]
-
-    for node_name in node_names:
-        node = cluster.instances[node_name]
-        node.query("DROP TABLE IF EXISTS s3.test SYNC")
-        node.query("DROP DATABASE IF EXISTS s3 SYNC")
-
-        drop_s3_metadata(node)
-        drop_shadow_information(node)
-
-    buckets = [cluster.minio_bucket, cluster.minio_bucket_2]
-    for bucket in buckets:
-        purge_s3(cluster, bucket)
-
-
-@pytest.mark.parametrize("db_atomic", [False, True])
-@pytest.mark.parametrize("zero_copy", [False, True])
-def test_restore_another_bucket_path(cluster, db_atomic, zero_copy):
-    suffix = "z" if zero_copy else "n"
-    nodes = [cluster.instances[f"node1{suffix}"], cluster.instances[f"node2{suffix}"]]
-
-    keys = 100
-    data_columns = 10
-    size = 1
-
-    columns = []
-    for c in range(0, data_columns):
-        columns.append("data{c} String".format(c=c))
-    schema = ", ".join(columns)
-
-    create_table(nodes[0], "test", schema, db_atomic=db_atomic)
-    uuid = get_table_uuid(nodes[0], db_atomic, "test")
-
-    dropped_keys = 0
-
-    for key in range(0, keys):
-        node = nodes[key % 2]
-        node.query(
-            "INSERT INTO s3.test SELECT {key}, * FROM generateRandom('{schema}') LIMIT {size}".format(
-                key=key, schema=schema, size=size
-            )
-        )
-        if not (key % 3):
-            dropped_keys += 1
-            node.query("ALTER TABLE s3.test DROP PARTITION '{key}'".format(key=key))
-
-    for key in range(0, keys):
-        if not ((key + 1) % 3):
-            dropped_keys += 1
-            node.query("ALTER TABLE s3.test DROP PARTITION '{key}'".format(key=key))
-
-    nodes[0].query("SYSTEM SYNC REPLICA s3.test")
-    nodes[1].query("SYSTEM SYNC REPLICA s3.test")
-
-    # To ensure parts have merged
-    nodes[0].query("OPTIMIZE TABLE s3.test")
-    nodes[1].query("OPTIMIZE TABLE s3.test")
-
-    assert nodes[0].query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(size * (keys - dropped_keys))
-    assert nodes[1].query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(size * (keys - dropped_keys))
-
-    node_another_bucket = cluster.instances["node_another_bucket"]
-
-    create_restore_file(node_another_bucket, bucket="root")
-    node_another_bucket.restart_clickhouse(stop_start_wait_sec=120)
-    create_table(
-        node_another_bucket, "test", schema, attach=True, db_atomic=db_atomic, uuid=uuid
-    )
-
-    assert node_another_bucket.query(
-        "SELECT count(*) FROM s3.test FORMAT Values"
-    ) == "({})".format(size * (keys - dropped_keys))
-
-    drop_table(cluster)

From fd8297996188afe698ed58a049d525f3192fc455 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Sun, 7 May 2023 02:00:24 +0000
Subject: [PATCH 252/689] Automatic style fix

---
 tests/integration/ci-runner.py                | 13 +++---
 .../test_async_insert_memory/test.py          |  2 +-
 tests/integration/test_check_table/test.py    | 22 ++++-----
 .../test_cluster_discovery/test.py            |  2 +-
 tests/integration/test_mysql_protocol/test.py | 16 +++----
 tests/integration/test_partition/test.py      |  4 +-
 .../test_replicated_database/test.py          |  9 ++--
 .../s3_mocks/unstable_server.py               |  2 +-
 tests/integration/test_storage_s3/test.py     | 17 ++++---
 tests/integration/test_storage_url/test.py    | 22 ++++-----
 tests/integration/test_system_merges/test.py  | 45 ++++++++++---------
 11 files changed, 78 insertions(+), 76 deletions(-)

diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py
index be438dad492..d6d17abe725 100755
--- a/tests/integration/ci-runner.py
+++ b/tests/integration/ci-runner.py
@@ -295,11 +295,14 @@ class ClickhouseIntegrationTestsRunner:
     def _pre_pull_images(self, repo_path):
         image_cmd = self._get_runner_image_cmd(repo_path)
 
-        cmd = "cd {repo_path}/tests/integration && " "timeout -s 9 1h ./runner {runner_opts} {image_cmd} --pre-pull --command '{command}' ".format(
-            repo_path=repo_path,
-            runner_opts=self._get_runner_opts(),
-            image_cmd=image_cmd,
-            command=r""" echo Pre Pull finished """,
+        cmd = (
+            "cd {repo_path}/tests/integration && "
+            "timeout -s 9 1h ./runner {runner_opts} {image_cmd} --pre-pull --command '{command}' ".format(
+                repo_path=repo_path,
+                runner_opts=self._get_runner_opts(),
+                image_cmd=image_cmd,
+                command=r""" echo Pre Pull finished """,
+            )
         )
 
         for i in range(5):
diff --git a/tests/integration/test_async_insert_memory/test.py b/tests/integration/test_async_insert_memory/test.py
index f897007f7bb..5d2e5503680 100644
--- a/tests/integration/test_async_insert_memory/test.py
+++ b/tests/integration/test_async_insert_memory/test.py
@@ -43,7 +43,7 @@ def test_memory_usage():
 
     response = node.get_query_request(
         "SELECT groupArray(number) FROM numbers(1000000) SETTINGS max_memory_usage_for_user={}".format(
-            30 * (2 ** 23)
+            30 * (2**23)
         ),
         user="A",
     )
diff --git a/tests/integration/test_check_table/test.py b/tests/integration/test_check_table/test.py
index 60492141e69..613ac3fb35f 100644
--- a/tests/integration/test_check_table/test.py
+++ b/tests/integration/test_check_table/test.py
@@ -244,13 +244,10 @@ def test_check_replicated_table_corruption(started_cluster):
     )
 
     node1.query_with_retry("SYSTEM SYNC REPLICA replicated_mt_1")
-    assert (
-        node1.query(
-            "CHECK TABLE replicated_mt_1 PARTITION 201901",
-            settings={"check_query_single_value_result": 0},
-        )
-        == "{}\t1\t\n".format(part_name)
-    )
+    assert node1.query(
+        "CHECK TABLE replicated_mt_1 PARTITION 201901",
+        settings={"check_query_single_value_result": 0},
+    ) == "{}\t1\t\n".format(part_name)
     assert node1.query("SELECT count() from replicated_mt_1") == "4\n"
 
     remove_part_from_disk(node2, "replicated_mt_1", part_name)
@@ -262,11 +259,8 @@ def test_check_replicated_table_corruption(started_cluster):
     )
 
     node1.query("SYSTEM SYNC REPLICA replicated_mt_1")
-    assert (
-        node1.query(
-            "CHECK TABLE replicated_mt_1 PARTITION 201901",
-            settings={"check_query_single_value_result": 0},
-        )
-        == "{}\t1\t\n".format(part_name)
-    )
+    assert node1.query(
+        "CHECK TABLE replicated_mt_1 PARTITION 201901",
+        settings={"check_query_single_value_result": 0},
+    ) == "{}\t1\t\n".format(part_name)
     assert node1.query("SELECT count() from replicated_mt_1") == "4\n"
diff --git a/tests/integration/test_cluster_discovery/test.py b/tests/integration/test_cluster_discovery/test.py
index a2e7e15b956..ad3deb5b142 100644
--- a/tests/integration/test_cluster_discovery/test.py
+++ b/tests/integration/test_cluster_discovery/test.py
@@ -61,7 +61,7 @@ def check_on_cluster(
         print(f"Retry {retry}/{retries} unsuccessful, result: {node_results}")
 
         if retry != retries:
-            time.sleep(2 ** retry)
+            time.sleep(2**retry)
     else:
         msg = msg or f"Wrong '{what}' result"
         raise Exception(
diff --git a/tests/integration/test_mysql_protocol/test.py b/tests/integration/test_mysql_protocol/test.py
index e224853da32..5e03c4883b4 100644
--- a/tests/integration/test_mysql_protocol/test.py
+++ b/tests/integration/test_mysql_protocol/test.py
@@ -828,14 +828,14 @@ def test_types(started_cluster):
 
     result = cursor.fetchall()[0]
     expected = [
-        ("Int8_column", -(2 ** 7)),
-        ("UInt8_column", 2 ** 8 - 1),
-        ("Int16_column", -(2 ** 15)),
-        ("UInt16_column", 2 ** 16 - 1),
-        ("Int32_column", -(2 ** 31)),
-        ("UInt32_column", 2 ** 32 - 1),
-        ("Int64_column", -(2 ** 63)),
-        ("UInt64_column", 2 ** 64 - 1),
+        ("Int8_column", -(2**7)),
+        ("UInt8_column", 2**8 - 1),
+        ("Int16_column", -(2**15)),
+        ("UInt16_column", 2**16 - 1),
+        ("Int32_column", -(2**31)),
+        ("UInt32_column", 2**32 - 1),
+        ("Int64_column", -(2**63)),
+        ("UInt64_column", 2**64 - 1),
         ("String_column", "тест"),
         ("FixedString_column", "тест"),
         ("Float32_column", 1.5),
diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py
index 066d0bcf800..93f03f4420e 100644
--- a/tests/integration/test_partition/test.py
+++ b/tests/integration/test_partition/test.py
@@ -561,7 +561,9 @@ def test_make_clone_in_detached(started_cluster):
         ["cp", "-r", path + "all_0_0_0", path + "detached/broken_all_0_0_0"]
     )
     assert_eq_with_retry(instance, "select * from clone_in_detached", "\n")
-    assert ["broken_all_0_0_0",] == sorted(
+    assert [
+        "broken_all_0_0_0",
+    ] == sorted(
         instance.exec_in_container(["ls", path + "detached/"]).strip().split("\n")
     )
 
diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py
index e17170f630c..2ab2fe499ff 100644
--- a/tests/integration/test_replicated_database/test.py
+++ b/tests/integration/test_replicated_database/test.py
@@ -492,9 +492,12 @@ def test_alters_from_different_replicas(started_cluster):
     dummy_node.stop_clickhouse(kill=True)
 
     settings = {"distributed_ddl_task_timeout": 5}
-    assert "There are 1 unfinished hosts (0 of them are currently active)" in competing_node.query_and_get_error(
-        "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added0 UInt32;",
-        settings=settings,
+    assert (
+        "There are 1 unfinished hosts (0 of them are currently active)"
+        in competing_node.query_and_get_error(
+            "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added0 UInt32;",
+            settings=settings,
+        )
     )
     settings = {
         "distributed_ddl_task_timeout": 5,
diff --git a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py
index 16697596306..103dd30340c 100644
--- a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py
+++ b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py
@@ -8,7 +8,7 @@ import sys
 
 def gen_n_digit_number(n):
     assert 0 < n < 19
-    return random.randint(10 ** (n - 1), 10 ** n - 1)
+    return random.randint(10 ** (n - 1), 10**n - 1)
 
 
 sum_in_4_column = 0
diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py
index 78e33534022..d9ac70f51ad 100644
--- a/tests/integration/test_storage_s3/test.py
+++ b/tests/integration/test_storage_s3/test.py
@@ -538,13 +538,16 @@ def test_multipart(started_cluster, maybe_auth, positive):
         assert csv_data == get_s3_file_content(started_cluster, bucket, filename)
 
     # select uploaded data from many threads
-    select_query = "select sum(column1), sum(column2), sum(column3) " "from s3('http://{host}:{port}/{bucket}/{filename}', {auth}'CSV', '{table_format}')".format(
-        host=started_cluster.minio_redirect_host,
-        port=started_cluster.minio_redirect_port,
-        bucket=bucket,
-        filename=filename,
-        auth=maybe_auth,
-        table_format=table_format,
+    select_query = (
+        "select sum(column1), sum(column2), sum(column3) "
+        "from s3('http://{host}:{port}/{bucket}/{filename}', {auth}'CSV', '{table_format}')".format(
+            host=started_cluster.minio_redirect_host,
+            port=started_cluster.minio_redirect_port,
+            bucket=bucket,
+            filename=filename,
+            auth=maybe_auth,
+            table_format=table_format,
+        )
     )
     try:
         select_result = run_query(
diff --git a/tests/integration/test_storage_url/test.py b/tests/integration/test_storage_url/test.py
index 2ef04c444f8..f3ed087d3d0 100644
--- a/tests/integration/test_storage_url/test.py
+++ b/tests/integration/test_storage_url/test.py
@@ -49,21 +49,15 @@ def test_table_function_url_access_rights():
         f"SELECT * FROM url('http://nginx:80/test_1', 'TSV')", user="u1"
     )
 
-    assert (
-        node1.query(
-            f"DESCRIBE TABLE url('http://nginx:80/test_1', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')",
-            user="u1",
-        )
-        == TSV([["column1", "UInt32"], ["column2", "UInt32"], ["column3", "UInt32"]])
-    )
+    assert node1.query(
+        f"DESCRIBE TABLE url('http://nginx:80/test_1', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')",
+        user="u1",
+    ) == TSV([["column1", "UInt32"], ["column2", "UInt32"], ["column3", "UInt32"]])
 
-    assert (
-        node1.query(
-            f"DESCRIBE TABLE url('http://nginx:80/not-exist', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')",
-            user="u1",
-        )
-        == TSV([["column1", "UInt32"], ["column2", "UInt32"], ["column3", "UInt32"]])
-    )
+    assert node1.query(
+        f"DESCRIBE TABLE url('http://nginx:80/not-exist', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')",
+        user="u1",
+    ) == TSV([["column1", "UInt32"], ["column2", "UInt32"], ["column3", "UInt32"]])
 
     expected_error = "necessary to have grant URL ON *.*"
     assert expected_error in node1.query_and_get_error(
diff --git a/tests/integration/test_system_merges/test.py b/tests/integration/test_system_merges/test.py
index d4b9b403871..ff303afe19e 100644
--- a/tests/integration/test_system_merges/test.py
+++ b/tests/integration/test_system_merges/test.py
@@ -202,33 +202,36 @@ def test_mutation_simple(started_cluster, replicated):
             sleep_time=0.1,
         )
 
-        assert split_tsv(
-            node_check.query(
-                """
+        assert (
+            split_tsv(
+                node_check.query(
+                    """
             SELECT database, table, num_parts, source_part_names, source_part_paths, result_part_name, result_part_path, partition_id, is_mutation
                 FROM system.merges
                 WHERE table = '{name}'
         """.format(
-                    name=table_name
+                        name=table_name
+                    )
                 )
             )
-        ) == [
-            [
-                db_name,
-                table_name,
-                "1",
-                "['{}']".format(part),
-                "['{clickhouse}/{table_path}/{}/']".format(
-                    part, clickhouse=clickhouse_path, table_path=table_path
-                ),
-                result_part,
-                "{clickhouse}/{table_path}/{}/".format(
-                    result_part, clickhouse=clickhouse_path, table_path=table_path
-                ),
-                "all",
-                "1",
-            ],
-        ]
+            == [
+                [
+                    db_name,
+                    table_name,
+                    "1",
+                    "['{}']".format(part),
+                    "['{clickhouse}/{table_path}/{}/']".format(
+                        part, clickhouse=clickhouse_path, table_path=table_path
+                    ),
+                    result_part,
+                    "{clickhouse}/{table_path}/{}/".format(
+                        result_part, clickhouse=clickhouse_path, table_path=table_path
+                    ),
+                    "all",
+                    "1",
+                ],
+            ]
+        )
         t.join()
 
         assert (

From 72e1f751bb7c819f9c8bafbda915a983185d7edc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 04:16:18 +0200
Subject: [PATCH 253/689] Fix error in #48300

---
 src/Functions/transform.cpp                   | 30 +++++++++----------
 .../02732_transform_fuzz.reference            |  0
 .../0_stateless/02732_transform_fuzz.sql      |  1 +
 3 files changed, 16 insertions(+), 15 deletions(-)
 create mode 100644 tests/queries/0_stateless/02732_transform_fuzz.reference
 create mode 100644 tests/queries/0_stateless/02732_transform_fuzz.sql

diff --git a/src/Functions/transform.cpp b/src/Functions/transform.cpp
index 33e03b541e9..9970012ef4a 100644
--- a/src/Functions/transform.cpp
+++ b/src/Functions/transform.cpp
@@ -4,9 +4,7 @@
 #include <Columns/ColumnArray.h>
 #include <Columns/ColumnConst.h>
 #include <Columns/ColumnDecimal.h>
-#include <Columns/ColumnNullable.h>
 #include <Columns/ColumnString.h>
-#include <Columns/ColumnsNumber.h>
 #include <Core/DecimalFunctions.h>
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/getLeastSupertype.h>
@@ -15,10 +13,7 @@
 #include <Functions/IFunction.h>
 #include <Interpreters/castColumn.h>
 #include <Interpreters/convertFieldToType.h>
-#include <base/StringRef.h>
 #include <Common/Arena.h>
-#include <Common/FieldVisitorConvertToNumber.h>
-#include <Common/FieldVisitorDump.h>
 #include <Common/HashTable/HashMap.h>
 #include <Common/typeid_cast.h>
 
@@ -635,22 +630,27 @@ namespace
             }
         }
 
-        static void checkAllowedType(const DataTypePtr & dt)
+        static void checkAllowedType(const DataTypePtr & type)
         {
-            if (dt->isNullable())
-                checkAllowedTypeHelper(static_cast<const DataTypeNullable *>(dt.get())->getNestedType());
+            if (type->isNullable())
+                checkAllowedTypeHelper(static_cast<const DataTypeNullable *>(type.get())->getNestedType());
             else
-                checkAllowedTypeHelper(dt);
+                checkAllowedTypeHelper(type);
         }
 
-        static void checkAllowedTypeHelper(const DataTypePtr & dt)
+        static void checkAllowedTypeHelper(const DataTypePtr & type)
         {
-            if (isStringOrFixedString(dt))
+            if (isStringOrFixedString(type))
                 return;
-            auto dtsize = dt->getMaximumSizeOfValueInMemory();
-            if (dtsize <= sizeof(UInt64))
-                return;
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected type {} in function 'transform'", dt->getName());
+
+            if (type->haveMaximumSizeOfValue())
+            {
+                auto data_type_size = type->getMaximumSizeOfValueInMemory();
+                if (data_type_size <= sizeof(UInt64))
+                    return;
+            }
+
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected type {} in function 'transform'", type->getName());
         }
 
         /// Can be called from different threads. It works only on the first call.
diff --git a/tests/queries/0_stateless/02732_transform_fuzz.reference b/tests/queries/0_stateless/02732_transform_fuzz.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/02732_transform_fuzz.sql b/tests/queries/0_stateless/02732_transform_fuzz.sql
new file mode 100644
index 00000000000..c2918d4da81
--- /dev/null
+++ b/tests/queries/0_stateless/02732_transform_fuzz.sql
@@ -0,0 +1 @@
+SELECT caseWithExpr(arrayReduce(NULL, []), []); -- { serverError BAD_ARGUMENTS }

From cfb35cb251c0e038e04543864d36f952b608e26b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 04:42:46 +0200
Subject: [PATCH 254/689] Fix typo: "as much slots" -> "as many slots"

---
 programs/server/Server.cpp                    |   4 -
 src/Common/ConcurrencyControl.cpp             | 173 ++++++++++++++++++
 src/Common/ConcurrencyControl.h               | 164 ++---------------
 .../tests/gtest_concurrency_control.cpp       |   6 +-
 4 files changed, 196 insertions(+), 151 deletions(-)
 create mode 100644 src/Common/ConcurrencyControl.cpp

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index bbd536d9300..3f415462178 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -27,7 +27,6 @@
 #include <Common/ConcurrencyControl.h>
 #include <Common/Macros.h>
 #include <Common/ShellCommand.h>
-#include <Common/StringUtils/StringUtils.h>
 #include <Common/ZooKeeper/ZooKeeper.h>
 #include <Common/ZooKeeper/ZooKeeperNodeCache.h>
 #include <Common/getMultipleKeysFromConfig.h>
@@ -98,9 +97,7 @@
 #include "config_version.h"
 
 #if defined(OS_LINUX)
-#    include <cstddef>
 #    include <cstdlib>
-#    include <sys/socket.h>
 #    include <sys/un.h>
 #    include <sys/mman.h>
 #    include <sys/ptrace.h>
@@ -108,7 +105,6 @@
 #endif
 
 #if USE_SSL
-#    include <Poco/Net/Context.h>
 #    include <Poco/Net/SecureServerSocket.h>
 #endif
 
diff --git a/src/Common/ConcurrencyControl.cpp b/src/Common/ConcurrencyControl.cpp
new file mode 100644
index 00000000000..de46f0e90ca
--- /dev/null
+++ b/src/Common/ConcurrencyControl.cpp
@@ -0,0 +1,173 @@
+#include <Common/ConcurrencyControl.h>
+#include <Common/Exception.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
+ConcurrencyControl::Slot::~Slot()
+{
+    allocation->release();
+}
+
+ConcurrencyControl::Slot::Slot(AllocationPtr && allocation_)
+    : allocation(std::move(allocation_))
+{
+}
+
+ConcurrencyControl::Allocation::~Allocation()
+{
+    // We have to lock parent's mutex to avoid race with grant()
+    // NOTE: shortcut can be added, but it requires Allocation::mutex lock even to check if shortcut is possible
+    parent.free(this);
+}
+
+[[nodiscard]] ConcurrencyControl::SlotPtr ConcurrencyControl::Allocation::tryAcquire()
+{
+    SlotCount value = granted.load();
+    while (value)
+    {
+        if (granted.compare_exchange_strong(value, value - 1))
+        {
+            std::unique_lock lock{mutex};
+            return SlotPtr(new Slot(shared_from_this())); // can't use std::make_shared due to private ctor
+        }
+    }
+    return {}; // avoid unnecessary locking
+}
+
+ConcurrencyControl::SlotCount ConcurrencyControl::Allocation::grantedCount() const
+{
+    return granted;
+}
+
+ConcurrencyControl::Allocation::Allocation(ConcurrencyControl & parent_, SlotCount limit_, SlotCount granted_, Waiters::iterator waiter_)
+    : parent(parent_)
+    , limit(limit_)
+    , allocated(granted_)
+    , granted(granted_)
+    , waiter(waiter_)
+{
+    if (allocated < limit)
+        *waiter = this;
+}
+
+// Grant single slot to allocation, returns true iff more slot(s) are required
+bool ConcurrencyControl::Allocation::grant()
+{
+    std::unique_lock lock{mutex};
+    granted++;
+    allocated++;
+    return allocated < limit;
+}
+
+// Release one slot and grant it to other allocation if required
+void ConcurrencyControl::Allocation::release()
+{
+    parent.release(1);
+    std::unique_lock lock{mutex};
+    released++;
+    if (released > allocated)
+        abort();
+}
+
+ConcurrencyControl::ConcurrencyControl()
+    : cur_waiter(waiters.end())
+{
+}
+
+ConcurrencyControl::~ConcurrencyControl()
+{
+    if (!waiters.empty())
+        abort();
+}
+
+[[nodiscard]] ConcurrencyControl::AllocationPtr ConcurrencyControl::allocate(SlotCount min, SlotCount max)
+{
+    if (min > max)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "ConcurrencyControl: invalid allocation requirements");
+
+    std::unique_lock lock{mutex};
+
+    // Acquire as many slots as we can, but not lower than `min`
+    SlotCount granted = std::max(min, std::min(max, available(lock)));
+    cur_concurrency += granted;
+
+    // Create allocation and start waiting if more slots are required
+    if (granted < max)
+        return AllocationPtr(new Allocation(*this, max, granted,
+            waiters.insert(cur_waiter, nullptr /* pointer is set by Allocation ctor */)));
+    else
+        return AllocationPtr(new Allocation(*this, max, granted));
+}
+
+void ConcurrencyControl::setMaxConcurrency(ConcurrencyControl::SlotCount value)
+{
+    std::unique_lock lock{mutex};
+    max_concurrency = std::max<SlotCount>(1, value); // never allow max_concurrency to be zero
+    schedule(lock);
+}
+
+ConcurrencyControl & ConcurrencyControl::instance()
+{
+    static ConcurrencyControl result;
+    return result;
+}
+
+void ConcurrencyControl::free(Allocation * allocation)
+{
+    // Allocation is allowed to be canceled even if there are:
+    //  - `amount`: granted slots (acquired slots are not possible, because Slot holds AllocationPtr)
+    //  - `waiter`: active waiting for more slots to be allocated
+    // Thus Allocation destruction may require the following lock, to avoid race conditions
+    std::unique_lock lock{mutex};
+    auto [amount, waiter] = allocation->cancel();
+
+    cur_concurrency -= amount;
+    if (waiter)
+    {
+        if (cur_waiter == *waiter)
+            cur_waiter = waiters.erase(*waiter);
+        else
+            waiters.erase(*waiter);
+    }
+    schedule(lock);
+}
+
+void ConcurrencyControl::release(SlotCount amount)
+{
+    std::unique_lock lock{mutex};
+    cur_concurrency -= amount;
+    schedule(lock);
+}
+
+// Round-robin scheduling of available slots among waiting allocations
+void ConcurrencyControl::schedule(std::unique_lock<std::mutex> &)
+{
+    while (cur_concurrency < max_concurrency && !waiters.empty())
+    {
+        cur_concurrency++;
+        if (cur_waiter == waiters.end())
+            cur_waiter = waiters.begin();
+        Allocation * allocation = *cur_waiter;
+        if (allocation->grant())
+            ++cur_waiter;
+        else
+            cur_waiter = waiters.erase(cur_waiter); // last required slot has just been granted -- stop waiting
+    }
+}
+
+ConcurrencyControl::SlotCount ConcurrencyControl::available(std::unique_lock<std::mutex> &) const
+{
+    if (cur_concurrency < max_concurrency)
+        return max_concurrency - cur_concurrency;
+    else
+        return 0;
+}
+
+}
diff --git a/src/Common/ConcurrencyControl.h b/src/Common/ConcurrencyControl.h
index 14e54363e04..ba87e3b3a0c 100644
--- a/src/Common/ConcurrencyControl.h
+++ b/src/Common/ConcurrencyControl.h
@@ -5,17 +5,10 @@
 #include <mutex>
 #include <memory>
 #include <list>
-#include <condition_variable>
 
-#include <Common/Exception.h>
 
 namespace DB
 {
-namespace ErrorCodes
-{
-    extern const int LOGICAL_ERROR;
-}
-}
 
 /*
  * Controls how many threads can be allocated for a query (or another activity).
@@ -53,17 +46,12 @@ public:
     // Scoped guard for acquired slot, see Allocation::tryAcquire()
     struct Slot : boost::noncopyable
     {
-        ~Slot()
-        {
-            allocation->release();
-        }
+        ~Slot();
 
     private:
         friend struct Allocation; // for ctor
 
-        explicit Slot(AllocationPtr && allocation_)
-            : allocation(std::move(allocation_))
-        {}
+        explicit Slot(AllocationPtr && allocation_);
 
         AllocationPtr allocation;
     };
@@ -74,47 +62,18 @@ public:
     // Manages group of slots for a single query, see ConcurrencyControl::allocate(min, max)
     struct Allocation : std::enable_shared_from_this<Allocation>, boost::noncopyable
     {
-        ~Allocation()
-        {
-            // We have to lock parent's mutex to avoid race with grant()
-            // NOTE: shortcut can be added, but it requires Allocation::mutex lock even to check if shortcut is possible
-            parent.free(this);
-        }
+        ~Allocation();
 
         // Take one already granted slot if available. Lock-free iff there is no granted slot.
-        [[nodiscard]] SlotPtr tryAcquire()
-        {
-            SlotCount value = granted.load();
-            while (value)
-            {
-                if (granted.compare_exchange_strong(value, value - 1))
-                {
-                    std::unique_lock lock{mutex};
-                    return SlotPtr(new Slot(shared_from_this())); // can't use std::make_shared due to private ctor
-                }
-            }
-            return {}; // avoid unnecessary locking
-        }
+        [[nodiscard]] SlotPtr tryAcquire();
 
-        SlotCount grantedCount() const
-        {
-            return granted;
-        }
+        SlotCount grantedCount() const;
 
     private:
         friend struct Slot; // for release()
         friend class ConcurrencyControl; // for grant(), free() and ctor
 
-        Allocation(ConcurrencyControl & parent_, SlotCount limit_, SlotCount granted_, Waiters::iterator waiter_ = {})
-            : parent(parent_)
-            , limit(limit_)
-            , allocated(granted_)
-            , granted(granted_)
-            , waiter(waiter_)
-        {
-            if (allocated < limit)
-                *waiter = this;
-        }
+        Allocation(ConcurrencyControl & parent_, SlotCount limit_, SlotCount granted_, Waiters::iterator waiter_ = {});
 
         auto cancel()
         {
@@ -126,23 +85,10 @@ public:
         }
 
         // Grant single slot to allocation, returns true iff more slot(s) are required
-        bool grant()
-        {
-            std::unique_lock lock{mutex};
-            granted++;
-            allocated++;
-            return allocated < limit;
-        }
+        bool grant();
 
         // Release one slot and grant it to other allocation if required
-        void release()
-        {
-            parent.release(1);
-            std::unique_lock lock{mutex};
-            released++;
-            if (released > allocated)
-                abort();
-        }
+        void release();
 
         ConcurrencyControl & parent;
         const SlotCount limit;
@@ -157,106 +103,32 @@ public:
     };
 
 public:
-    ConcurrencyControl()
-        : cur_waiter(waiters.end())
-    {}
+    ConcurrencyControl();
 
     // WARNING: all Allocation objects MUST be destructed before ConcurrencyControl
     // NOTE: Recommended way to achieve this is to use `instance()` and do graceful shutdown of queries
-    ~ConcurrencyControl()
-    {
-        if (!waiters.empty())
-            abort();
-    }
+    ~ConcurrencyControl();
 
     // Allocate at least `min` and at most `max` slots.
     // If not all `max` slots were successfully allocated, a subscription for later allocation is created
     // Use `Allocation::tryAcquire()` to acquire allocated slot, before running a thread.
-    [[nodiscard]] AllocationPtr allocate(SlotCount min, SlotCount max)
-    {
-        if (min > max)
-            throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "ConcurrencyControl: invalid allocation requirements");
+    [[nodiscard]] AllocationPtr allocate(SlotCount min, SlotCount max);
 
-        std::unique_lock lock{mutex};
+    void setMaxConcurrency(SlotCount value);
 
-        // Acquire as much slots as we can, but not lower than `min`
-        SlotCount granted = std::max(min, std::min(max, available(lock)));
-        cur_concurrency += granted;
-
-        // Create allocation and start waiting if more slots are required
-        if (granted < max)
-            return AllocationPtr(new Allocation(*this, max, granted,
-                waiters.insert(cur_waiter, nullptr /* pointer is set by Allocation ctor */)));
-        else
-            return AllocationPtr(new Allocation(*this, max, granted));
-    }
-
-    void setMaxConcurrency(SlotCount value)
-    {
-        std::unique_lock lock{mutex};
-        max_concurrency = std::max<SlotCount>(1, value); // never allow max_concurrency to be zero
-        schedule(lock);
-    }
-
-    static ConcurrencyControl & instance()
-    {
-        static ConcurrencyControl result;
-        return result;
-    }
+    static ConcurrencyControl & instance();
 
 private:
     friend struct Allocation; // for free() and release()
 
-    void free(Allocation * allocation)
-    {
-        // Allocation is allowed to be canceled even if there are:
-        //  - `amount`: granted slots (acquired slots are not possible, because Slot holds AllocationPtr)
-        //  - `waiter`: active waiting for more slots to be allocated
-        // Thus Allocation destruction may require the following lock, to avoid race conditions
-        std::unique_lock lock{mutex};
-        auto [amount, waiter] = allocation->cancel();
+    void free(Allocation * allocation);
 
-        cur_concurrency -= amount;
-        if (waiter)
-        {
-            if (cur_waiter == *waiter)
-                cur_waiter = waiters.erase(*waiter);
-            else
-                waiters.erase(*waiter);
-        }
-        schedule(lock);
-    }
-
-    void release(SlotCount amount)
-    {
-        std::unique_lock lock{mutex};
-        cur_concurrency -= amount;
-        schedule(lock);
-    }
+    void release(SlotCount amount);
 
     // Round-robin scheduling of available slots among waiting allocations
-    void schedule(std::unique_lock<std::mutex> &)
-    {
-        while (cur_concurrency < max_concurrency && !waiters.empty())
-        {
-            cur_concurrency++;
-            if (cur_waiter == waiters.end())
-                cur_waiter = waiters.begin();
-            Allocation * allocation = *cur_waiter;
-            if (allocation->grant())
-                ++cur_waiter;
-            else
-                cur_waiter = waiters.erase(cur_waiter); // last required slot has just been granted -- stop waiting
-        }
-    }
+    void schedule(std::unique_lock<std::mutex> &);
 
-    SlotCount available(std::unique_lock<std::mutex> &) const
-    {
-        if (cur_concurrency < max_concurrency)
-            return max_concurrency - cur_concurrency;
-        else
-            return 0;
-    }
+    SlotCount available(std::unique_lock<std::mutex> &) const;
 
     std::mutex mutex;
     Waiters waiters;
@@ -264,3 +136,5 @@ private:
     SlotCount max_concurrency = Unlimited;
     SlotCount cur_concurrency = 0;
 };
+
+}
diff --git a/src/Common/tests/gtest_concurrency_control.cpp b/src/Common/tests/gtest_concurrency_control.cpp
index 2ffb16511f3..852f4e72cf7 100644
--- a/src/Common/tests/gtest_concurrency_control.cpp
+++ b/src/Common/tests/gtest_concurrency_control.cpp
@@ -9,6 +9,8 @@
 #include <Common/ConcurrencyControl.h>
 #include <Common/randomSeed.h>
 
+using namespace DB;
+
 struct ConcurrencyControlTest
 {
     ConcurrencyControl cc;
@@ -276,9 +278,9 @@ TEST(ConcurrencyControl, MultipleThreads)
             queries.emplace_back([&, max_threads = max_threads_distribution(rng)]
             {
                 run_query(max_threads);
-                finished++;
+                ++finished;
             });
-            started++;
+            ++started;
         }
         sleepForMicroseconds(5); // wait some queries to finish
         t.cc.setMaxConcurrency(cfg_max_concurrency - started % 3); // emulate configuration updates

From d0181a68a9143535d2497bb57a453e6a8ad188d1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 06:05:28 +0300
Subject: [PATCH 255/689] Update 01051_system_stack_trace.sql

---
 tests/queries/0_stateless/01051_system_stack_trace.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01051_system_stack_trace.sql b/tests/queries/0_stateless/01051_system_stack_trace.sql
index a7e6660d06e..7eb2a05dc87 100644
--- a/tests/queries/0_stateless/01051_system_stack_trace.sql
+++ b/tests/queries/0_stateless/01051_system_stack_trace.sql
@@ -1,4 +1,4 @@
-SET storage_system_stack_trace_pipe_read_timeout_ms = 10000;
+SET storage_system_stack_trace_pipe_read_timeout_ms = 1000;
 
 -- { echo }
 SELECT count() > 0 FROM system.stack_trace WHERE query_id != '';

From fd9c9bb25bba37a7cd964be9f6ca01b40f68db53 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Fri, 5 May 2023 01:52:33 +0000
Subject: [PATCH 256/689] Split ReadWriteBufferFromHTTP.h into .h and .cpp file

---
 src/IO/ReadWriteBufferFromHTTP.cpp | 899 +++++++++++++++++++++++++++++
 src/IO/ReadWriteBufferFromHTTP.h   | 856 +++------------------------
 2 files changed, 967 insertions(+), 788 deletions(-)
 create mode 100644 src/IO/ReadWriteBufferFromHTTP.cpp

diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp
new file mode 100644
index 00000000000..8c3ab704d2b
--- /dev/null
+++ b/src/IO/ReadWriteBufferFromHTTP.cpp
@@ -0,0 +1,899 @@
+#include "ReadWriteBufferFromHTTP.h"
+
+namespace ProfileEvents
+{
+extern const Event ReadBufferSeekCancelConnection;
+}
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int TOO_MANY_REDIRECTS;
+    extern const int HTTP_RANGE_NOT_SATISFIABLE;
+    extern const int BAD_ARGUMENTS;
+    extern const int CANNOT_SEEK_THROUGH_FILE;
+    extern const int SEEK_POSITION_OUT_OF_BOUND;
+    extern const int UNKNOWN_FILE_SIZE;
+}
+
+template <typename TSessionFactory>
+UpdatableSession<TSessionFactory>::UpdatableSession(const Poco::URI & uri, UInt64 max_redirects_, std::shared_ptr<TSessionFactory> session_factory_)
+    : max_redirects{max_redirects_}
+    , initial_uri(uri)
+    , session_factory(std::move(session_factory_))
+{
+    session = session_factory->buildNewSession(uri);
+}
+
+template <typename TSessionFactory>
+typename UpdatableSession<TSessionFactory>::SessionPtr UpdatableSession<TSessionFactory>::getSession() { return session; }
+
+template <typename TSessionFactory>
+void UpdatableSession<TSessionFactory>::updateSession(const Poco::URI & uri)
+{
+    ++redirects;
+    if (redirects <= max_redirects)
+        session = session_factory->buildNewSession(uri);
+    else
+        throw Exception(ErrorCodes::TOO_MANY_REDIRECTS, "Too many redirects while trying to access {}", initial_uri.toString());
+}
+
+template <typename TSessionFactory>
+std::shared_ptr<UpdatableSession<TSessionFactory>> UpdatableSession<TSessionFactory>::clone(const Poco::URI & uri)
+{
+    return std::make_shared<UpdatableSession<TSessionFactory>>(uri, max_redirects, session_factory);
+}
+
+
+namespace detail
+{
+
+static bool isRetriableError(const Poco::Net::HTTPResponse::HTTPStatus http_status) noexcept
+{
+    static constexpr std::array non_retriable_errors{
+        Poco::Net::HTTPResponse::HTTPStatus::HTTP_BAD_REQUEST,
+        Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED,
+        Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND,
+        Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN,
+        Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_IMPLEMENTED,
+        Poco::Net::HTTPResponse::HTTPStatus::HTTP_METHOD_NOT_ALLOWED};
+
+    return std::all_of(
+        non_retriable_errors.begin(), non_retriable_errors.end(), [&](const auto status) { return http_status != status; });
+}
+
+static Poco::URI getUriAfterRedirect(const Poco::URI & prev_uri, Poco::Net::HTTPResponse & response)
+{
+    auto location = response.get("Location");
+    auto location_uri = Poco::URI(location);
+    if (!location_uri.isRelative())
+        return location_uri;
+    /// Location header contains relative path. So we need to concatenate it
+    /// with path from the original URI and normalize it.
+    auto path = std::filesystem::weakly_canonical(std::filesystem::path(prev_uri.getPath()) / location);
+    location_uri = prev_uri;
+    location_uri.setPath(path);
+    return location_uri;
+}
+
+template <typename UpdatableSessionPtr>
+bool ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::withPartialContent(const HTTPRange & range) const
+{
+    /**
+     * Add range header if we have some passed range
+     * or if we want to retry GET request on purpose.
+     */
+    return range.begin || range.end || retry_with_range_header;
+}
+
+template <typename UpdatableSessionPtr>
+size_t ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::getRangeBegin() const { return read_range.begin.value_or(0); }
+
+template <typename UpdatableSessionPtr>
+size_t ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::getOffset() const { return getRangeBegin() + offset_from_begin_pos; }
+
+template <typename UpdatableSessionPtr>
+std::istream * ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::callImpl(
+    UpdatableSessionPtr & current_session, Poco::URI uri_, Poco::Net::HTTPResponse & response,
+    const std::string & method_, bool for_object_info)
+{
+    // With empty path poco will send "POST  HTTP/1.1" its bug.
+    if (uri_.getPath().empty())
+        uri_.setPath("/");
+
+    Poco::Net::HTTPRequest request(method_, uri_.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1);
+    request.setHost(uri_.getHost()); // use original, not resolved host name in header
+
+    if (out_stream_callback)
+        request.setChunkedTransferEncoding(true);
+    else if (method == Poco::Net::HTTPRequest::HTTP_POST)
+        request.setContentLength(0);    /// No callback - no body
+
+    for (auto & [header, value] : http_header_entries)
+        request.set(header, value);
+
+    std::optional<HTTPRange> range;
+    if (!for_object_info)
+    {
+        if (withPartialContent(read_range))
+            range = HTTPRange{getOffset(), read_range.end};
+    }
+
+    if (range)
+    {
+        String range_header_value;
+        if (range->end)
+            range_header_value = fmt::format("bytes={}-{}", *range->begin, *range->end);
+        else
+            range_header_value = fmt::format("bytes={}-", *range->begin);
+        LOG_TEST(log, "Adding header: Range: {}", range_header_value);
+        request.set("Range", range_header_value);
+    }
+
+    if (!credentials.getUsername().empty())
+        credentials.authenticate(request);
+
+    LOG_TRACE(log, "Sending request to {}", uri_.toString());
+
+    auto sess = current_session->getSession();
+    try
+    {
+        auto & stream_out = sess->sendRequest(request);
+
+        if (out_stream_callback)
+            out_stream_callback(stream_out);
+
+        auto result_istr = receiveResponse(*sess, request, response, true);
+        response.getCookies(cookies);
+
+        /// we can fetch object info while the request is being processed
+        /// and we don't want to override any context used by it
+        if (!for_object_info)
+            content_encoding = response.get("Content-Encoding", "");
+
+        return result_istr;
+    }
+    catch (const Poco::Exception & e)
+    {
+        /// We use session data storage as storage for exception text
+        /// Depend on it we can deduce to reconnect session or reresolve session host
+        sess->attachSessionData(e.message());
+        throw;
+    }
+}
+
+template <typename UpdatableSessionPtr>
+size_t ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::getFileSize()
+{
+    if (!file_info)
+        file_info = getFileInfo();
+
+    if (file_info->file_size)
+        return *file_info->file_size;
+
+    throw Exception(ErrorCodes::UNKNOWN_FILE_SIZE, "Cannot find out file size for: {}", uri.toString());
+}
+
+template <typename UpdatableSessionPtr>
+bool ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::checkIfActuallySeekable()
+{
+    if (!file_info)
+        file_info = getFileInfo();
+    return file_info->seekable;
+}
+
+template <typename UpdatableSessionPtr>
+String ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::getFileName() const { return uri.toString(); }
+
+template <typename UpdatableSessionPtr>
+void ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::getHeadResponse(Poco::Net::HTTPResponse & response)
+{
+    for (size_t i = 0; i < settings.http_max_tries; ++i)
+    {
+        try
+        {
+            callWithRedirects(response, Poco::Net::HTTPRequest::HTTP_HEAD, true, true);
+            break;
+        }
+        catch (const Poco::Exception & e)
+        {
+            if (i == settings.http_max_tries - 1 || !isRetriableError(response.getStatus()))
+                throw;
+
+            LOG_ERROR(log, "Failed to make HTTP_HEAD request to {}. Error: {}", uri.toString(), e.displayText());
+        }
+    }
+}
+
+template <typename UpdatableSessionPtr>
+void ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::setupExternalBuffer()
+{
+    /**
+    * use_external_buffer -- means we read into the buffer which
+    * was passed to us from somewhere else. We do not check whether
+    * previously returned buffer was read or not (no hasPendingData() check is needed),
+    * because this branch means we are prefetching data,
+    * each nextImpl() call we can fill a different buffer.
+    */
+    impl->set(internal_buffer.begin(), internal_buffer.size());
+    assert(working_buffer.begin() != nullptr);
+    assert(!internal_buffer.empty());
+}
+
+template <typename UpdatableSessionPtr>
+ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::ReadWriteBufferFromHTTPBase(
+    UpdatableSessionPtr session_,
+    Poco::URI uri_,
+    const Poco::Net::HTTPBasicCredentials & credentials_,
+    const std::string & method_,
+    OutStreamCallback out_stream_callback_,
+    size_t buffer_size_,
+    const ReadSettings & settings_,
+    HTTPHeaderEntries http_header_entries_,
+    const RemoteHostFilter * remote_host_filter_,
+    bool delay_initialization,
+    bool use_external_buffer_,
+    bool http_skip_not_found_url_,
+    std::optional<HTTPFileInfo> file_info_)
+    : SeekableReadBuffer(nullptr, 0)
+    , uri {uri_}
+    , method {!method_.empty() ? method_ : out_stream_callback_ ? Poco::Net::HTTPRequest::HTTP_POST : Poco::Net::HTTPRequest::HTTP_GET}
+    , session {session_}
+    , out_stream_callback {out_stream_callback_}
+    , credentials {credentials_}
+    , http_header_entries {std::move(http_header_entries_)}
+    , remote_host_filter {remote_host_filter_}
+    , buffer_size {buffer_size_}
+    , use_external_buffer {use_external_buffer_}
+    , file_info(file_info_)
+    , http_skip_not_found_url(http_skip_not_found_url_)
+    , settings {settings_}
+    , log(&Poco::Logger::get("ReadWriteBufferFromHTTP"))
+{
+    if (settings.http_max_tries <= 0 || settings.http_retry_initial_backoff_ms <= 0
+        || settings.http_retry_initial_backoff_ms >= settings.http_retry_max_backoff_ms)
+        throw Exception(
+            ErrorCodes::BAD_ARGUMENTS,
+            "Invalid setting for http backoff, "
+            "must be http_max_tries >= 1 (current is {}) and "
+            "0 < http_retry_initial_backoff_ms < settings.http_retry_max_backoff_ms (now 0 < {} < {})",
+            settings.http_max_tries,
+            settings.http_retry_initial_backoff_ms,
+            settings.http_retry_max_backoff_ms);
+
+    // Configure User-Agent if it not already set.
+    const std::string user_agent = "User-Agent";
+    auto iter = std::find_if(
+        http_header_entries.begin(),
+        http_header_entries.end(),
+        [&user_agent](const HTTPHeaderEntry & entry) { return entry.name == user_agent; });
+
+    if (iter == http_header_entries.end())
+    {
+        http_header_entries.emplace_back("User-Agent", fmt::format("ClickHouse/{}", VERSION_STRING));
+    }
+
+    if (!delay_initialization)
+    {
+        initialize();
+        if (exception)
+            std::rethrow_exception(exception);
+    }
+}
+
+template <typename UpdatableSessionPtr>
+void ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::callWithRedirects(Poco::Net::HTTPResponse & response, const String & method_, bool throw_on_all_errors, bool for_object_info)
+{
+    UpdatableSessionPtr current_session = nullptr;
+
+    /// we can fetch object info while the request is being processed
+    /// and we don't want to override any context used by it
+    if (for_object_info)
+        current_session = session->clone(uri);
+    else
+        current_session = session;
+
+    call(current_session, response, method_, throw_on_all_errors, for_object_info);
+    Poco::URI prev_uri = uri;
+
+    while (isRedirect(response.getStatus()))
+    {
+        Poco::URI uri_redirect = getUriAfterRedirect(prev_uri, response);
+        prev_uri = uri_redirect;
+        if (remote_host_filter)
+            remote_host_filter->checkURL(uri_redirect);
+
+        current_session->updateSession(uri_redirect);
+
+        /// we can fetch object info while the request is being processed
+        /// and we don't want to override any context used by it
+        auto result_istr = callImpl(current_session, uri_redirect, response, method, for_object_info);
+        if (!for_object_info)
+            istr = result_istr;
+    }
+}
+
+template <typename UpdatableSessionPtr>
+void ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::call(UpdatableSessionPtr & current_session, Poco::Net::HTTPResponse & response, const String & method_, bool throw_on_all_errors, bool for_object_info)
+{
+    try
+    {
+        /// we can fetch object info while the request is being processed
+        /// and we don't want to override any context used by it
+        auto result_istr = callImpl(current_session, saved_uri_redirect ? *saved_uri_redirect : uri, response, method_, for_object_info);
+        if (!for_object_info)
+            istr = result_istr;
+    }
+    catch (...)
+    {
+        /// we can fetch object info while the request is being processed
+        /// and we don't want to override any context used by it
+        if (for_object_info)
+            throw;
+
+        if (throw_on_all_errors)
+            throw;
+
+        auto http_status = response.getStatus();
+
+        if (http_status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND && http_skip_not_found_url)
+        {
+            initialization_error = InitializeError::SKIP_NOT_FOUND_URL;
+        }
+        else if (!isRetriableError(http_status))
+        {
+            initialization_error = InitializeError::NON_RETRYABLE_ERROR;
+            exception = std::current_exception();
+        }
+        else
+        {
+            throw;
+        }
+    }
+}
+
+template <typename UpdatableSessionPtr>
+void ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::initialize()
+{
+    Poco::Net::HTTPResponse response;
+
+    call(session, response, method);
+    if (initialization_error != InitializeError::NONE)
+        return;
+
+    while (isRedirect(response.getStatus()))
+    {
+        Poco::URI uri_redirect = getUriAfterRedirect(saved_uri_redirect.value_or(uri), response);
+        if (remote_host_filter)
+            remote_host_filter->checkURL(uri_redirect);
+
+        session->updateSession(uri_redirect);
+
+        istr = callImpl(session, uri_redirect, response, method);
+        saved_uri_redirect = uri_redirect;
+    }
+
+    if (response.hasContentLength())
+        LOG_DEBUG(log, "Received response with content length: {}", response.getContentLength());
+
+    if (withPartialContent(read_range) && response.getStatus() != Poco::Net::HTTPResponse::HTTPStatus::HTTP_PARTIAL_CONTENT)
+    {
+        /// Having `200 OK` instead of `206 Partial Content` is acceptable in case we retried with range.begin == 0.
+        if (getOffset() != 0)
+        {
+            if (!exception)
+            {
+                exception = std::make_exception_ptr(Exception(
+                    ErrorCodes::HTTP_RANGE_NOT_SATISFIABLE,
+                    "Cannot read with range: [{}, {}] (response status: {}, reason: {})",
+                    *read_range.begin,
+                    read_range.end ? toString(*read_range.end) : "-",
+                    toString(response.getStatus()), response.getReason()));
+            }
+
+            /// Retry 200OK
+            if (response.getStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_OK)
+                initialization_error = InitializeError::RETRYABLE_ERROR;
+            else
+                initialization_error = InitializeError::NON_RETRYABLE_ERROR;
+
+            return;
+        }
+        else if (read_range.end)
+        {
+            /// We could have range.begin == 0 and range.end != 0 in case of DiskWeb and failing to read with partial content
+            /// will affect only performance, so a warning is enough.
+            LOG_WARNING(log, "Unable to read with range header: [{}, {}]", getRangeBegin(), *read_range.end);
+        }
+    }
+
+    // Remember file size. It'll be used to report eof in next nextImpl() call.
+    if (!read_range.end && response.hasContentLength())
+        file_info = parseFileInfo(response, withPartialContent(read_range) ? getOffset() : 0);
+
+    try
+    {
+        impl = std::make_unique<ReadBufferFromIStream>(*istr, buffer_size);
+
+        if (use_external_buffer)
+        {
+            setupExternalBuffer();
+        }
+    }
+    catch (const Poco::Exception & e)
+    {
+        /// We use session data storage as storage for exception text
+        /// Depend on it we can deduce to reconnect session or reresolve session host
+        auto sess = session->getSession();
+        sess->attachSessionData(e.message());
+        throw;
+    }
+}
+
+template <typename UpdatableSessionPtr>
+bool ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::nextImpl()
+{
+    if (initialization_error == InitializeError::SKIP_NOT_FOUND_URL)
+        return false;
+    assert(initialization_error == InitializeError::NONE);
+
+    if (next_callback)
+        next_callback(count());
+
+    if ((read_range.end && getOffset() > read_range.end.value()) ||
+        (file_info && file_info->file_size && getOffset() >= file_info->file_size.value()))
+        return false;
+
+    if (impl)
+    {
+        if (use_external_buffer)
+        {
+            setupExternalBuffer();
+        }
+        else
+        {
+            /**
+            * impl was initialized before, pass position() to it to make
+            * sure there is no pending data which was not read.
+            */
+            if (!working_buffer.empty())
+                impl->position() = position();
+        }
+    }
+
+    bool result = false;
+    size_t milliseconds_to_wait = settings.http_retry_initial_backoff_ms;
+    bool last_attempt = false;
+
+    auto on_retriable_error = [&]()
+    {
+        retry_with_range_header = true;
+        impl.reset();
+        auto http_session = session->getSession();
+        http_session->reset();
+        if (!last_attempt)
+        {
+            sleepForMilliseconds(milliseconds_to_wait);
+            milliseconds_to_wait = std::min(milliseconds_to_wait * 2, settings.http_retry_max_backoff_ms);
+        }
+    };
+
+    for (size_t i = 0;; ++i)
+    {
+        if (last_attempt)
+            break;
+        last_attempt = i + 1 >= settings.http_max_tries;
+
+        exception = nullptr;
+        initialization_error = InitializeError::NONE;
+
+        try
+        {
+            if (!impl)
+            {
+                initialize();
+
+                if (initialization_error == InitializeError::NON_RETRYABLE_ERROR)
+                {
+                    assert(exception);
+                    break;
+                }
+                else if (initialization_error == InitializeError::SKIP_NOT_FOUND_URL)
+                {
+                    return false;
+                }
+                else if (initialization_error == InitializeError::RETRYABLE_ERROR)
+                {
+                    LOG_ERROR(
+                        log,
+                        "HTTP request to `{}` failed at try {}/{} with bytes read: {}/{}. "
+                        "(Current backoff wait is {}/{} ms)",
+                        uri.toString(), i + 1, settings.http_max_tries, getOffset(),
+                        read_range.end ? toString(*read_range.end) : "unknown",
+                        milliseconds_to_wait, settings.http_retry_max_backoff_ms);
+
+                    assert(exception);
+                    on_retriable_error();
+                    continue;
+                }
+
+                assert(!exception);
+
+                if (use_external_buffer)
+                {
+                    setupExternalBuffer();
+                }
+            }
+
+            result = impl->next();
+            exception = nullptr;
+            break;
+        }
+        catch (const Poco::Exception & e)
+        {
+            /// Too many open files - non-retryable.
+            if (e.code() == POCO_EMFILE)
+                throw;
+
+            /** Retry request unconditionally if nothing has been read yet.
+                 * Otherwise if it is GET method retry with range header.
+                 */
+            bool can_retry_request = !offset_from_begin_pos || method == Poco::Net::HTTPRequest::HTTP_GET;
+            if (!can_retry_request)
+                throw;
+
+            LOG_ERROR(
+                log,
+                "HTTP request to `{}` failed at try {}/{} with bytes read: {}/{}. "
+                "Error: {}. (Current backoff wait is {}/{} ms)",
+                uri.toString(),
+                i + 1,
+                settings.http_max_tries,
+                getOffset(),
+                read_range.end ? toString(*read_range.end) : "unknown",
+                e.displayText(),
+                milliseconds_to_wait,
+                settings.http_retry_max_backoff_ms);
+
+            on_retriable_error();
+            exception = std::current_exception();
+        }
+    }
+
+    if (exception)
+        std::rethrow_exception(exception);
+
+    if (!result)
+        return false;
+
+    internal_buffer = impl->buffer();
+    working_buffer = internal_buffer;
+    offset_from_begin_pos += working_buffer.size();
+    return true;
+}
+
+template <typename UpdatableSessionPtr>
+off_t ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::getPosition() { return getOffset() - available(); }
+
+template <typename UpdatableSessionPtr>
+off_t ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::seek(off_t offset_, int whence)
+{
+    if (whence != SEEK_SET)
+        throw Exception(ErrorCodes::CANNOT_SEEK_THROUGH_FILE, "Only SEEK_SET mode is allowed.");
+
+    if (offset_ < 0)
+        throw Exception(ErrorCodes::SEEK_POSITION_OUT_OF_BOUND, "Seek position is out of bounds. Offset: {}",
+            offset_);
+
+    off_t current_offset = getOffset();
+    if (!working_buffer.empty() && size_t(offset_) >= current_offset - working_buffer.size() && offset_ < current_offset)
+    {
+        pos = working_buffer.end() - (current_offset - offset_);
+        assert(pos >= working_buffer.begin());
+        assert(pos < working_buffer.end());
+
+        return getPosition();
+    }
+
+    if (impl)
+    {
+        auto position = getPosition();
+        if (offset_ > position)
+        {
+            size_t diff = offset_ - position;
+            if (diff < settings.remote_read_min_bytes_for_seek)
+            {
+                ignore(diff);
+                return offset_;
+            }
+        }
+
+        if (!atEndOfRequestedRangeGuess())
+            ProfileEvents::increment(ProfileEvents::ReadBufferSeekCancelConnection);
+        impl.reset();
+    }
+
+    resetWorkingBuffer();
+    read_range.begin = offset_;
+    offset_from_begin_pos = 0;
+
+    return offset_;
+}
+
+template <typename UpdatableSessionPtr>
+void ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::setReadUntilPosition(size_t until)
+{
+    until = std::max(until, 1ul);
+    if (read_range.end && *read_range.end + 1 == until)
+        return;
+    read_range.end = until - 1;
+    read_range.begin = getPosition();
+    resetWorkingBuffer();
+    if (impl)
+    {
+        if (!atEndOfRequestedRangeGuess())
+            ProfileEvents::increment(ProfileEvents::ReadBufferSeekCancelConnection);
+        impl.reset();
+    }
+}
+
+template <typename UpdatableSessionPtr>
+void ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::setReadUntilEnd()
+{
+    if (!read_range.end)
+        return;
+    read_range.end.reset();
+    read_range.begin = getPosition();
+    resetWorkingBuffer();
+    if (impl)
+    {
+        if (!atEndOfRequestedRangeGuess())
+            ProfileEvents::increment(ProfileEvents::ReadBufferSeekCancelConnection);
+        impl.reset();
+    }
+}
+
+template <typename UpdatableSessionPtr>
+bool ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::supportsRightBoundedReads() const { return true; }
+
+template <typename UpdatableSessionPtr>
+bool ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::atEndOfRequestedRangeGuess()
+{
+    if (!impl)
+        return true;
+    if (read_range.end)
+        return getPosition() > static_cast<off_t>(*read_range.end);
+    if (file_info && file_info->file_size)
+        return getPosition() >= static_cast<off_t>(*file_info->file_size);
+    return false;
+}
+
+template <typename UpdatableSessionPtr>
+std::string ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::getResponseCookie(const std::string & name, const std::string & def) const
+{
+    for (const auto & cookie : cookies)
+        if (cookie.getName() == name)
+            return cookie.getValue();
+    return def;
+}
+
+template <typename UpdatableSessionPtr>
+void ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::setNextCallback(NextCallback next_callback_)
+{
+    next_callback = next_callback_;
+    /// Some data maybe already read
+    next_callback(count());
+}
+
+template <typename UpdatableSessionPtr>
+const std::string & ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::getCompressionMethod() const { return content_encoding; }
+
+template <typename UpdatableSessionPtr>
+std::optional<time_t> ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::getLastModificationTime()
+{
+    return getFileInfo().last_modified;
+}
+
+template <typename UpdatableSessionPtr>
+HTTPFileInfo ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::getFileInfo()
+{
+    Poco::Net::HTTPResponse response;
+    try
+    {
+        getHeadResponse(response);
+    }
+    catch (HTTPException & e)
+    {
+        /// Maybe the web server doesn't support HEAD requests.
+        /// E.g. webhdfs reports status 400.
+        /// We should proceed in hopes that the actual GET request will succeed.
+        /// (Unless the error in transient. Don't want to nondeterministically sometimes
+        /// fall back to slow whole-file reads when HEAD is actually supported; that sounds
+        /// like a nightmare to debug.)
+        if (e.getHTTPStatus() >= 400 && e.getHTTPStatus() <= 499 &&
+            e.getHTTPStatus() != Poco::Net::HTTPResponse::HTTP_TOO_MANY_REQUESTS)
+            return HTTPFileInfo{};
+
+        throw;
+    }
+    return parseFileInfo(response, 0);
+}
+
+template <typename UpdatableSessionPtr>
+HTTPFileInfo ReadWriteBufferFromHTTPBase<UpdatableSessionPtr>::parseFileInfo(const Poco::Net::HTTPResponse & response, size_t requested_range_begin)
+{
+    HTTPFileInfo res;
+
+    if (response.hasContentLength())
+    {
+        res.file_size = response.getContentLength();
+
+        if (response.getStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_PARTIAL_CONTENT)
+        {
+            *res.file_size += requested_range_begin;
+            res.seekable = true;
+        }
+        else
+        {
+            res.seekable = response.has("Accept-Ranges") && response.get("Accept-Ranges") == "bytes";
+        }
+    }
+
+    if (response.has("Last-Modified"))
+    {
+        String date_str = response.get("Last-Modified");
+        struct tm info;
+        char * end = strptime(date_str.data(), "%a, %d %b %Y %H:%M:%S %Z", &info);
+        if (end == date_str.data() + date_str.size())
+            res.last_modified = timegm(&info);
+    }
+
+    return res;
+}
+
+}
+
+SessionFactory::SessionFactory(const ConnectionTimeouts & timeouts_)
+    : timeouts(timeouts_) {}
+
+SessionFactory::SessionType SessionFactory::buildNewSession(const Poco::URI & uri)
+{
+    return makeHTTPSession(uri, timeouts);
+}
+
+ReadWriteBufferFromHTTP::ReadWriteBufferFromHTTP(
+    Poco::URI uri_,
+    const std::string & method_,
+    OutStreamCallback out_stream_callback_,
+    const ConnectionTimeouts & timeouts,
+    const Poco::Net::HTTPBasicCredentials & credentials_,
+    const UInt64 max_redirects,
+    size_t buffer_size_,
+    const ReadSettings & settings_,
+    const HTTPHeaderEntries & http_header_entries_,
+    const RemoteHostFilter * remote_host_filter_,
+    bool delay_initialization_,
+    bool use_external_buffer_,
+    bool skip_not_found_url_,
+    std::optional<HTTPFileInfo> file_info_)
+    : Parent(
+        std::make_shared<SessionType>(uri_, max_redirects, std::make_shared<SessionFactory>(timeouts)),
+        uri_,
+        credentials_,
+        method_,
+        out_stream_callback_,
+        buffer_size_,
+        settings_,
+        http_header_entries_,
+        remote_host_filter_,
+        delay_initialization_,
+        use_external_buffer_,
+        skip_not_found_url_,
+        file_info_) {}
+
+RangedReadWriteBufferFromHTTPFactory::RangedReadWriteBufferFromHTTPFactory(
+    Poco::URI uri_,
+    std::string method_,
+    OutStreamCallback out_stream_callback_,
+    ConnectionTimeouts timeouts_,
+    const Poco::Net::HTTPBasicCredentials & credentials_,
+    UInt64 max_redirects_,
+    size_t buffer_size_,
+    ReadSettings settings_,
+    HTTPHeaderEntries http_header_entries_,
+    const RemoteHostFilter * remote_host_filter_,
+    bool delay_initialization_,
+    bool use_external_buffer_,
+    bool skip_not_found_url_)
+    : uri(uri_)
+    , method(std::move(method_))
+    , out_stream_callback(out_stream_callback_)
+    , timeouts(std::move(timeouts_))
+    , credentials(credentials_)
+    , max_redirects(max_redirects_)
+    , buffer_size(buffer_size_)
+    , settings(std::move(settings_))
+    , http_header_entries(std::move(http_header_entries_))
+    , remote_host_filter(remote_host_filter_)
+    , delay_initialization(delay_initialization_)
+    , use_external_buffer(use_external_buffer_)
+    , skip_not_found_url(skip_not_found_url_) {}
+
+std::unique_ptr<SeekableReadBuffer> RangedReadWriteBufferFromHTTPFactory::getReader()
+{
+    return std::make_unique<ReadWriteBufferFromHTTP>(
+        uri,
+        method,
+        out_stream_callback,
+        timeouts,
+        credentials,
+        max_redirects,
+        buffer_size,
+        settings,
+        http_header_entries,
+        remote_host_filter,
+        delay_initialization,
+        use_external_buffer,
+        skip_not_found_url,
+        file_info);
+}
+
+size_t RangedReadWriteBufferFromHTTPFactory::getFileSize()
+{
+    auto s = getFileInfo().file_size;
+    if (!s)
+        throw Exception(ErrorCodes::UNKNOWN_FILE_SIZE, "Cannot find out file size for: {}", uri.toString());
+    return *s;
+}
+
+bool RangedReadWriteBufferFromHTTPFactory::checkIfActuallySeekable()
+{
+    return getFileInfo().seekable;
+}
+
+HTTPFileInfo RangedReadWriteBufferFromHTTPFactory::getFileInfo()
+{
+    if (!file_info)
+        file_info = static_cast<ReadWriteBufferFromHTTP*>(getReader().get())->getFileInfo();
+    return *file_info;
+}
+
+String RangedReadWriteBufferFromHTTPFactory::getFileName() const { return uri.toString(); }
+
+
+PooledSessionFactory::PooledSessionFactory(
+    const ConnectionTimeouts & timeouts_, size_t per_endpoint_pool_size_)
+    : timeouts(timeouts_)
+    , per_endpoint_pool_size(per_endpoint_pool_size_) {}
+
+PooledSessionFactory::SessionType PooledSessionFactory::buildNewSession(const Poco::URI & uri)
+{
+    return makePooledHTTPSession(uri, timeouts, per_endpoint_pool_size);
+}
+
+
+PooledReadWriteBufferFromHTTP::PooledReadWriteBufferFromHTTP(
+    Poco::URI uri_,
+    const std::string & method_,
+    OutStreamCallback out_stream_callback_,
+    const ConnectionTimeouts & timeouts_,
+    const Poco::Net::HTTPBasicCredentials & credentials_,
+    size_t buffer_size_,
+    const UInt64 max_redirects,
+    size_t max_connections_per_endpoint)
+    : Parent(
+        std::make_shared<SessionType>(uri_, max_redirects, std::make_shared<PooledSessionFactory>(timeouts_, max_connections_per_endpoint)),
+        uri_,
+        credentials_,
+        method_,
+        out_stream_callback_,
+        buffer_size_) {}
+
+template class UpdatableSession<SessionFactory>;
+template class UpdatableSession<PooledSessionFactory>;
+template class detail::ReadWriteBufferFromHTTPBase<std::shared_ptr<UpdatableSession<SessionFactory>>>;
+template class detail::ReadWriteBufferFromHTTPBase<std::shared_ptr<UpdatableSession<PooledSessionFactory>>>;
+
+}
diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h
index 8629a0a3349..de1946ced48 100644
--- a/src/IO/ReadWriteBufferFromHTTP.h
+++ b/src/IO/ReadWriteBufferFromHTTP.h
@@ -27,25 +27,8 @@
 
 #include <filesystem>
 
-namespace ProfileEvents
-{
-extern const Event ReadBufferSeekCancelConnection;
-}
-
 namespace DB
 {
-/** Perform HTTP POST request and provide response to read.
-  */
-
-namespace ErrorCodes
-{
-    extern const int TOO_MANY_REDIRECTS;
-    extern const int HTTP_RANGE_NOT_SATISFIABLE;
-    extern const int BAD_ARGUMENTS;
-    extern const int CANNOT_SEEK_THROUGH_FILE;
-    extern const int SEEK_POSITION_OUT_OF_BOUND;
-    extern const int UNKNOWN_FILE_SIZE;
-}
 
 template <typename TSessionFactory>
 class UpdatableSession
@@ -53,29 +36,14 @@ class UpdatableSession
 public:
     using SessionPtr = typename TSessionFactory::SessionType;
 
-    explicit UpdatableSession(const Poco::URI & uri, UInt64 max_redirects_, std::shared_ptr<TSessionFactory> session_factory_)
-        : max_redirects{max_redirects_}
-        , initial_uri(uri)
-        , session_factory(std::move(session_factory_))
-    {
-        session = session_factory->buildNewSession(uri);
-    }
+    explicit UpdatableSession(const Poco::URI & uri, UInt64 max_redirects_, std::shared_ptr<TSessionFactory> session_factory_);
 
-    SessionPtr getSession() { return session; }
+    SessionPtr getSession();
 
-    void updateSession(const Poco::URI & uri)
-    {
-        ++redirects;
-        if (redirects <= max_redirects)
-            session = session_factory->buildNewSession(uri);
-        else
-            throw Exception(ErrorCodes::TOO_MANY_REDIRECTS, "Too many redirects while trying to access {}", initial_uri.toString());
-    }
+    void updateSession(const Poco::URI & uri);
+
+    std::shared_ptr<UpdatableSession<TSessionFactory>> clone(const Poco::URI & uri);
 
-    std::shared_ptr<UpdatableSession<TSessionFactory>> clone(const Poco::URI & uri)
-    {
-        return std::make_shared<UpdatableSession<TSessionFactory>>(uri, max_redirects, session_factory);
-    }
 private:
     SessionPtr session;
     UInt64 redirects{0};
@@ -85,29 +53,29 @@ private:
 };
 
 
+/// Information from HTTP response header.
+struct HTTPFileInfo
+{
+    // nullopt if the server doesn't report it.
+    std::optional<size_t> file_size;
+    std::optional<time_t> last_modified;
+    bool seekable = false;
+};
+
+
 namespace detail
 {
+    /// Byte range, including right bound [begin, end].
+    struct HTTPRange
+    {
+        std::optional<size_t> begin;
+        std::optional<size_t> end;
+    };
+
     template <typename UpdatableSessionPtr>
     class ReadWriteBufferFromHTTPBase : public SeekableReadBuffer, public WithFileName, public WithFileSize
     {
-    public:
-        /// Information from HTTP response header.
-        struct FileInfo
-        {
-            // nullopt if the server doesn't report it.
-            std::optional<size_t> file_size;
-            std::optional<time_t> last_modified;
-            bool seekable = false;
-        };
-
     protected:
-        /// HTTP range, including right bound [begin, end].
-        struct Range
-        {
-            std::optional<size_t> begin;
-            std::optional<size_t> end;
-        };
-
         Poco::URI uri;
         std::string method;
         std::string content_encoding;
@@ -126,8 +94,8 @@ namespace detail
         bool use_external_buffer;
 
         size_t offset_from_begin_pos = 0;
-        Range read_range;
-        std::optional<FileInfo> file_info;
+        HTTPRange read_range;
+        std::optional<HTTPFileInfo> file_info;
 
         /// Delayed exception in case retries with partial content are not satisfiable.
         std::exception_ptr exception;
@@ -140,106 +108,19 @@ namespace detail
         ReadSettings settings;
         Poco::Logger * log;
 
-        bool withPartialContent(const Range & range) const
-        {
-            /**
-             * Add range header if we have some passed range
-             * or if we want to retry GET request on purpose.
-             */
-            return range.begin || range.end || retry_with_range_header;
-        }
+        bool withPartialContent(const HTTPRange & range) const;
 
-        size_t getRangeBegin() const { return read_range.begin.value_or(0); }
+        size_t getRangeBegin() const;
 
-        size_t getOffset() const { return getRangeBegin() + offset_from_begin_pos; }
+        size_t getOffset() const;
 
-        template <bool for_object_info = false>
-        std::istream * callImpl(UpdatableSessionPtr & current_session, Poco::URI uri_, Poco::Net::HTTPResponse & response, const std::string & method_)
-        {
-            // With empty path poco will send "POST  HTTP/1.1" its bug.
-            if (uri_.getPath().empty())
-                uri_.setPath("/");
+        std::istream * callImpl(UpdatableSessionPtr & current_session, Poco::URI uri_, Poco::Net::HTTPResponse & response, const std::string & method_, bool for_object_info = false);
 
-            Poco::Net::HTTPRequest request(method_, uri_.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1);
-            request.setHost(uri_.getHost()); // use original, not resolved host name in header
+        size_t getFileSize() override;
 
-            if (out_stream_callback)
-                request.setChunkedTransferEncoding(true);
-            else if (method == Poco::Net::HTTPRequest::HTTP_POST)
-                request.setContentLength(0);    /// No callback - no body
+        bool checkIfActuallySeekable() override;
 
-            for (auto & [header, value] : http_header_entries)
-                request.set(header, value);
-
-            std::optional<Range> range;
-            if constexpr (!for_object_info)
-            {
-                if (withPartialContent(read_range))
-                    range = Range{getOffset(), read_range.end};
-            }
-
-            if (range)
-            {
-                String range_header_value;
-                if (range->end)
-                    range_header_value = fmt::format("bytes={}-{}", *range->begin, *range->end);
-                else
-                    range_header_value = fmt::format("bytes={}-", *range->begin);
-                LOG_TEST(log, "Adding header: Range: {}", range_header_value);
-                request.set("Range", range_header_value);
-            }
-
-            if (!credentials.getUsername().empty())
-                credentials.authenticate(request);
-
-            LOG_TRACE(log, "Sending request to {}", uri_.toString());
-
-            auto sess = current_session->getSession();
-            try
-            {
-                auto & stream_out = sess->sendRequest(request);
-
-                if (out_stream_callback)
-                    out_stream_callback(stream_out);
-
-                auto result_istr = receiveResponse(*sess, request, response, true);
-                response.getCookies(cookies);
-
-                /// we can fetch object info while the request is being processed
-                /// and we don't want to override any context used by it
-                if constexpr (!for_object_info)
-                    content_encoding = response.get("Content-Encoding", "");
-
-                return result_istr;
-            }
-            catch (const Poco::Exception & e)
-            {
-                /// We use session data storage as storage for exception text
-                /// Depend on it we can deduce to reconnect session or reresolve session host
-                sess->attachSessionData(e.message());
-                throw;
-            }
-        }
-
-        size_t getFileSize() override
-        {
-            if (!file_info)
-                file_info = getFileInfo();
-
-            if (file_info->file_size)
-                return *file_info->file_size;
-
-            throw Exception(ErrorCodes::UNKNOWN_FILE_SIZE, "Cannot find out file size for: {}", uri.toString());
-        }
-
-        bool checkIfActuallySeekable() override
-        {
-            if (!file_info)
-                file_info = getFileInfo();
-            return file_info->seekable;
-        }
-
-        String getFileName() const override { return uri.toString(); }
+        String getFileName() const override;
 
         enum class InitializeError
         {
@@ -254,38 +135,9 @@ namespace detail
         InitializeError initialization_error = InitializeError::NONE;
 
     private:
-        void getHeadResponse(Poco::Net::HTTPResponse & response)
-        {
-            for (size_t i = 0; i < settings.http_max_tries; ++i)
-            {
-                try
-                {
-                    callWithRedirects<true>(response, Poco::Net::HTTPRequest::HTTP_HEAD, true);
-                    break;
-                }
-                catch (const Poco::Exception & e)
-                {
-                    if (i == settings.http_max_tries - 1 || !isRetriableError(response.getStatus()))
-                        throw;
+        void getHeadResponse(Poco::Net::HTTPResponse & response);
 
-                    LOG_ERROR(log, "Failed to make HTTP_HEAD request to {}. Error: {}", uri.toString(), e.displayText());
-                }
-            }
-        }
-
-        void setupExternalBuffer()
-        {
-            /**
-            * use_external_buffer -- means we read into the buffer which
-            * was passed to us from somewhere else. We do not check whether
-            * previously returned buffer was read or not (no hasPendingData() check is needed),
-            * because this branch means we are prefetching data,
-            * each nextImpl() call we can fill a different buffer.
-            */
-            impl->set(internal_buffer.begin(), internal_buffer.size());
-            assert(working_buffer.begin() != nullptr);
-            assert(!internal_buffer.empty());
-        }
+        void setupExternalBuffer();
 
     public:
         using NextCallback = std::function<void(size_t)>;
@@ -304,563 +156,60 @@ namespace detail
             bool delay_initialization = false,
             bool use_external_buffer_ = false,
             bool http_skip_not_found_url_ = false,
-            std::optional<FileInfo> file_info_ = std::nullopt)
-            : SeekableReadBuffer(nullptr, 0)
-            , uri {uri_}
-            , method {!method_.empty() ? method_ : out_stream_callback_ ? Poco::Net::HTTPRequest::HTTP_POST : Poco::Net::HTTPRequest::HTTP_GET}
-            , session {session_}
-            , out_stream_callback {out_stream_callback_}
-            , credentials {credentials_}
-            , http_header_entries {std::move(http_header_entries_)}
-            , remote_host_filter {remote_host_filter_}
-            , buffer_size {buffer_size_}
-            , use_external_buffer {use_external_buffer_}
-            , file_info(file_info_)
-            , http_skip_not_found_url(http_skip_not_found_url_)
-            , settings {settings_}
-            , log(&Poco::Logger::get("ReadWriteBufferFromHTTP"))
-        {
-            if (settings.http_max_tries <= 0 || settings.http_retry_initial_backoff_ms <= 0
-                || settings.http_retry_initial_backoff_ms >= settings.http_retry_max_backoff_ms)
-                throw Exception(
-                    ErrorCodes::BAD_ARGUMENTS,
-                    "Invalid setting for http backoff, "
-                    "must be http_max_tries >= 1 (current is {}) and "
-                    "0 < http_retry_initial_backoff_ms < settings.http_retry_max_backoff_ms (now 0 < {} < {})",
-                    settings.http_max_tries,
-                    settings.http_retry_initial_backoff_ms,
-                    settings.http_retry_max_backoff_ms);
+            std::optional<HTTPFileInfo> file_info_ = std::nullopt);
 
-            // Configure User-Agent if it not already set.
-            const std::string user_agent = "User-Agent";
-            auto iter = std::find_if(
-                http_header_entries.begin(),
-                http_header_entries.end(),
-                [&user_agent](const HTTPHeaderEntry & entry) { return entry.name == user_agent; });
+        void callWithRedirects(Poco::Net::HTTPResponse & response, const String & method_, bool throw_on_all_errors = false, bool for_object_info = false);
 
-            if (iter == http_header_entries.end())
-            {
-                http_header_entries.emplace_back("User-Agent", fmt::format("ClickHouse/{}", VERSION_STRING));
-            }
-
-            if (!delay_initialization)
-            {
-                initialize();
-                if (exception)
-                    std::rethrow_exception(exception);
-            }
-        }
-
-        static bool isRetriableError(const Poco::Net::HTTPResponse::HTTPStatus http_status) noexcept
-        {
-            static constexpr std::array non_retriable_errors{
-                Poco::Net::HTTPResponse::HTTPStatus::HTTP_BAD_REQUEST,
-                Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED,
-                Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND,
-                Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN,
-                Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_IMPLEMENTED,
-                Poco::Net::HTTPResponse::HTTPStatus::HTTP_METHOD_NOT_ALLOWED};
-
-            return std::all_of(
-                non_retriable_errors.begin(), non_retriable_errors.end(), [&](const auto status) { return http_status != status; });
-        }
-
-        static Poco::URI getUriAfterRedirect(const Poco::URI & prev_uri, Poco::Net::HTTPResponse & response)
-        {
-            auto location = response.get("Location");
-            auto location_uri = Poco::URI(location);
-            if (!location_uri.isRelative())
-                return location_uri;
-            /// Location header contains relative path. So we need to concatenate it
-            /// with path from the original URI and normalize it.
-            auto path = std::filesystem::weakly_canonical(std::filesystem::path(prev_uri.getPath()) / location);
-            location_uri = prev_uri;
-            location_uri.setPath(path);
-            return location_uri;
-        }
-
-        template <bool for_object_info = false>
-        void callWithRedirects(Poco::Net::HTTPResponse & response, const String & method_, bool throw_on_all_errors = false)
-        {
-            UpdatableSessionPtr current_session = nullptr;
-
-            /// we can fetch object info while the request is being processed
-            /// and we don't want to override any context used by it
-            if constexpr (for_object_info)
-                current_session = session->clone(uri);
-            else
-                current_session = session;
-
-            call<for_object_info>(current_session, response, method_, throw_on_all_errors);
-            Poco::URI prev_uri = uri;
-
-            while (isRedirect(response.getStatus()))
-            {
-                Poco::URI uri_redirect = getUriAfterRedirect(prev_uri, response);
-                prev_uri = uri_redirect;
-                if (remote_host_filter)
-                    remote_host_filter->checkURL(uri_redirect);
-
-                current_session->updateSession(uri_redirect);
-
-                /// we can fetch object info while the request is being processed
-                /// and we don't want to override any context used by it
-                auto result_istr = callImpl<for_object_info>(current_session, uri_redirect, response, method);
-                if constexpr (!for_object_info)
-                    istr = result_istr;
-            }
-        }
-
-        template <bool for_object_info = false>
-        void call(UpdatableSessionPtr & current_session, Poco::Net::HTTPResponse & response, const String & method_, bool throw_on_all_errors = false)
-        {
-            try
-            {
-                /// we can fetch object info while the request is being processed
-                /// and we don't want to override any context used by it
-                auto result_istr = callImpl<for_object_info>(current_session, saved_uri_redirect ? *saved_uri_redirect : uri, response, method_);
-                if constexpr (!for_object_info)
-                    istr = result_istr;
-            }
-            catch (...)
-            {
-                /// we can fetch object info while the request is being processed
-                /// and we don't want to override any context used by it
-                if constexpr (for_object_info)
-                {
-                    throw;
-                }
-                else
-                {
-                    if (throw_on_all_errors)
-                        throw;
-
-                    auto http_status = response.getStatus();
-
-                    if (http_status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND && http_skip_not_found_url)
-                    {
-                        initialization_error = InitializeError::SKIP_NOT_FOUND_URL;
-                    }
-                    else if (!isRetriableError(http_status))
-                    {
-                        initialization_error = InitializeError::NON_RETRYABLE_ERROR;
-                        exception = std::current_exception();
-                    }
-                    else
-                    {
-                        throw;
-                    }
-                }
-            }
-        }
+        void call(UpdatableSessionPtr & current_session, Poco::Net::HTTPResponse & response, const String & method_, bool throw_on_all_errors = false, bool for_object_info = false);
 
         /**
          * Throws if error is retryable, otherwise sets initialization_error = NON_RETRYABLE_ERROR and
          * saves exception into `exception` variable. In case url is not found and skip_not_found_url == true,
          * sets initialization_error = SKIP_NOT_FOUND_URL, otherwise throws.
          */
-        void initialize()
-        {
-            Poco::Net::HTTPResponse response;
+        void initialize();
 
-            call(session, response, method);
-            if (initialization_error != InitializeError::NONE)
-                return;
+        bool nextImpl() override;
 
-            while (isRedirect(response.getStatus()))
-            {
-                Poco::URI uri_redirect = getUriAfterRedirect(saved_uri_redirect.value_or(uri), response);
-                if (remote_host_filter)
-                    remote_host_filter->checkURL(uri_redirect);
+        off_t getPosition() override;
 
-                session->updateSession(uri_redirect);
+        off_t seek(off_t offset_, int whence) override;
 
-                istr = callImpl(session, uri_redirect, response, method);
-                saved_uri_redirect = uri_redirect;
-            }
+        void setReadUntilPosition(size_t until) override;
 
-            if (response.hasContentLength())
-                LOG_DEBUG(log, "Received response with content length: {}", response.getContentLength());
+        void setReadUntilEnd() override;
 
-            if (withPartialContent(read_range) && response.getStatus() != Poco::Net::HTTPResponse::HTTPStatus::HTTP_PARTIAL_CONTENT)
-            {
-                /// Having `200 OK` instead of `206 Partial Content` is acceptable in case we retried with range.begin == 0.
-                if (getOffset() != 0)
-                {
-                    if (!exception)
-                    {
-                        exception = std::make_exception_ptr(Exception(
-                            ErrorCodes::HTTP_RANGE_NOT_SATISFIABLE,
-                            "Cannot read with range: [{}, {}] (response status: {}, reason: {})",
-                            *read_range.begin,
-                            read_range.end ? toString(*read_range.end) : "-",
-                            toString(response.getStatus()), response.getReason()));
-                    }
-
-                    /// Retry 200OK
-                    if (response.getStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_OK)
-                        initialization_error = InitializeError::RETRYABLE_ERROR;
-                    else
-                        initialization_error = InitializeError::NON_RETRYABLE_ERROR;
-
-                    return;
-                }
-                else if (read_range.end)
-                {
-                    /// We could have range.begin == 0 and range.end != 0 in case of DiskWeb and failing to read with partial content
-                    /// will affect only performance, so a warning is enough.
-                    LOG_WARNING(log, "Unable to read with range header: [{}, {}]", getRangeBegin(), *read_range.end);
-                }
-            }
-
-            // Remember file size. It'll be used to report eof in next nextImpl() call.
-            if (!read_range.end && response.hasContentLength())
-                file_info = parseFileInfo(response, withPartialContent(read_range) ? getOffset() : 0);
-
-            try
-            {
-                impl = std::make_unique<ReadBufferFromIStream>(*istr, buffer_size);
-
-                if (use_external_buffer)
-                {
-                    setupExternalBuffer();
-                }
-            }
-            catch (const Poco::Exception & e)
-            {
-                /// We use session data storage as storage for exception text
-                /// Depend on it we can deduce to reconnect session or reresolve session host
-                auto sess = session->getSession();
-                sess->attachSessionData(e.message());
-                throw;
-            }
-        }
-
-        bool nextImpl() override
-        {
-            if (initialization_error == InitializeError::SKIP_NOT_FOUND_URL)
-                return false;
-            assert(initialization_error == InitializeError::NONE);
-
-            if (next_callback)
-                next_callback(count());
-
-            if ((read_range.end && getOffset() > read_range.end.value()) ||
-                (file_info && file_info->file_size && getOffset() >= file_info->file_size.value()))
-                return false;
-
-            if (impl)
-            {
-                if (use_external_buffer)
-                {
-                    setupExternalBuffer();
-                }
-                else
-                {
-                    /**
-                    * impl was initialized before, pass position() to it to make
-                    * sure there is no pending data which was not read.
-                    */
-                    if (!working_buffer.empty())
-                        impl->position() = position();
-                }
-            }
-
-            bool result = false;
-            size_t milliseconds_to_wait = settings.http_retry_initial_backoff_ms;
-            bool last_attempt = false;
-
-            auto on_retriable_error = [&]()
-            {
-                retry_with_range_header = true;
-                impl.reset();
-                auto http_session = session->getSession();
-                http_session->reset();
-                if (!last_attempt)
-                {
-                    sleepForMilliseconds(milliseconds_to_wait);
-                    milliseconds_to_wait = std::min(milliseconds_to_wait * 2, settings.http_retry_max_backoff_ms);
-                }
-            };
-
-            for (size_t i = 0;; ++i)
-            {
-                if (last_attempt)
-                    break;
-                last_attempt = i + 1 >= settings.http_max_tries;
-
-                exception = nullptr;
-                initialization_error = InitializeError::NONE;
-
-                try
-                {
-                    if (!impl)
-                    {
-                        initialize();
-
-                        if (initialization_error == InitializeError::NON_RETRYABLE_ERROR)
-                        {
-                            assert(exception);
-                            break;
-                        }
-                        else if (initialization_error == InitializeError::SKIP_NOT_FOUND_URL)
-                        {
-                            return false;
-                        }
-                        else if (initialization_error == InitializeError::RETRYABLE_ERROR)
-                        {
-                            LOG_ERROR(
-                                log,
-                                "HTTP request to `{}` failed at try {}/{} with bytes read: {}/{}. "
-                                "(Current backoff wait is {}/{} ms)",
-                                uri.toString(), i + 1, settings.http_max_tries, getOffset(),
-                                read_range.end ? toString(*read_range.end) : "unknown",
-                                milliseconds_to_wait, settings.http_retry_max_backoff_ms);
-
-                            assert(exception);
-                            on_retriable_error();
-                            continue;
-                        }
-
-                        assert(!exception);
-
-                        if (use_external_buffer)
-                        {
-                            setupExternalBuffer();
-                        }
-                    }
-
-                    result = impl->next();
-                    exception = nullptr;
-                    break;
-                }
-                catch (const Poco::Exception & e)
-                {
-                    /// Too many open files - non-retryable.
-                    if (e.code() == POCO_EMFILE)
-                        throw;
-
-                    /** Retry request unconditionally if nothing has been read yet.
-                      * Otherwise if it is GET method retry with range header.
-                      */
-                    bool can_retry_request = !offset_from_begin_pos || method == Poco::Net::HTTPRequest::HTTP_GET;
-                    if (!can_retry_request)
-                        throw;
-
-                    LOG_ERROR(
-                        log,
-                        "HTTP request to `{}` failed at try {}/{} with bytes read: {}/{}. "
-                        "Error: {}. (Current backoff wait is {}/{} ms)",
-                        uri.toString(),
-                        i + 1,
-                        settings.http_max_tries,
-                        getOffset(),
-                        read_range.end ? toString(*read_range.end) : "unknown",
-                        e.displayText(),
-                        milliseconds_to_wait,
-                        settings.http_retry_max_backoff_ms);
-
-                    on_retriable_error();
-                    exception = std::current_exception();
-                }
-            }
-
-            if (exception)
-                std::rethrow_exception(exception);
-
-            if (!result)
-                return false;
-
-            internal_buffer = impl->buffer();
-            working_buffer = internal_buffer;
-            offset_from_begin_pos += working_buffer.size();
-            return true;
-        }
-
-        off_t getPosition() override { return getOffset() - available(); }
-
-        off_t seek(off_t offset_, int whence) override
-        {
-            if (whence != SEEK_SET)
-                throw Exception(ErrorCodes::CANNOT_SEEK_THROUGH_FILE, "Only SEEK_SET mode is allowed.");
-
-            if (offset_ < 0)
-                throw Exception(ErrorCodes::SEEK_POSITION_OUT_OF_BOUND, "Seek position is out of bounds. Offset: {}",
-                    offset_);
-
-            off_t current_offset = getOffset();
-            if (!working_buffer.empty() && size_t(offset_) >= current_offset - working_buffer.size() && offset_ < current_offset)
-            {
-                pos = working_buffer.end() - (current_offset - offset_);
-                assert(pos >= working_buffer.begin());
-                assert(pos < working_buffer.end());
-
-                return getPosition();
-            }
-
-            if (impl)
-            {
-                auto position = getPosition();
-                if (offset_ > position)
-                {
-                    size_t diff = offset_ - position;
-                    if (diff < settings.remote_read_min_bytes_for_seek)
-                    {
-                        ignore(diff);
-                        return offset_;
-                    }
-                }
-
-                if (!atEndOfRequestedRangeGuess())
-                    ProfileEvents::increment(ProfileEvents::ReadBufferSeekCancelConnection);
-                impl.reset();
-            }
-
-            resetWorkingBuffer();
-            read_range.begin = offset_;
-            offset_from_begin_pos = 0;
-
-            return offset_;
-        }
-
-        void setReadUntilPosition(size_t until) override
-        {
-            until = std::max(until, 1ul);
-            if (read_range.end && *read_range.end + 1 == until)
-                return;
-            read_range.end = until - 1;
-            read_range.begin = getPosition();
-            resetWorkingBuffer();
-            if (impl)
-            {
-                if (!atEndOfRequestedRangeGuess())
-                    ProfileEvents::increment(ProfileEvents::ReadBufferSeekCancelConnection);
-                impl.reset();
-            }
-        }
-
-        void setReadUntilEnd() override
-        {
-            if (!read_range.end)
-                return;
-            read_range.end.reset();
-            read_range.begin = getPosition();
-            resetWorkingBuffer();
-            if (impl)
-            {
-                if (!atEndOfRequestedRangeGuess())
-                    ProfileEvents::increment(ProfileEvents::ReadBufferSeekCancelConnection);
-                impl.reset();
-            }
-        }
-
-        bool supportsRightBoundedReads() const override { return true; }
+        bool supportsRightBoundedReads() const override;
 
         // If true, if we destroy impl now, no work was wasted. Just for metrics.
-        bool atEndOfRequestedRangeGuess()
-        {
-            if (!impl)
-                return true;
-            if (read_range.end)
-                return getPosition() > static_cast<off_t>(*read_range.end);
-            if (file_info && file_info->file_size)
-                return getPosition() >= static_cast<off_t>(*file_info->file_size);
-            return false;
-        }
+        bool atEndOfRequestedRangeGuess();
 
-        std::string getResponseCookie(const std::string & name, const std::string & def) const
-        {
-            for (const auto & cookie : cookies)
-                if (cookie.getName() == name)
-                    return cookie.getValue();
-            return def;
-        }
+        std::string getResponseCookie(const std::string & name, const std::string & def) const;
 
         /// Set function to call on each nextImpl, useful when you need to track
         /// progress.
         /// NOTE: parameter on each call is not incremental -- it's all bytes count
         /// passed through the buffer
-        void setNextCallback(NextCallback next_callback_)
-        {
-            next_callback = next_callback_;
-            /// Some data maybe already read
-            next_callback(count());
-        }
+        void setNextCallback(NextCallback next_callback_);
 
-        const std::string & getCompressionMethod() const { return content_encoding; }
+        const std::string & getCompressionMethod() const;
 
-        std::optional<time_t> getLastModificationTime()
-        {
-            return getFileInfo().last_modified;
-        }
+        std::optional<time_t> getLastModificationTime();
 
-        FileInfo getFileInfo()
-        {
-            Poco::Net::HTTPResponse response;
-            try
-            {
-                getHeadResponse(response);
-            }
-            catch (HTTPException & e)
-            {
-                /// Maybe the web server doesn't support HEAD requests.
-                /// E.g. webhdfs reports status 400.
-                /// We should proceed in hopes that the actual GET request will succeed.
-                /// (Unless the error in transient. Don't want to nondeterministically sometimes
-                /// fall back to slow whole-file reads when HEAD is actually supported; that sounds
-                /// like a nightmare to debug.)
-                if (e.getHTTPStatus() >= 400 && e.getHTTPStatus() <= 499 &&
-                    e.getHTTPStatus() != Poco::Net::HTTPResponse::HTTP_TOO_MANY_REQUESTS)
-                    return FileInfo{};
+        HTTPFileInfo getFileInfo();
 
-                throw;
-            }
-            return parseFileInfo(response, 0);
-        }
-
-        FileInfo parseFileInfo(const Poco::Net::HTTPResponse & response, size_t requested_range_begin)
-        {
-            FileInfo res;
-
-            if (response.hasContentLength())
-            {
-                res.file_size = response.getContentLength();
-
-                if (response.getStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_PARTIAL_CONTENT)
-                {
-                    *res.file_size += requested_range_begin;
-                    res.seekable = true;
-                }
-                else
-                {
-                    res.seekable = response.has("Accept-Ranges") && response.get("Accept-Ranges") == "bytes";
-                }
-            }
-
-            if (response.has("Last-Modified"))
-            {
-                String date_str = response.get("Last-Modified");
-                struct tm info;
-                char * end = strptime(date_str.data(), "%a, %d %b %Y %H:%M:%S %Z", &info);
-                if (end == date_str.data() + date_str.size())
-                    res.last_modified = timegm(&info);
-            }
-
-            return res;
-        }
+        HTTPFileInfo parseFileInfo(const Poco::Net::HTTPResponse & response, size_t requested_range_begin);
     };
 }
 
 class SessionFactory
 {
 public:
-    explicit SessionFactory(const ConnectionTimeouts & timeouts_)
-        : timeouts(timeouts_)
-    {}
+    explicit SessionFactory(const ConnectionTimeouts & timeouts_);
 
     using SessionType = HTTPSessionPtr;
 
-    SessionType buildNewSession(const Poco::URI & uri) { return makeHTTPSession(uri, timeouts); }
+    SessionType buildNewSession(const Poco::URI & uri);
 private:
     ConnectionTimeouts timeouts;
 };
@@ -885,23 +234,7 @@ public:
         bool delay_initialization_ = true,
         bool use_external_buffer_ = false,
         bool skip_not_found_url_ = false,
-        std::optional<FileInfo> file_info_ = std::nullopt)
-        : Parent(
-            std::make_shared<SessionType>(uri_, max_redirects, std::make_shared<SessionFactory>(timeouts)),
-            uri_,
-            credentials_,
-            method_,
-            out_stream_callback_,
-            buffer_size_,
-            settings_,
-            http_header_entries_,
-            remote_host_filter_,
-            delay_initialization_,
-            use_external_buffer_,
-            skip_not_found_url_,
-            file_info_)
-    {
-    }
+        std::optional<HTTPFileInfo> file_info_ = std::nullopt);
 };
 
 class RangedReadWriteBufferFromHTTPFactory : public SeekableReadBufferFactory, public WithFileName
@@ -922,63 +255,17 @@ public:
         const RemoteHostFilter * remote_host_filter_ = nullptr,
         bool delay_initialization_ = true,
         bool use_external_buffer_ = false,
-        bool skip_not_found_url_ = false)
-        : uri(uri_)
-        , method(std::move(method_))
-        , out_stream_callback(out_stream_callback_)
-        , timeouts(std::move(timeouts_))
-        , credentials(credentials_)
-        , max_redirects(max_redirects_)
-        , buffer_size(buffer_size_)
-        , settings(std::move(settings_))
-        , http_header_entries(std::move(http_header_entries_))
-        , remote_host_filter(remote_host_filter_)
-        , delay_initialization(delay_initialization_)
-        , use_external_buffer(use_external_buffer_)
-        , skip_not_found_url(skip_not_found_url_)
-    {
-    }
+        bool skip_not_found_url_ = false);
 
-    std::unique_ptr<SeekableReadBuffer> getReader() override
-    {
-        return std::make_unique<ReadWriteBufferFromHTTP>(
-            uri,
-            method,
-            out_stream_callback,
-            timeouts,
-            credentials,
-            max_redirects,
-            buffer_size,
-            settings,
-            http_header_entries,
-            remote_host_filter,
-            delay_initialization,
-            use_external_buffer,
-            skip_not_found_url,
-            file_info);
-    }
+    std::unique_ptr<SeekableReadBuffer> getReader() override;
 
-    size_t getFileSize() override
-    {
-        auto s = getFileInfo().file_size;
-        if (!s)
-            throw Exception(ErrorCodes::UNKNOWN_FILE_SIZE, "Cannot find out file size for: {}", uri.toString());
-        return *s;
-    }
+    size_t getFileSize() override;
 
-    bool checkIfActuallySeekable() override
-    {
-        return getFileInfo().seekable;
-    }
+    bool checkIfActuallySeekable() override;
 
-    ReadWriteBufferFromHTTP::FileInfo getFileInfo()
-    {
-        if (!file_info)
-            file_info = static_cast<ReadWriteBufferFromHTTP*>(getReader().get())->getFileInfo();
-        return *file_info;
-    }
+    HTTPFileInfo getFileInfo();
 
-    String getFileName() const override { return uri.toString(); }
+    String getFileName() const override;
 
 private:
     Poco::URI uri;
@@ -991,7 +278,7 @@ private:
     ReadSettings settings;
     HTTPHeaderEntries http_header_entries;
     const RemoteHostFilter * remote_host_filter;
-    std::optional<ReadWriteBufferFromHTTP::FileInfo> file_info;
+    std::optional<HTTPFileInfo> file_info;
     bool delay_initialization;
     bool use_external_buffer;
     bool skip_not_found_url;
@@ -1001,14 +288,11 @@ class PooledSessionFactory
 {
 public:
     explicit PooledSessionFactory(
-        const ConnectionTimeouts & timeouts_, size_t per_endpoint_pool_size_)
-        : timeouts(timeouts_)
-        , per_endpoint_pool_size(per_endpoint_pool_size_)
-    {}
+        const ConnectionTimeouts & timeouts_, size_t per_endpoint_pool_size_);
 
     using SessionType = PooledHTTPSessionPtr;
 
-    SessionType buildNewSession(const Poco::URI & uri) { return makePooledHTTPSession(uri, timeouts, per_endpoint_pool_size); }
+    SessionType buildNewSession(const Poco::URI & uri);
 private:
     ConnectionTimeouts timeouts;
     size_t per_endpoint_pool_size;
@@ -1028,16 +312,12 @@ public:
         const Poco::Net::HTTPBasicCredentials & credentials_ = {},
         size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE,
         const UInt64 max_redirects = 0,
-        size_t max_connections_per_endpoint = DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT)
-        : Parent(
-            std::make_shared<SessionType>(uri_, max_redirects, std::make_shared<PooledSessionFactory>(timeouts_, max_connections_per_endpoint)),
-            uri_,
-            credentials_,
-            method_,
-            out_stream_callback_,
-            buffer_size_)
-    {
-    }
+        size_t max_connections_per_endpoint = DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT);
 };
 
+extern template class UpdatableSession<SessionFactory>;
+extern template class UpdatableSession<PooledSessionFactory>;
+extern template class detail::ReadWriteBufferFromHTTPBase<std::shared_ptr<UpdatableSession<SessionFactory>>>;
+extern template class detail::ReadWriteBufferFromHTTPBase<std::shared_ptr<UpdatableSession<PooledSessionFactory>>>;
+
 }

From fc4288d3ac3d85e5a09cea41596379a4b7dfc35b Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 7 May 2023 06:54:21 +0000
Subject: [PATCH 257/689] Fix FreeBSD build

---
 utils/keeper-data-dumper/CMakeLists.txt | 2 +-
 utils/memcpy-bench/CMakeLists.txt       | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/utils/keeper-data-dumper/CMakeLists.txt b/utils/keeper-data-dumper/CMakeLists.txt
index 8c260c2122e..136d6f2268c 100644
--- a/utils/keeper-data-dumper/CMakeLists.txt
+++ b/utils/keeper-data-dumper/CMakeLists.txt
@@ -1,5 +1,5 @@
 if (NOT TARGET ch_contrib::nuraft)
-    message (${RECONFIGURE_MESSAGE_LEVEL} "Not building keeper-data-dumper due to nuraft is disabled")
+    message (WARNING "Not building keeper-data-dumper due to nuraft is disabled")
     return ()
 endif ()
 
diff --git a/utils/memcpy-bench/CMakeLists.txt b/utils/memcpy-bench/CMakeLists.txt
index f2cc4bdc8c5..c0b0b8a589d 100644
--- a/utils/memcpy-bench/CMakeLists.txt
+++ b/utils/memcpy-bench/CMakeLists.txt
@@ -1,5 +1,6 @@
 # memcpy_jart.S contains position dependent code
 if (CMAKE_POSITION_INDEPENDENT_CODE OR NOT OS_LINUX OR NOT ARCH_AMD64)
+    message (WARNING "Not building memcpy-bench due to PIC enabled or not Linux or not x86")
     return ()
 endif ()
 

From a2ed457219871cc5d11205c2deaebe486bb3a294 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Sun, 7 May 2023 14:02:24 +0200
Subject: [PATCH 258/689] Better concurrent parts removal with zero copy

---
 src/Storages/MergeTree/MergeTreeData.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 4470524d306..9da0e8a2751 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -2451,12 +2451,14 @@ void MergeTreeData::clearPartsFromFilesystemImpl(const DataPartsVector & parts_t
     std::mutex part_names_mutex;
     ThreadPool pool(CurrentMetrics::MergeTreePartsCleanerThreads, CurrentMetrics::MergeTreePartsCleanerThreadsActive, num_threads);
 
+    /// This flag disallow straightforward concurrent parts removal. It's required only in case
+    /// when we have parts on zero-copy disk + at least some of them were mutated.
     bool has_zero_copy_parts = false;
     if (settings->allow_remote_fs_zero_copy_replication && dynamic_cast<StorageReplicatedMergeTree *>(this) != nullptr)
     {
         has_zero_copy_parts = std::any_of(
             parts_to_remove.begin(), parts_to_remove.end(),
-            [] (const auto & data_part) { return data_part->isStoredOnRemoteDiskWithZeroCopySupport(); }
+            [] (const auto & data_part) { return data_part->isStoredOnRemoteDiskWithZeroCopySupport() && data_part->info.getMutationVersion() > 0; }
         );
     }
 

From 31b0f19a2a9efe10f81e34507abbd44159b42872 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 7 May 2023 12:12:35 +0000
Subject: [PATCH 259/689] CMake: Don't build utils unless explicitly requested

---
 utils/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt
index 5519b9bb983..1eeadde8ff1 100644
--- a/utils/CMakeLists.txt
+++ b/utils/CMakeLists.txt
@@ -14,7 +14,7 @@ add_subdirectory (config-processor)
 add_subdirectory (report)
 
 # Not used in package
-if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
+if (ENABLE_UTILS)
     add_subdirectory (compressor)
     add_subdirectory (corrector_utf8)
     add_subdirectory (zookeeper-cli)

From 33e691ad511aeff2ce35600d04c3c3a8a7081bb9 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Sun, 7 May 2023 14:26:21 +0200
Subject: [PATCH 260/689] Better name

---
 src/Storages/MergeTree/MergeTreeData.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 9da0e8a2751..d60307cc6c5 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -2453,17 +2453,17 @@ void MergeTreeData::clearPartsFromFilesystemImpl(const DataPartsVector & parts_t
 
     /// This flag disallow straightforward concurrent parts removal. It's required only in case
     /// when we have parts on zero-copy disk + at least some of them were mutated.
-    bool has_zero_copy_parts = false;
+    bool remove_parts_in_order = false;
     if (settings->allow_remote_fs_zero_copy_replication && dynamic_cast<StorageReplicatedMergeTree *>(this) != nullptr)
     {
-        has_zero_copy_parts = std::any_of(
+        remove_parts_in_order = std::any_of(
             parts_to_remove.begin(), parts_to_remove.end(),
             [] (const auto & data_part) { return data_part->isStoredOnRemoteDiskWithZeroCopySupport() && data_part->info.getMutationVersion() > 0; }
         );
     }
 
 
-    if (!has_zero_copy_parts)
+    if (!remove_parts_in_order)
     {
         /// NOTE: Under heavy system load you may get "Cannot schedule a task" from ThreadPool.
         LOG_DEBUG(

From 6473ec54eef61543078a30ce253a9f53a4abb4f8 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Sun, 7 May 2023 15:04:45 +0200
Subject: [PATCH 261/689] Fix

---
 src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
index 73a58611a17..7847cac6058 100644
--- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
+++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
@@ -936,6 +936,7 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep()
 
     if (result)
     {
+        bool download_current_segment_succeeded = false;
         if (download_current_segment)
         {
             chassert(file_offset_of_buffer_end + size - 1 <= file_segment.range().right);
@@ -954,6 +955,7 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep()
                         || file_segment.getCurrentWriteOffset(false) == implementation_buffer->getFileOffsetOfBufferEnd());
 
                     LOG_TEST(log, "Successfully written {} bytes", size);
+                    download_current_segment_succeeded = true;
 
                     // The implementation_buffer is valid and positioned correctly (at file_segment->getCurrentWriteOffset()).
                     // Later reads for this file segment can reuse it.
@@ -962,14 +964,15 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep()
                     implementation_buffer_can_be_reused = true;
                 }
                 else
-                {
-                    chassert(file_segment.state() == FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION);
                     LOG_TRACE(log, "Bypassing cache because writeCache method failed");
-                }
             }
             else
-            {
                 LOG_TRACE(log, "No space left in cache to reserve {} bytes, will continue without cache download", size);
+
+            if (!success)
+            {
+                read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE;
+                chassert(file_segment.state() == FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION);
             }
         }
 
@@ -990,6 +993,8 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep()
 
         file_offset_of_buffer_end += size;
 
+        if (download_current_segment && download_current_segment_succeeded)
+            chassert(file_segment.getCurrentWriteOffset(false) >= file_offset_of_buffer_end);
         chassert(file_offset_of_buffer_end <= read_until_position);
     }
 

From aa09b6154bed69b6ab479ae427954634378806df Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 7 May 2023 13:06:35 +0000
Subject: [PATCH 262/689] Various cleanups

---
 src/Functions/makeDate.cpp | 156 ++++++++++++++++++++-----------------
 1 file changed, 85 insertions(+), 71 deletions(-)

diff --git a/src/Functions/makeDate.cpp b/src/Functions/makeDate.cpp
index ae12659c8de..a3a8bf4ed21 100644
--- a/src/Functions/makeDate.cpp
+++ b/src/Functions/makeDate.cpp
@@ -1,5 +1,6 @@
 #include <Functions/IFunction.h>
 #include <Functions/FunctionFactory.h>
+#include <Functions/FunctionHelpers.h>
 #include <DataTypes/DataTypeDate.h>
 #include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDateTime.h>
@@ -48,6 +49,19 @@ public:
     size_t getNumberOfArguments() const override { return 0; }
 
 protected:
+    template <class ArgumentNames>
+    Columns convertMandatoryArguments(const ColumnsWithTypeAndName & arguments, const ArgumentNames & argument_names) const
+    {
+        Columns converted_arguments;
+        const DataTypePtr converted_argument_type = std::make_shared<DataTypeFloat32>();
+        for (size_t i = 0; i < argument_names.size(); ++i)
+        {
+            ColumnPtr argument_column = castColumn(arguments[i], converted_argument_type);
+            argument_column = argument_column->convertToFullColumnIfConst();
+            converted_arguments.push_back(argument_column);
+        }
+        return converted_arguments;
+    }
 };
 
 /// Common implementation for makeDate, makeDate32
@@ -55,8 +69,8 @@ template <typename Traits>
 class FunctionMakeDate : public FunctionWithNumericParamsBase
 {
 private:
-    static constexpr std::array argument_names_month_day = {"year", "month", "day"};
-    static constexpr std::array argument_names_dayofyear = {"year", "dayofyear"};
+    static constexpr std::array mandatory_argument_names_year_month_day = {"year", "month", "day"};
+    static constexpr std::array mandatory_argument_names_year_dayofyear = {"year", "dayofyear"};
 
 public:
     static constexpr auto name = Traits::name;
@@ -67,19 +81,24 @@ public:
 
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
-        if (arguments.size() != argument_names_month_day.size() && arguments.size() != argument_names_dayofyear.size())
-            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
-                "Function {} requires {} or {} arguments, but {} given",
-                getName(), argument_names_month_day.size(), argument_names_dayofyear.size(), arguments.size());
+        const bool isYearMonthDayVariant = (arguments.size() == 3);
 
-        for (size_t i = 0; i < arguments.size(); ++i)
+        if (isYearMonthDayVariant)
         {
-            DataTypePtr argument_type = arguments[i].type;
-            if (!isNumber(argument_type))
-                    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                        "Argument '{}' for function {} must be a number",
-                        (arguments.size() == argument_names_month_day.size()) ? argument_names_month_day[i] : argument_names_dayofyear[i], getName());
-
+            FunctionArgumentDescriptors args{
+                {mandatory_argument_names_year_month_day[0], &isNumber<IDataType>, nullptr, "Number"},
+                {mandatory_argument_names_year_month_day[1], &isNumber<IDataType>, nullptr, "Number"},
+                {mandatory_argument_names_year_month_day[2], &isNumber<IDataType>, nullptr, "Number"}
+            };
+            validateFunctionArgumentTypes(*this, arguments, args);
+        }
+        else
+        {
+            FunctionArgumentDescriptors args{
+                {mandatory_argument_names_year_dayofyear[0], &isNumber<IDataType>, nullptr, "Number"},
+                {mandatory_argument_names_year_dayofyear[1], &isNumber<IDataType>, nullptr, "Number"}
+            };
+            validateFunctionArgumentTypes(*this, arguments, args);
         }
 
         return std::make_shared<typename Traits::ReturnDataType>();
@@ -87,15 +106,13 @@ public:
 
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
+        const bool isYearMonthDayVariant = (arguments.size() == 3);
+
         Columns converted_arguments;
-        const DataTypePtr converted_argument_type = std::make_shared<DataTypeFloat32>();
-        converted_arguments.clear();
-        for (const auto & argument : arguments)
-        {
-            ColumnPtr argument_column = castColumn(argument, converted_argument_type);
-            argument_column = argument_column->convertToFullColumnIfConst();
-            converted_arguments.push_back(argument_column);
-        }
+        if (isYearMonthDayVariant)
+            converted_arguments = convertMandatoryArguments(arguments, mandatory_argument_names_year_month_day);
+        else
+            converted_arguments = convertMandatoryArguments(arguments, mandatory_argument_names_year_dayofyear);
 
         auto res_column = Traits::ReturnDataType::ColumnType::create(input_rows_count);
         auto & result_data = res_column->getData();
@@ -103,7 +120,7 @@ public:
         const auto & date_lut = DateLUT::instance();
         const Int32 max_days_since_epoch = date_lut.makeDayNum(Traits::MAX_DATE[0], Traits::MAX_DATE[1], Traits::MAX_DATE[2]);
 
-        if (converted_arguments.size() == argument_names_month_day.size())
+        if (isYearMonthDayVariant)
         {
             const auto & year_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[0]).getData();
             const auto & month_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[1]).getData();
@@ -132,7 +149,6 @@ public:
         }
         else
         {
-            /// case argument_names_dayofyear:
             const auto & year_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[0]).getData();
             const auto & dayofyear_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[1]).getData();
 
@@ -185,35 +201,7 @@ struct MakeDate32Traits
 class FunctionMakeDateTimeBase : public FunctionWithNumericParamsBase
 {
 protected:
-    static constexpr std::array argument_names = {"year", "month", "day", "hour", "minute", "second"};
-
-    void checkRequiredArguments(const ColumnsWithTypeAndName & arguments, size_t optional_argument_count) const
-    {
-        if (arguments.size() < argument_names.size() || arguments.size() > argument_names.size() + optional_argument_count)
-            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
-                "Function {} requires {} to {} arguments, but {} given",
-                getName(), argument_names.size(), argument_names.size() + optional_argument_count, arguments.size());
-
-        for (size_t i = 0; i < argument_names.size(); ++i)
-        {
-            DataTypePtr argument_type = arguments[i].type;
-            if (!isNumber(argument_type))
-                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "Argument '{}' for function {} must be a number", argument_names[i], getName());
-        }
-    }
-
-    void convertRequiredArguments(const ColumnsWithTypeAndName & arguments, Columns & converted_arguments) const
-    {
-        const DataTypePtr converted_argument_type = std::make_shared<DataTypeFloat32>();
-        converted_arguments.clear();
-        for (size_t i = 0; i < argument_names.size(); ++i)
-        {
-            ColumnPtr argument_column = castColumn(arguments[i], converted_argument_type);
-            argument_column = argument_column->convertToFullColumnIfConst();
-            converted_arguments.push_back(argument_column);
-        }
-    }
+    static constexpr std::array mandatory_argument_names = {"year", "month", "day", "hour", "minute", "second"};
 
     template <typename T>
     static Int64 dateTime(T year, T month, T day_of_month, T hour, T minute, T second, const DateLUTImpl & lut)
@@ -271,11 +259,24 @@ public:
 
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
-        checkRequiredArguments(arguments, optional_argument_names.size());
+        FunctionArgumentDescriptors mandatory_args{
+            {mandatory_argument_names[0], &isNumber<IDataType>, nullptr, "Number"},
+            {mandatory_argument_names[1], &isNumber<IDataType>, nullptr, "Number"},
+            {mandatory_argument_names[2], &isNumber<IDataType>, nullptr, "Number"},
+            {mandatory_argument_names[3], &isNumber<IDataType>, nullptr, "Number"},
+            {mandatory_argument_names[4], &isNumber<IDataType>, nullptr, "Number"},
+            {mandatory_argument_names[5], &isNumber<IDataType>, nullptr, "Number"}
+        };
+
+        FunctionArgumentDescriptors optional_args{
+            {optional_argument_names[0], &isString<IDataType>, nullptr, "String"}
+        };
+
+        validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
 
         /// Optional timezone argument
         std::string timezone;
-        if (arguments.size() == argument_names.size() + 1)
+        if (arguments.size() == mandatory_argument_names.size() + 1)
             timezone = extractTimezone(arguments.back());
 
         return std::make_shared<DataTypeDateTime>(timezone);
@@ -285,11 +286,10 @@ public:
     {
         /// Optional timezone argument
         std::string timezone;
-        if (arguments.size() == argument_names.size() + 1)
+        if (arguments.size() == mandatory_argument_names.size() + 1)
             timezone = extractTimezone(arguments.back());
 
-        Columns converted_arguments;
-        convertRequiredArguments(arguments, converted_arguments);
+        Columns converted_arguments = convertMandatoryArguments(arguments, mandatory_argument_names);
 
         auto res_column = ColumnDateTime::create(input_rows_count);
         auto & result_data = res_column->getData();
@@ -325,7 +325,7 @@ public:
     }
 };
 
-/// makeDateTime64(year, month, day, hour, minute, second, [fraction], [precision], [timezone])
+/// makeDateTime64(year, month, day, hour, minute, second[, fraction[, precision[, timezone]]])
 class FunctionMakeDateTime64 : public FunctionMakeDateTimeBase
 {
 private:
@@ -341,11 +341,26 @@ public:
 
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
-        checkRequiredArguments(arguments, optional_argument_names.size());
+        FunctionArgumentDescriptors mandatory_args{
+            {mandatory_argument_names[0], &isNumber<IDataType>, nullptr, "Number"},
+            {mandatory_argument_names[1], &isNumber<IDataType>, nullptr, "Number"},
+            {mandatory_argument_names[2], &isNumber<IDataType>, nullptr, "Number"},
+            {mandatory_argument_names[3], &isNumber<IDataType>, nullptr, "Number"},
+            {mandatory_argument_names[4], &isNumber<IDataType>, nullptr, "Number"},
+            {mandatory_argument_names[5], &isNumber<IDataType>, nullptr, "Number"}
+        };
 
-        if (arguments.size() >= argument_names.size() + 1)
+        FunctionArgumentDescriptors optional_args{
+            {optional_argument_names[0], &isNumber<IDataType>, nullptr, "Number"},
+            {optional_argument_names[1], &isNumber<IDataType>, nullptr, "Number"},
+            {optional_argument_names[2], &isString<IDataType>, nullptr, "String"}
+        };
+
+        validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
+
+        if (arguments.size() >= mandatory_argument_names.size() + 1)
         {
-            const auto& fraction_argument = arguments[argument_names.size()];
+            const auto& fraction_argument = arguments[mandatory_argument_names.size()];
             if (!isNumber(fraction_argument.type))
                 throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
                     "Argument 'fraction' for function {} must be a number", getName());
@@ -353,12 +368,12 @@ public:
 
         /// Optional precision argument
         Int64 precision = DEFAULT_PRECISION;
-        if (arguments.size() >= argument_names.size() + 2)
-            precision = extractPrecision(arguments[argument_names.size() + 1]);
+        if (arguments.size() >= mandatory_argument_names.size() + 2)
+            precision = extractPrecision(arguments[mandatory_argument_names.size() + 1]);
 
         /// Optional timezone argument
         std::string timezone;
-        if (arguments.size() == argument_names.size() + 3)
+        if (arguments.size() == mandatory_argument_names.size() + 3)
             timezone = extractTimezone(arguments.back());
 
         return std::make_shared<DataTypeDateTime64>(precision, timezone);
@@ -368,22 +383,21 @@ public:
     {
         /// Optional precision argument
         Int64 precision = DEFAULT_PRECISION;
-        if (arguments.size() >= argument_names.size() + 2)
-            precision = extractPrecision(arguments[argument_names.size() + 1]);
+        if (arguments.size() >= mandatory_argument_names.size() + 2)
+            precision = extractPrecision(arguments[mandatory_argument_names.size() + 1]);
 
         /// Optional timezone argument
         std::string timezone;
-        if (arguments.size() == argument_names.size() + 3)
+        if (arguments.size() == mandatory_argument_names.size() + 3)
             timezone = extractTimezone(arguments.back());
 
-        Columns converted_arguments;
-        convertRequiredArguments(arguments, converted_arguments);
+        Columns converted_arguments = convertMandatoryArguments(arguments, mandatory_argument_names);
 
         /// Optional fraction argument
         const ColumnVector<Float64>::Container * fraction_data = nullptr;
-        if (arguments.size() >= argument_names.size() + 1)
+        if (arguments.size() >= mandatory_argument_names.size() + 1)
         {
-            ColumnPtr fraction_column = castColumn(arguments[argument_names.size()], std::make_shared<DataTypeFloat64>());
+            ColumnPtr fraction_column = castColumn(arguments[mandatory_argument_names.size()], std::make_shared<DataTypeFloat64>());
             fraction_column = fraction_column->convertToFullColumnIfConst();
             converted_arguments.push_back(fraction_column);
             fraction_data = &typeid_cast<const ColumnFloat64 &>(*converted_arguments[6]).getData();

From b9957959714ac15f87832c43447584d88602f740 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 7 May 2023 13:27:57 +0000
Subject: [PATCH 263/689] Fix style

---
 src/Functions/makeDate.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Functions/makeDate.cpp b/src/Functions/makeDate.cpp
index a3a8bf4ed21..1e4f3604c94 100644
--- a/src/Functions/makeDate.cpp
+++ b/src/Functions/makeDate.cpp
@@ -21,7 +21,6 @@ namespace DB
 {
 namespace ErrorCodes
 {
-    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int ARGUMENT_OUT_OF_BOUND;
 }

From 27e1357ee67ac12ffbb502864a87661837d49809 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 17:42:18 +0300
Subject: [PATCH 264/689] Update test_in_memory_parts_still_read.py

---
 .../test_in_memory_parts_still_read.py                        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py b/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
index 94baf03c83a..d55f155918e 100644
--- a/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
+++ b/tests/integration/test_backward_compatibility/test_in_memory_parts_still_read.py
@@ -29,14 +29,14 @@ def test_in_memory_parts_still_read(start_cluster):
     node.query(
         "CREATE TABLE t (x UInt64, s String, a Array(Tuple(Map(String, LowCardinality(String)), Date32, DateTime64(3)))) ENGINE = MergeTree ORDER BY s SETTINGS min_rows_for_compact_part = 1000000, min_bytes_for_compact_part = '1G', in_memory_parts_enable_wal = 1"
     )
-    node.query("INSERT INTO t SELECT generateRandom() LIMIT 100")
+    node.query("INSERT INTO t SELECT * FROM generateRandom() LIMIT 100")
 
     assert node.query("SELECT count() FROM t WHERE NOT ignore(*)") == "100\n"
 
     node.restart_with_latest_version()
     assert node.query("SELECT count() FROM t WHERE NOT ignore(*)") == "100\n"
 
-    node.query("INSERT INTO t SELECT generateRandom() LIMIT 100")
+    node.query("INSERT INTO t SELECT * FROM generateRandom() LIMIT 100")
 
     assert node.query("SELECT count() FROM t WHERE NOT ignore(*)") == "200\n"
 

From 9eba74c1e1eb4356ed1d0600f3ce519160861051 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 17:45:45 +0300
Subject: [PATCH 265/689] Update test_vertical_merges_from_compact_parts.py

---
 .../test_vertical_merges_from_compact_parts.py                  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
index e16f74571cb..5979ad71a2e 100644
--- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
+++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
@@ -109,7 +109,7 @@ def test_vertical_merges_from_compact_parts(start_cluster):
         # that's why the error about different number of compressed files is expected and ok.
         (
             node_old.contains_in_log("CHECKSUM_DOESNT_MATCH")
-            and not node_old.contains_in_log("Different number of files: 7 compressed")
+            and not node_old.contains_in_log("Different number of files")
         )
         or (node_new.contains_in_log("CHECKSUM_DOESNT_MATCH"))
     )

From 98acbff68fd4175466ac1dd7e5554a81c8dac64a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 17:31:37 +0200
Subject: [PATCH 266/689] Disable distinct_in_order optimization

---
 src/Core/Settings.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index dbac8a7ec53..ffba3177d5b 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -733,7 +733,7 @@ class IColumn;
     M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \
     M(UInt64, grace_hash_join_initial_buckets, 1, "Initial number of grace hash join buckets", 0) \
     M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \
-    M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
+    M(Bool, optimize_distinct_in_order, false, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
     M(Bool, optimize_sorting_by_input_stream_properties, true, "Optimize sorting by sorting properties of input stream", 0) \
     M(UInt64, insert_keeper_max_retries, 20, "Max retries for keeper operations during insert", 0) \
     M(UInt64, insert_keeper_retry_initial_backoff_ms, 100, "Initial backoff timeout for keeper operations during insert", 0) \

From e275da1d31400c1171a6e1586bc6b3536c932d1a Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 23 Apr 2023 18:20:13 +0000
Subject: [PATCH 267/689] Remove deprecated logic for ccache

---
 cmake/ccache.cmake                      | 22 +++++++---------------
 src/AggregateFunctions/UniquesHashSet.h |  2 +-
 src/Functions/FunctionsHashing.h        | 17 +++++++----------
 3 files changed, 15 insertions(+), 26 deletions(-)

diff --git a/cmake/ccache.cmake b/cmake/ccache.cmake
index 285c8efb9e1..9a70e4aee32 100644
--- a/cmake/ccache.cmake
+++ b/cmake/ccache.cmake
@@ -9,27 +9,19 @@ if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_C_COMPILER_LAUNCHER MA
     return()
 endif()
 
-set(ENABLE_CCACHE "default" CACHE STRING "Deprecated, use COMPILER_CACHE=(auto|ccache|sccache|disabled)")
-if (NOT ENABLE_CCACHE STREQUAL "default")
-    message(WARNING "The -DENABLE_CCACHE is deprecated in favor of -DCOMPILER_CACHE")
-endif()
-
 set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (ccache, then sccache), 'ccache', 'sccache', or 'disabled'")
 
-# It has pretty complex logic, because the ENABLE_CCACHE is deprecated, but still should
-# control the COMPILER_CACHE
-# After it will be completely removed, the following block will be much simpler
-if (COMPILER_CACHE STREQUAL "ccache" OR (ENABLE_CCACHE AND NOT ENABLE_CCACHE STREQUAL "default"))
-    find_program (CCACHE_EXECUTABLE ccache)
-elseif(COMPILER_CACHE STREQUAL "disabled" OR NOT ENABLE_CCACHE STREQUAL "default")
-    message(STATUS "Using *ccache: no (disabled via configuration)")
-    return()
-elseif(COMPILER_CACHE STREQUAL "auto")
+if(COMPILER_CACHE STREQUAL "auto")
     find_program (CCACHE_EXECUTABLE ccache sccache)
+elseif (COMPILER_CACHE STREQUAL "ccache")
+    find_program (CCACHE_EXECUTABLE ccache)
 elseif(COMPILER_CACHE STREQUAL "sccache")
     find_program (CCACHE_EXECUTABLE sccache)
+elseif(COMPILER_CACHE STREQUAL "disabled")
+    message(STATUS "Using *ccache: no (disabled via configuration)")
+    return()
 else()
-    message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|ccache|sccache|disabled), given '${COMPILER_CACHE}'")
+    message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|ccache|sccache|disabled), value: '${COMPILER_CACHE}'")
 endif()
 
 
diff --git a/src/AggregateFunctions/UniquesHashSet.h b/src/AggregateFunctions/UniquesHashSet.h
index 2abf6ed00d6..ca6d31a716d 100644
--- a/src/AggregateFunctions/UniquesHashSet.h
+++ b/src/AggregateFunctions/UniquesHashSet.h
@@ -335,7 +335,7 @@ public:
         if constexpr (std::endian::native == std::endian::little)
             hash_value = hash(x);
         else
-            hash_value = __builtin_bswap32(hash(x));
+            hash_value = std::byteswap(hash(x));
         if (!good(hash_value))
             return;
 
diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h
index 034ef868cc7..3de757bfa3f 100644
--- a/src/Functions/FunctionsHashing.h
+++ b/src/Functions/FunctionsHashing.h
@@ -808,23 +808,20 @@ struct ImplBLAKE3
     static constexpr auto name = "BLAKE3";
     enum { length = 32 };
 
-    #if !USE_BLAKE3
-    [[noreturn]] static void apply(const char * begin, const size_t size, unsigned char* out_char_data)
+#if !USE_BLAKE3
+    [[noreturn]] static void apply(const char * /*begin*/, const size_t /*size*/, unsigned char * /*out_char_data*/)
     {
-        UNUSED(begin);
-        UNUSED(size);
-        UNUSED(out_char_data);
         throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "BLAKE3 is not available. Rust code or BLAKE3 itself may be disabled.");
     }
-    #else
+#else
     static void apply(const char * begin, const size_t size, unsigned char* out_char_data)
     {
-        #if defined(MEMORY_SANITIZER)
+#    if defined(MEMORY_SANITIZER)
             auto err_msg = blake3_apply_shim_msan_compat(begin, safe_cast<uint32_t>(size), out_char_data);
             __msan_unpoison(out_char_data, length);
-        #else
+#    else
             auto err_msg = blake3_apply_shim(begin, safe_cast<uint32_t>(size), out_char_data);
-        #endif
+#    endif
         if (err_msg != nullptr)
         {
             auto err_st = std::string(err_msg);
@@ -832,7 +829,7 @@ struct ImplBLAKE3
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function returned error message: {}", err_st);
         }
     }
-    #endif
+#endif
 };
 
 template <typename Impl>

From d0400846f119da1a6c8fd3681338f499e3649c31 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 17:50:04 +0200
Subject: [PATCH 268/689] Fix bug in DISTINCT

---
 src/Core/Settings.h                              |  2 +-
 .../queries/0_stateless/02733_distinct.reference | 10 ++++++++++
 tests/queries/0_stateless/02733_distinct.sql     | 16 ++++++++++++++++
 3 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/02733_distinct.reference
 create mode 100644 tests/queries/0_stateless/02733_distinct.sql

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index ffba3177d5b..d82df76cc6f 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -733,7 +733,7 @@ class IColumn;
     M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \
     M(UInt64, grace_hash_join_initial_buckets, 1, "Initial number of grace hash join buckets", 0) \
     M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \
-    M(Bool, optimize_distinct_in_order, false, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
+    M(Bool, optimize_distinct_in_order, false, "This optimization has a bug and it is disabled. Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
     M(Bool, optimize_sorting_by_input_stream_properties, true, "Optimize sorting by sorting properties of input stream", 0) \
     M(UInt64, insert_keeper_max_retries, 20, "Max retries for keeper operations during insert", 0) \
     M(UInt64, insert_keeper_retry_initial_backoff_ms, 100, "Initial backoff timeout for keeper operations during insert", 0) \
diff --git a/tests/queries/0_stateless/02733_distinct.reference b/tests/queries/0_stateless/02733_distinct.reference
new file mode 100644
index 00000000000..caaa76087c9
--- /dev/null
+++ b/tests/queries/0_stateless/02733_distinct.reference
@@ -0,0 +1,10 @@
+		1
+		2
+v1	v2	3
+v1	v2	4
+v1	v2	5
+		1
+		2
+v1	v2	3
+v1	v2	4
+v1	v2	5
diff --git a/tests/queries/0_stateless/02733_distinct.sql b/tests/queries/0_stateless/02733_distinct.sql
new file mode 100644
index 00000000000..e88900f3cac
--- /dev/null
+++ b/tests/queries/0_stateless/02733_distinct.sql
@@ -0,0 +1,16 @@
+DROP TABLE IF EXISTS test;
+CREATE TABLE test
+(
+    c1 String,
+    c2 String,
+    c3 String
+)
+ENGINE = ReplacingMergeTree
+ORDER BY (c1, c3);
+
+INSERT INTO test(c1, c2, c3) VALUES ('', '', '1'), ('', '', '2'),('v1', 'v2', '3'),('v1', 'v2', '4'),('v1', 'v2', '5');
+
+SELECT c1, c2, c3 FROM test GROUP BY c1, c2, c3 ORDER BY c1, c2, c3;
+SELECT DISTINCT c1, c2, c3 FROM test ORDER BY c1, c2, c3;
+
+DROP TABLE test;

From b2963eb4bebeb503789a983d1324860dd4819bff Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Sun, 7 May 2023 17:59:46 +0200
Subject: [PATCH 269/689] Add documentation how to install clickhouse on SUSE
 family

---
 docs/en/getting-started/install.md |  9 +++++++++
 docs/ru/getting-started/install.md | 28 +++++++++++++++++++++++++---
 docs/zh/getting-started/install.md | 11 +++++++++++
 3 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md
index d5c476f08b4..5fac768ddef 100644
--- a/docs/en/getting-started/install.md
+++ b/docs/en/getting-started/install.md
@@ -184,6 +184,15 @@ sudo yum install -y yum-utils
 sudo yum-config-manager --add-repo https://packages.clickhouse.com/rpm/clickhouse.repo
 ```
 
+For systems with `zypper` package manager (openSUSE, SUSE SLES):
+
+``` bash
+sudo zypper addrepo -r https://packages.clickhouse.com/rpm/clickhouse.repo -g
+sudo zypper --gpg-auto-import-keys refresh clickhouse-stable
+```
+
+Later any `yum install` can be replaced by `zypper install`. To specify a particular version, add `-$VERSION` to the end of the package name, e.g. `clickhouse-client-22.2.2.22`.
+
 #### Install ClickHouse server and client
 
 ```bash
diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md
index 02d8c62669d..cc6380907ae 100644
--- a/docs/ru/getting-started/install.md
+++ b/docs/ru/getting-started/install.md
@@ -77,15 +77,37 @@ clickhouse-client # or "clickhouse-client --password" if you set up a password.
 
 Команда ClickHouse в Яндексе рекомендует использовать официальные предкомпилированные `rpm` пакеты для CentOS, RedHat и всех остальных дистрибутивов Linux, основанных на rpm.
 
+#### Установка официального репозитория
+
 Сначала нужно подключить официальный репозиторий:
 
 ``` bash
 sudo yum install -y yum-utils
 sudo yum-config-manager --add-repo https://packages.clickhouse.com/rpm/clickhouse.repo
-sudo yum install -y clickhouse-server clickhouse-client
+```
 
-sudo /etc/init.d/clickhouse-server start
-clickhouse-client # or "clickhouse-client --password" if you set up a password.
+Для систем с пакетным менеджером `zypper` (openSUSE, SUSE SLES):
+
+``` bash
+sudo zypper addrepo -r https://packages.clickhouse.com/rpm/clickhouse.repo -g
+sudo zypper --gpg-auto-import-keys refresh clickhouse-stable
+```
+
+Далее любая команда `yum install` может быть заменена на `zypper install`. Чтобы указать желаемую версию, необходимо добавить `-$VERSION` в имени пакета, например `clickhouse-client-22.2.2.22`.
+
+#### Установка сервера и клиента
+
+``` bash
+sudo yum install -y clickhouse-server clickhouse-client
+```
+
+#### Запуск сервера
+
+``` bash
+sudo systemctl enable clickhouse-server
+sudo systemctl start clickhouse-server
+sudo systemctl status clickhouse-server
+clickhouse-client # илм "clickhouse-client --password" если установлен пароль
 ```
 
 <details markdown="1">
diff --git a/docs/zh/getting-started/install.md b/docs/zh/getting-started/install.md
index f622c51c0c4..81aab45d4ad 100644
--- a/docs/zh/getting-started/install.md
+++ b/docs/zh/getting-started/install.md
@@ -84,6 +84,17 @@ sudo /etc/init.d/clickhouse-server start
 clickhouse-client # or "clickhouse-client --password" if you set up a password.
 ```
 
+For systems with `zypper` package manager (openSUSE, SUSE SLES):
+
+``` bash
+sudo zypper addrepo -r https://packages.clickhouse.com/rpm/clickhouse.repo -g
+sudo zypper --gpg-auto-import-keys refresh clickhouse-stable
+sudo zypper install -y clickhouse-server clickhouse-client
+
+sudo /etc/init.d/clickhouse-server start
+clickhouse-client # or "clickhouse-client --password" if you set up a password.
+```
+
 <details markdown="1">
 
 <summary>Deprecated Method for installing rpm-packages</summary>

From 1ae8a5be3bf55baa6c2cc17cabcd7d4011b71178 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 18:24:38 +0200
Subject: [PATCH 270/689] Update a test

---
 tests/integration/test_merge_tree_load_parts/test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_merge_tree_load_parts/test.py b/tests/integration/test_merge_tree_load_parts/test.py
index 049dd516647..a95b6674b75 100644
--- a/tests/integration/test_merge_tree_load_parts/test.py
+++ b/tests/integration/test_merge_tree_load_parts/test.py
@@ -227,7 +227,7 @@ def test_merge_tree_load_parts_filesystem_error(started_cluster):
     # It can be a filesystem exception triggered at initialization of part storage but it hard
     # to trigger it because it should be an exception on stat/listDirectory.
     # The most easy way to trigger such exception is to use chmod but clickhouse server
-    # is run with root user in integration test and this won't work. So let's do some
+    # is run with root user in integration test and this won't work. So let's do
     # some stupid things: create a table without adaptive granularity and change mark
     # extensions of data files in part to make clickhouse think that it's a compact part which
     # cannot be created in such table. This will trigger a LOGICAL_ERROR on part creation.
@@ -240,7 +240,7 @@ def test_merge_tree_load_parts_filesystem_error(started_cluster):
         ).strip()
 
         node3.exec_in_container(
-            ["bash", "-c", f"mv {part_path}id.mrk {part_path}id.mrk3"], privileged=True
+            ["bash", "-c", f"mv {part_path}id.cmrk {part_path}id.cmrk3"], privileged=True
         )
 
     corrupt_part("mt_load_parts", "all_1_1_0")

From fe76bde5b50de75f91c2440674b563aaa38dc973 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Sun, 7 May 2023 16:33:07 +0000
Subject: [PATCH 271/689] Automatic style fix

---
 tests/integration/test_merge_tree_load_parts/test.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_merge_tree_load_parts/test.py b/tests/integration/test_merge_tree_load_parts/test.py
index a95b6674b75..118c31ea864 100644
--- a/tests/integration/test_merge_tree_load_parts/test.py
+++ b/tests/integration/test_merge_tree_load_parts/test.py
@@ -240,7 +240,8 @@ def test_merge_tree_load_parts_filesystem_error(started_cluster):
         ).strip()
 
         node3.exec_in_container(
-            ["bash", "-c", f"mv {part_path}id.cmrk {part_path}id.cmrk3"], privileged=True
+            ["bash", "-c", f"mv {part_path}id.cmrk {part_path}id.cmrk3"],
+            privileged=True,
         )
 
     corrupt_part("mt_load_parts", "all_1_1_0")

From e1fa92e078dd4c4810eaf1e8518618cf5bcfe85d Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Sun, 7 May 2023 18:36:35 +0200
Subject: [PATCH 272/689] Add openSUSE and SLES to docs dictionary, sort it

---
 docs/en/getting-started/install.md             |  2 +-
 docs/ru/getting-started/install.md             |  2 +-
 docs/zh/getting-started/install.md             |  2 +-
 .../aspell-ignore/en/aspell-dict.txt           | 18 ++++++++++--------
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md
index 5fac768ddef..308e3197ad7 100644
--- a/docs/en/getting-started/install.md
+++ b/docs/en/getting-started/install.md
@@ -184,7 +184,7 @@ sudo yum install -y yum-utils
 sudo yum-config-manager --add-repo https://packages.clickhouse.com/rpm/clickhouse.repo
 ```
 
-For systems with `zypper` package manager (openSUSE, SUSE SLES):
+For systems with `zypper` package manager (openSUSE, SLES):
 
 ``` bash
 sudo zypper addrepo -r https://packages.clickhouse.com/rpm/clickhouse.repo -g
diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md
index cc6380907ae..f041db43d0f 100644
--- a/docs/ru/getting-started/install.md
+++ b/docs/ru/getting-started/install.md
@@ -86,7 +86,7 @@ sudo yum install -y yum-utils
 sudo yum-config-manager --add-repo https://packages.clickhouse.com/rpm/clickhouse.repo
 ```
 
-Для систем с пакетным менеджером `zypper` (openSUSE, SUSE SLES):
+Для систем с пакетным менеджером `zypper` (openSUSE, SLES):
 
 ``` bash
 sudo zypper addrepo -r https://packages.clickhouse.com/rpm/clickhouse.repo -g
diff --git a/docs/zh/getting-started/install.md b/docs/zh/getting-started/install.md
index 81aab45d4ad..e65cfea62cd 100644
--- a/docs/zh/getting-started/install.md
+++ b/docs/zh/getting-started/install.md
@@ -84,7 +84,7 @@ sudo /etc/init.d/clickhouse-server start
 clickhouse-client # or "clickhouse-client --password" if you set up a password.
 ```
 
-For systems with `zypper` package manager (openSUSE, SUSE SLES):
+For systems with `zypper` package manager (openSUSE, SLES):
 
 ``` bash
 sudo zypper addrepo -r https://packages.clickhouse.com/rpm/clickhouse.repo -g
diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt
index 8c5d877755f..65844a4946e 100644
--- a/utils/check-style/aspell-ignore/en/aspell-dict.txt
+++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt
@@ -22,9 +22,9 @@ CSVWithNamesAndTypes
 CamelCase
 CapnProto
 CentOS
-ClickableSquare
 ClickHouse
 ClickHouse's
+ClickableSquare
 CodeBlock
 CodeLLDB
 Config
@@ -86,16 +86,16 @@ Jepsen
 KDevelop
 LGPL
 LLDB
+LLVM's
 LOCALTIME
 LOCALTIMESTAMP
 LibFuzzer
 LineAsString
 LinksDeployment
-LLVM's
 LowCardinality
 MEMTABLE
-MVCC
 MSan
+MVCC
 MacOS
 Memcheck
 MemorySanitizer
@@ -111,24 +111,25 @@ NULLIF
 NVME
 NYPD
 NuRaft
-Observability
 OLAP
 OLTP
 ObjectId
+Observability
 Ok
 OpenSSL
 OpenSUSE
 OpenStack
 OpenTelemetry
 PAAMAYIM
-Parsers
 ParquetMetadata
+Parsers
 Postgres
 Precompiled
 PrettyCompact
 PrettyCompactMonoBlock
 PrettyCompactNoEscapes
 PrettyCompactNoEscapesMonoBlock
+PrettyJSONEachRow
 PrettyMonoBlock
 PrettyNoEscapes
 PrettyNoEscapesMonoBlock
@@ -136,7 +137,6 @@ PrettySpace
 PrettySpaceMonoBlock
 PrettySpaceNoEscapes
 PrettySpaceNoEscapesMonoBlock
-PrettyJSONEachRow
 Protobuf
 ProtobufSingle
 QEMU
@@ -155,6 +155,7 @@ SATA
 SELECTs
 SERIALIZABLE
 SIMD
+SLES
 SMALLINT
 SQLInsert
 SQLSTATE
@@ -293,9 +294,9 @@ dmesg
 dont
 dragonbox
 durations
+encodings
 endian
 enum
-encodings
 exFAT
 fastops
 fcoverage
@@ -404,6 +405,7 @@ num
 obfuscator
 odbc
 ok
+openSUSE
 openldap
 opentelemetry
 overcommit
@@ -431,6 +433,7 @@ prettycompact
 prettycompactmonoblock
 prettycompactnoescapes
 prettycompactnoescapesmonoblock
+prettyjsoneachrow
 prettymonoblock
 prettynoescapes
 prettynoescapesmonoblock
@@ -438,7 +441,6 @@ prettyspace
 prettyspacemonoblock
 prettyspacenoescapes
 prettyspacenoescapesmonoblock
-prettyjsoneachrow
 prlimit
 prometheus
 proto

From 1cb02c9da38dc1f7a8020c699005f78396f65d7a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 21:20:12 +0300
Subject: [PATCH 273/689] Update tests/queries/0_stateless/02733_distinct.sql

Co-authored-by: Igor Nikonov <954088+devcrafter@users.noreply.github.com>
---
 tests/queries/0_stateless/02733_distinct.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02733_distinct.sql b/tests/queries/0_stateless/02733_distinct.sql
index e88900f3cac..7e657c43fe5 100644
--- a/tests/queries/0_stateless/02733_distinct.sql
+++ b/tests/queries/0_stateless/02733_distinct.sql
@@ -11,6 +11,6 @@ ORDER BY (c1, c3);
 INSERT INTO test(c1, c2, c3) VALUES ('', '', '1'), ('', '', '2'),('v1', 'v2', '3'),('v1', 'v2', '4'),('v1', 'v2', '5');
 
 SELECT c1, c2, c3 FROM test GROUP BY c1, c2, c3 ORDER BY c1, c2, c3;
-SELECT DISTINCT c1, c2, c3 FROM test ORDER BY c1, c2, c3;
+SELECT DISTINCT c1, c2, c3 FROM test;
 
 DROP TABLE test;

From c841a187ab9a98e6d3906f1c08cf3b7ceda79688 Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Sun, 7 May 2023 18:34:34 +0000
Subject: [PATCH 274/689] Try to fix integration test
 'test_ssl_cert_authentication'

---
 .../test_ssl_cert_authentication/test.py      | 33 +++++++++++--------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py
index 514897ad345..641680b07d3 100644
--- a/tests/integration/test_ssl_cert_authentication/test.py
+++ b/tests/integration/test_ssl_cert_authentication/test.py
@@ -164,19 +164,26 @@ def get_ssl_context(cert_name):
 def execute_query_https(
     query, user, enable_ssl_auth=True, cert_name=None, password=None
 ):
-    url = (
-        f"https://{instance.ip_address}:{HTTPS_PORT}/?query={urllib.parse.quote(query)}"
-    )
-    request = urllib.request.Request(url)
-    request.add_header("X-ClickHouse-User", user)
-    if enable_ssl_auth:
-        request.add_header("X-ClickHouse-SSL-Certificate-Auth", "on")
-    if password:
-        request.add_header("X-ClickHouse-Key", password)
-    response = urllib.request.urlopen(
-        request, context=get_ssl_context(cert_name)
-    ).read()
-    return response.decode("utf-8")
+    retries = 10
+    while True:
+        try:
+            url = (
+            f"https://{instance.ip_address}:{HTTPS_PORT}/?query={urllib.parse.quote(query)}"
+            )
+            request = urllib.request.Request(url)
+            request.add_header("X-ClickHouse-User", user)
+            if enable_ssl_auth:
+                request.add_header("X-ClickHouse-SSL-Certificate-Auth", "on")
+            if password:
+                request.add_header("X-ClickHouse-Key", password)
+            response = urllib.request.urlopen(
+                request, context=get_ssl_context(cert_name)
+            ).read()
+            return response.decode("utf-8")
+        except BrokenPipeError:
+            retries -= 1
+            if retries == 0:
+                raise
 
 
 def test_https():

From d614bde1c764a165d49081099fb83276cdae1190 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 7 May 2023 19:20:16 +0000
Subject: [PATCH 275/689] Fixup docs of extractKeyValuePairs*()

---
 .../functions/string-functions.md             | 55 ++++++++++---------
 1 file changed, 30 insertions(+), 25 deletions(-)

diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md
index c543cda5ae2..92df811365a 100644
--- a/docs/en/sql-reference/functions/string-functions.md
+++ b/docs/en/sql-reference/functions/string-functions.md
@@ -1218,64 +1218,66 @@ Result:
 
 ## extractKeyValuePairs
 
-Extracts key-value pairs from any string. The string does not need to be 100% structured in a key value pair format;
+Extracts key-value pairs, i.e. a [Map(String, String)](../../sql-reference/data-types/map.md), from a string. Parsing is robust towards noise (e.g. log files).
 
-It can contain noise (e.g. log files). The key-value pair format to be interpreted should be specified via function arguments.
-
-A key-value pair consists of a key followed by a `key_value_delimiter` and a value. Quoted keys and values are also supported. Key value pairs must be separated by pair delimiters.
+A key-value pair consists of a key, followed by a `key_value_delimiter` and a value. Key value pairs must be separated by `pair_delimiter`. Quoted keys and values are also supported. 
 
 **Syntax**
+
 ``` sql
-extractKeyValuePairs(data, [key_value_delimiter], [pair_delimiter], [quoting_character])
+extractKeyValuePairs(data[, key_value_delimiter[, pair_delimiter[, quoting_character]]])
 ```
 
 **Arguments**
+
 - `data` - String to extract key-value pairs from. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
 - `key_value_delimiter` - Character to be used as delimiter between the key and the value. Defaults to `:`. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
-- `pair_delimiters` - Set of character to be used as delimiters between pairs. Defaults to `\space`, `,` and `;`. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
+- `pair_delimiters` - Set of character to be used as delimiters between pairs. Defaults to ` `, `,` and `;`. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
 - `quoting_character` - Character to be used as quoting character. Defaults to `"`. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
 
 **Returned values**
-- The extracted key-value pairs in a Map(String, String).
+
+- A [Map(String, String)](../../sql-reference/data-types/map.md) of key-value pairs.
 
 **Examples**
 
-Query:
+Simple case:
 
-**Simple case**
 ``` sql
-arthur :) select extractKeyValuePairs('name:neymar, age:31 team:psg,nationality:brazil') as kv
-
 SELECT extractKeyValuePairs('name:neymar, age:31 team:psg,nationality:brazil') as kv
+```
 
-Query id: f9e0ca6f-3178-4ee2-aa2c-a5517abb9cee
+Result:
 
+``` Result:
 ┌─kv──────────────────────────────────────────────────────────────────────┐
 │ {'name':'neymar','age':'31','team':'psg','nationality':'brazil'}        │
 └─────────────────────────────────────────────────────────────────────────┘
 ```
 
-**Single quote as quoting character**
+Single quote as quoting character:
+
 ``` sql
-arthur :) select extractKeyValuePairs('name:\'neymar\';\'age\':31;team:psg;nationality:brazil,last_key:last_value', ':', ';,', '\'') as kv
-
 SELECT extractKeyValuePairs('name:\'neymar\';\'age\':31;team:psg;nationality:brazil,last_key:last_value', ':', ';,', '\'') as kv
+```
 
-Query id: 0e22bf6b-9844-414a-99dc-32bf647abd5e
+Result:
 
+``` text
 ┌─kv───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
 │ {'name':'neymar','age':'31','team':'psg','nationality':'brazil','last_key':'last_value'}                                 │
 └──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
 ```
 
-**Escape sequences without escape sequences support**
+Escape sequences without escape sequences support:
+
 ``` sql
-arthur :) select extractKeyValuePairs('age:a\\x0A\\n\\0') as kv
-
 SELECT extractKeyValuePairs('age:a\\x0A\\n\\0') AS kv
+```
 
-Query id: e9fd26ee-b41f-4a11-b17f-25af6fd5d356
+Result:
 
+``` text
 ┌─kv─────────────────────┐
 │ {'age':'a\\x0A\\n\\0'} │
 └────────────────────────┘
@@ -1285,7 +1287,7 @@ Query id: e9fd26ee-b41f-4a11-b17f-25af6fd5d356
 
 Same as `extractKeyValuePairs` but with escaping support.
 
-Escape sequences supported: `\x`, `\N`, `\a`, `\b`, `\e`, `\f`, `\n`, `\r`, `\t`, `\v` and `\0`.
+Supported escape sequences: `\x`, `\N`, `\a`, `\b`, `\e`, `\f`, `\n`, `\r`, `\t`, `\v` and `\0`.
 Non standard escape sequences are returned as it is (including the backslash) unless they are one of the following:
 `\\`, `'`, `"`, `backtick`, `/`, `=` or ASCII control characters (c <= 31).
 
@@ -1296,14 +1298,17 @@ input string: `a: "aaaa\"bbb"`. The expected output is: `a: aaaa\"bbbb`.
 
 Leading escape sequences will be skipped in keys and will be considered invalid for values.
 
-**Escape sequences with escape sequence support turned on**
+**Examples**
+
+Escape sequences with escape sequence support turned on:
+
 ``` sql
-arthur :) select extractKeyValuePairsWithEscaping('age:a\\x0A\\n\\0') as kv
-
 SELECT extractKeyValuePairsWithEscaping('age:a\\x0A\\n\\0') AS kv
+```
 
-Query id: 44c114f0-5658-4c75-ab87-4574de3a1645
+Result:
 
+``` result
 ┌─kv────────────────┐
 │ {'age':'a\n\n\0'} │
 └───────────────────┘

From d5f08ba87bf1d3006eb1e344ac788d92570d86c7 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 7 May 2023 19:22:29 +0000
Subject: [PATCH 276/689] Move extractKeyValuePairs*() docs into maps docs

---
 .../functions/string-functions.md             | 98 -------------------
 .../functions/tuple-map-functions.md          | 98 +++++++++++++++++++
 2 files changed, 98 insertions(+), 98 deletions(-)

diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md
index 92df811365a..8489b9d950b 100644
--- a/docs/en/sql-reference/functions/string-functions.md
+++ b/docs/en/sql-reference/functions/string-functions.md
@@ -1215,101 +1215,3 @@ Result:
 │ A240             │
 └──────────────────┘
 ```
-
-## extractKeyValuePairs
-
-Extracts key-value pairs, i.e. a [Map(String, String)](../../sql-reference/data-types/map.md), from a string. Parsing is robust towards noise (e.g. log files).
-
-A key-value pair consists of a key, followed by a `key_value_delimiter` and a value. Key value pairs must be separated by `pair_delimiter`. Quoted keys and values are also supported. 
-
-**Syntax**
-
-``` sql
-extractKeyValuePairs(data[, key_value_delimiter[, pair_delimiter[, quoting_character]]])
-```
-
-**Arguments**
-
-- `data` - String to extract key-value pairs from. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
-- `key_value_delimiter` - Character to be used as delimiter between the key and the value. Defaults to `:`. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
-- `pair_delimiters` - Set of character to be used as delimiters between pairs. Defaults to ` `, `,` and `;`. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
-- `quoting_character` - Character to be used as quoting character. Defaults to `"`. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
-
-**Returned values**
-
-- A [Map(String, String)](../../sql-reference/data-types/map.md) of key-value pairs.
-
-**Examples**
-
-Simple case:
-
-``` sql
-SELECT extractKeyValuePairs('name:neymar, age:31 team:psg,nationality:brazil') as kv
-```
-
-Result:
-
-``` Result:
-┌─kv──────────────────────────────────────────────────────────────────────┐
-│ {'name':'neymar','age':'31','team':'psg','nationality':'brazil'}        │
-└─────────────────────────────────────────────────────────────────────────┘
-```
-
-Single quote as quoting character:
-
-``` sql
-SELECT extractKeyValuePairs('name:\'neymar\';\'age\':31;team:psg;nationality:brazil,last_key:last_value', ':', ';,', '\'') as kv
-```
-
-Result:
-
-``` text
-┌─kv───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
-│ {'name':'neymar','age':'31','team':'psg','nationality':'brazil','last_key':'last_value'}                                 │
-└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
-```
-
-Escape sequences without escape sequences support:
-
-``` sql
-SELECT extractKeyValuePairs('age:a\\x0A\\n\\0') AS kv
-```
-
-Result:
-
-``` text
-┌─kv─────────────────────┐
-│ {'age':'a\\x0A\\n\\0'} │
-└────────────────────────┘
-```
-
-## extractKeyValuePairsWithEscaping
-
-Same as `extractKeyValuePairs` but with escaping support.
-
-Supported escape sequences: `\x`, `\N`, `\a`, `\b`, `\e`, `\f`, `\n`, `\r`, `\t`, `\v` and `\0`.
-Non standard escape sequences are returned as it is (including the backslash) unless they are one of the following:
-`\\`, `'`, `"`, `backtick`, `/`, `=` or ASCII control characters (c <= 31).
-
-This function will satisfy the use case where pre-escaping and post-escaping are not suitable. For instance, consider the following
-input string: `a: "aaaa\"bbb"`. The expected output is: `a: aaaa\"bbbb`.
-- Pre-escaping: Pre-escaping it will output: `a: "aaaa"bbb"` and `extractKeyValuePairs` will then output: `a: aaaa`
-- Post-escaping: `extractKeyValuePairs` will output `a: aaaa\` and post-escaping will keep it as it is.
-
-Leading escape sequences will be skipped in keys and will be considered invalid for values.
-
-**Examples**
-
-Escape sequences with escape sequence support turned on:
-
-``` sql
-SELECT extractKeyValuePairsWithEscaping('age:a\\x0A\\n\\0') AS kv
-```
-
-Result:
-
-``` result
-┌─kv────────────────┐
-│ {'age':'a\n\n\0'} │
-└───────────────────┘
-```
diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md
index 786ea47f12c..74a4cd5b240 100644
--- a/docs/en/sql-reference/functions/tuple-map-functions.md
+++ b/docs/en/sql-reference/functions/tuple-map-functions.md
@@ -109,6 +109,104 @@ SELECT mapFromArrays([1, 2, 3], map('a', 1, 'b', 2, 'c', 3))
 └───────────────────────────────────────────────────────┘
 ```
 
+## extractKeyValuePairs
+
+Extracts key-value pairs, i.e. a [Map(String, String)](../../sql-reference/data-types/map.md), from a string. Parsing is robust towards noise (e.g. log files).
+
+A key-value pair consists of a key, followed by a `key_value_delimiter` and a value. Key value pairs must be separated by `pair_delimiter`. Quoted keys and values are also supported. 
+
+**Syntax**
+
+``` sql
+extractKeyValuePairs(data[, key_value_delimiter[, pair_delimiter[, quoting_character]]])
+```
+
+**Arguments**
+
+- `data` - String to extract key-value pairs from. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
+- `key_value_delimiter` - Character to be used as delimiter between the key and the value. Defaults to `:`. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
+- `pair_delimiters` - Set of character to be used as delimiters between pairs. Defaults to ` `, `,` and `;`. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
+- `quoting_character` - Character to be used as quoting character. Defaults to `"`. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
+
+**Returned values**
+
+- A [Map(String, String)](../../sql-reference/data-types/map.md) of key-value pairs.
+
+**Examples**
+
+Simple case:
+
+``` sql
+SELECT extractKeyValuePairs('name:neymar, age:31 team:psg,nationality:brazil') as kv
+```
+
+Result:
+
+``` Result:
+┌─kv──────────────────────────────────────────────────────────────────────┐
+│ {'name':'neymar','age':'31','team':'psg','nationality':'brazil'}        │
+└─────────────────────────────────────────────────────────────────────────┘
+```
+
+Single quote as quoting character:
+
+``` sql
+SELECT extractKeyValuePairs('name:\'neymar\';\'age\':31;team:psg;nationality:brazil,last_key:last_value', ':', ';,', '\'') as kv
+```
+
+Result:
+
+``` text
+┌─kv───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
+│ {'name':'neymar','age':'31','team':'psg','nationality':'brazil','last_key':'last_value'}                                 │
+└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
+```
+
+Escape sequences without escape sequences support:
+
+``` sql
+SELECT extractKeyValuePairs('age:a\\x0A\\n\\0') AS kv
+```
+
+Result:
+
+``` text
+┌─kv─────────────────────┐
+│ {'age':'a\\x0A\\n\\0'} │
+└────────────────────────┘
+```
+
+## extractKeyValuePairsWithEscaping
+
+Same as `extractKeyValuePairs` but with escaping support.
+
+Supported escape sequences: `\x`, `\N`, `\a`, `\b`, `\e`, `\f`, `\n`, `\r`, `\t`, `\v` and `\0`.
+Non standard escape sequences are returned as it is (including the backslash) unless they are one of the following:
+`\\`, `'`, `"`, `backtick`, `/`, `=` or ASCII control characters (c <= 31).
+
+This function will satisfy the use case where pre-escaping and post-escaping are not suitable. For instance, consider the following
+input string: `a: "aaaa\"bbb"`. The expected output is: `a: aaaa\"bbbb`.
+- Pre-escaping: Pre-escaping it will output: `a: "aaaa"bbb"` and `extractKeyValuePairs` will then output: `a: aaaa`
+- Post-escaping: `extractKeyValuePairs` will output `a: aaaa\` and post-escaping will keep it as it is.
+
+Leading escape sequences will be skipped in keys and will be considered invalid for values.
+
+**Examples**
+
+Escape sequences with escape sequence support turned on:
+
+``` sql
+SELECT extractKeyValuePairsWithEscaping('age:a\\x0A\\n\\0') AS kv
+```
+
+Result:
+
+``` result
+┌─kv────────────────┐
+│ {'age':'a\n\n\0'} │
+└───────────────────┘
+```
+
 ## mapAdd
 
 Collect all the keys and sum corresponding values.

From 3cadcaeaf0d5e898184a42e98cb3d57829db2163 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 7 May 2023 19:24:49 +0000
Subject: [PATCH 277/689] Add new aliases to docs

---
 docs/en/sql-reference/functions/tuple-map-functions.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md
index 74a4cd5b240..c84620160a8 100644
--- a/docs/en/sql-reference/functions/tuple-map-functions.md
+++ b/docs/en/sql-reference/functions/tuple-map-functions.md
@@ -121,6 +121,10 @@ A key-value pair consists of a key, followed by a `key_value_delimiter` and a va
 extractKeyValuePairs(data[, key_value_delimiter[, pair_delimiter[, quoting_character]]])
 ```
 
+Alias:
+- `str_to_map`
+- `mapFromString`
+
 **Arguments**
 
 - `data` - String to extract key-value pairs from. [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).

From 59e49abb324d771882c77b7618778f5097b53486 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 7 May 2023 19:30:50 +0000
Subject: [PATCH 278/689] Check case-insensitivity in tests

---
 .../02499_extract_key_value_pairs_multiple_input.reference    | 4 +++-
 .../02499_extract_key_value_pairs_multiple_input.sql          | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
index 3160825eb89..125afa19427 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
@@ -304,8 +304,9 @@ WITH
 SELECT
     x;
 {}
+-- check str_to_map alias (it is case-insensitive)
 WITH
-    str_to_map('name:neymar, age:31 team:psg,nationality:brazil') AS s_map,
+    sTr_tO_mAp('name:neymar, age:31 team:psg,nationality:brazil') AS s_map,
     CAST(
             arrayMap(
                     (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
@@ -315,6 +316,7 @@ WITH
 SELECT
     x;
 {'age':'31','name':'neymar','nationality':'brazil','team':'psg'}
+-- check mapFromString alias
 WITH
     mapFromString('name:neymar, age:31 team:psg,nationality:brazil') AS s_map,
     CAST(
diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
index eeb004ecd36..0a1500b1796 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
@@ -427,8 +427,9 @@ WITH
 SELECT
     x;
 
+-- check str_to_map alias (it is case-insensitive)
 WITH
-    str_to_map('name:neymar, age:31 team:psg,nationality:brazil') AS s_map,
+    sTr_tO_mAp('name:neymar, age:31 team:psg,nationality:brazil') AS s_map,
     CAST(
             arrayMap(
                     (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
@@ -438,6 +439,7 @@ WITH
 SELECT
     x;
 
+-- check mapFromString alias
 WITH
     mapFromString('name:neymar, age:31 team:psg,nationality:brazil') AS s_map,
     CAST(

From 4d5d5354d1368a2784091511dbb7ba510c36fa05 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 7 May 2023 19:44:58 +0000
Subject: [PATCH 279/689] Unflake 01660_system_parts_smoke

---
 tests/queries/0_stateless/01660_system_parts_smoke.sql | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01660_system_parts_smoke.sql b/tests/queries/0_stateless/01660_system_parts_smoke.sql
index 64cba86b8f6..ad73e33eee1 100644
--- a/tests/queries/0_stateless/01660_system_parts_smoke.sql
+++ b/tests/queries/0_stateless/01660_system_parts_smoke.sql
@@ -21,8 +21,8 @@ SELECT '# two parts';
 INSERT INTO data_01660 VALUES (0);
 INSERT INTO data_01660 VALUES (1);
 SELECT _state FROM system.parts WHERE database = currentDatabase() AND table = 'data_01660';
-SELECT name, _state FROM system.parts WHERE database = currentDatabase() AND table = 'data_01660';
-SELECT name, active FROM system.parts WHERE database = currentDatabase() AND table = 'data_01660';
+SELECT name, _state FROM system.parts WHERE database = currentDatabase() AND table = 'data_01660' ORDER BY name;
+SELECT name, active FROM system.parts WHERE database = currentDatabase() AND table = 'data_01660' ORDER BY name;
 
 -- OPTIMIZE to create Outdated parts
 SELECT '# optimize';
@@ -35,7 +35,7 @@ SELECT count(), _state FROM system.parts WHERE database = currentDatabase() AND
 -- Inactive parts are clearing by async process also
 SELECT '# truncate';
 TRUNCATE data_01660;
-SELECT if (count() > 0, 'HAVE PARTS', 'NO PARTS'), _state FROM system.parts WHERE database = currentDatabase() AND table = 'data_01660' GROUP BY _state;
+SELECT if (count() > 0, 'HAVE PARTS', 'NO PARTS'), _state FROM system.parts WHERE database = currentDatabase() AND table = 'data_01660' GROUP BY _state ORDER BY _state;
 
 -- But DROP does
 SELECT '# drop';

From 595fdbeca49a20490b67119c27111a787309779e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 22:56:41 +0300
Subject: [PATCH 280/689] Update 02733_distinct.sql

---
 tests/queries/0_stateless/02733_distinct.sql | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/queries/0_stateless/02733_distinct.sql b/tests/queries/0_stateless/02733_distinct.sql
index 7e657c43fe5..bbb26b17d8c 100644
--- a/tests/queries/0_stateless/02733_distinct.sql
+++ b/tests/queries/0_stateless/02733_distinct.sql
@@ -1,3 +1,6 @@
+-- Tags: no-random-settings
+-- there is a bug if `optimize_distinct_in_order` is true
+
 DROP TABLE IF EXISTS test;
 CREATE TABLE test
 (

From 156cd1de6f560ddb57a9aa9c70cffa226e582d5d Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 7 May 2023 22:03:37 +0200
Subject: [PATCH 281/689] Revert "Remove trash"

---
 .gitmodules         | 3 +++
 contrib/idxd-config | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 contrib/idxd-config

diff --git a/.gitmodules b/.gitmodules
index ed61ddb96ba..56aeea2bfe8 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -253,6 +253,9 @@
 [submodule "contrib/qpl"]
 	path = contrib/qpl
 	url = https://github.com/intel/qpl
+[submodule "contrib/idxd-config"]
+	path = contrib/idxd-config
+	url = https://github.com/intel/idxd-config
 [submodule "contrib/wyhash"]
 	path = contrib/wyhash
 	url = https://github.com/wangyi-fudan/wyhash
diff --git a/contrib/idxd-config b/contrib/idxd-config
new file mode 160000
index 00000000000..f6605c41a73
--- /dev/null
+++ b/contrib/idxd-config
@@ -0,0 +1 @@
+Subproject commit f6605c41a735e3fdfef2d2d18655a33af6490b99

From 736eac4a5df4b55781fe963c5ed5db3fa988c2e0 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 22:46:02 +0200
Subject: [PATCH 282/689] Remove commented code

---
 src/Processors/Transforms/WindowTransform.cpp | 55 -------------------
 1 file changed, 55 deletions(-)

diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp
index d2cce2cb890..daeafbca6e4 100644
--- a/src/Processors/Transforms/WindowTransform.cpp
+++ b/src/Processors/Transforms/WindowTransform.cpp
@@ -323,8 +323,6 @@ void WindowTransform::advancePartitionEnd()
 
     const RowNumber end = blocksEnd();
 
-//    fmt::print(stderr, "end {}, partition_end {}\n", end, partition_end);
-
     // If we're at the total end of data, we must end the partition. This is one
     // of the few places in calculations where we need special handling for end
     // of data, other places will work as usual based on
@@ -383,9 +381,6 @@ void WindowTransform::advancePartitionEnd()
     const auto block_rows = blockRowsNumber(partition_end);
     for (; partition_end.row < block_rows; ++partition_end.row)
     {
-//        fmt::print(stderr, "compare reference '{}' to compared '{}'\n",
-//            prev_frame_start, partition_end);
-
         size_t i = 0;
         for (; i < partition_by_columns; ++i)
         {
@@ -394,9 +389,6 @@ void WindowTransform::advancePartitionEnd()
             const auto * compared_column
                 = inputAt(partition_end)[partition_by_indices[i]].get();
 
-//            fmt::print(stderr, "reference '{}', compared '{}'\n",
-//                (*reference_column)[prev_frame_start.row],
-//                (*compared_column)[partition_end.row]);
             if (compared_column->compareAt(partition_end.row,
                     prev_frame_start.row, *reference_column,
                     1 /* nan_direction_hint */) != 0)
@@ -499,8 +491,6 @@ auto WindowTransform::moveRowNumber(const RowNumber & _x, int64_t offset) const
     // Check that it was reversible.
     auto [xx, oo] = moveRowNumberNoCheck(x, -(offset - o));
 
-//    fmt::print(stderr, "{} -> {}, result {}, {}, new offset {}, twice {}, {}\n",
-//        _x, offset, x, o, -(offset - o), xx, oo);
     assert(xx == _x);
     assert(oo == 0);
 #endif
@@ -520,9 +510,6 @@ void WindowTransform::advanceFrameStartRowsOffset()
 
     assertValid(frame_start);
 
-//    fmt::print(stderr, "frame start {} left {} partition start {}\n",
-//        frame_start, offset_left, partition_start);
-
     if (frame_start <= partition_start)
     {
         // Got to the beginning of partition and can't go further back.
@@ -685,8 +672,6 @@ bool WindowTransform::arePeers(const RowNumber & x, const RowNumber & y) const
 
 void WindowTransform::advanceFrameEndCurrentRow()
 {
-//    fmt::print(stderr, "starting from frame_end {}\n", frame_end);
-
     // We only process one block here, and frame_end must be already in it: if
     // we didn't find the end in the previous block, frame_end is now the first
     // row of the current block. We need this knowledge to write a simpler loop
@@ -722,14 +707,11 @@ void WindowTransform::advanceFrameEndCurrentRow()
     // Equality would mean "no data to process", for which we checked above.
     assert(frame_end.row < rows_end);
 
-//    fmt::print(stderr, "first row {} last {}\n", frame_end.row, rows_end);
-
     // Advance frame_end while it is still peers with the current row.
     for (; frame_end.row < rows_end; ++frame_end.row)
     {
         if (!arePeers(current_row, frame_end))
         {
-//            fmt::print(stderr, "{} and {} don't match\n", reference, frame_end);
             frame_ended = true;
             return;
         }
@@ -852,8 +834,6 @@ void WindowTransform::advanceFrameEnd()
             break;
     }
 
-//    fmt::print(stderr, "frame_end {} -> {}\n", frame_end_before, frame_end);
-
     // We might not have advanced the frame end if we found out we reached the
     // end of input or the partition, or if we still don't know the frame start.
     if (frame_end_before == frame_end)
@@ -865,9 +845,6 @@ void WindowTransform::advanceFrameEnd()
 // Update the aggregation states after the frame has changed.
 void WindowTransform::updateAggregationState()
 {
-//    fmt::print(stderr, "update agg states [{}, {}) -> [{}, {})\n",
-//        prev_frame_start, prev_frame_end, frame_start, frame_end);
-
     // Assert that the frame boundaries are known, have proper order wrt each
     // other, and have not gone back wrt the previous frame.
     assert(frame_started);
@@ -915,7 +892,6 @@ void WindowTransform::updateAggregationState()
 
         if (reset_aggregation)
         {
-//            fmt::print(stderr, "(2) reset aggregation\n");
             a->destroy(buf);
             a->create(buf);
         }
@@ -991,9 +967,6 @@ void WindowTransform::writeOutCurrentRow()
             a->insertMergeResultInto(buf, *result_column, arena.get());
         }
     }
-
-//    fmt::print(stderr, "wrote out aggregation state for current row '{}'\n",
-//        current_row);
 }
 
 static void assertSameColumns(const Columns & left_all,
@@ -1030,10 +1003,6 @@ static void assertSameColumns(const Columns & left_all,
 
 void WindowTransform::appendChunk(Chunk & chunk)
 {
-//    fmt::print(stderr, "new chunk, {} rows, finished={}\n", chunk.getNumRows(),
-//        input_is_finished);
-//    fmt::print(stderr, "chunk structure '{}'\n", chunk.dumpStructure());
-
     // First, prepare the new input block and add it to the queue. We might not
     // have it if it's end of data, though.
     if (!input_is_finished)
@@ -1093,9 +1062,6 @@ void WindowTransform::appendChunk(Chunk & chunk)
     for (;;)
     {
         advancePartitionEnd();
-//        fmt::print(stderr, "partition [{}, {}), {}\n",
-//            partition_start, partition_end, partition_ended);
-
         // Either we ran out of data or we found the end of partition (maybe
         // both, but this only happens at the total end of data).
         assert(partition_ended || partition_end == blocksEnd());
@@ -1109,10 +1075,6 @@ void WindowTransform::appendChunk(Chunk & chunk)
         // which is precisely the definition of `partition_end`.
         while (current_row < partition_end)
         {
-//            fmt::print(stderr, "(1) row {} frame [{}, {}) {}, {}\n",
-//                current_row, frame_start, frame_end,
-//                frame_started, frame_ended);
-
             // We now know that the current row is valid, so we can update the
             // peer group start.
             if (!arePeers(peer_group_start, current_row))
@@ -1152,10 +1114,6 @@ void WindowTransform::appendChunk(Chunk & chunk)
                 return;
             }
 
-//            fmt::print(stderr, "(2) row {} frame [{}, {}) {}, {}\n",
-//                current_row, frame_start, frame_end,
-//                frame_started, frame_ended);
-
             // The frame can be empty sometimes, e.g. the boundaries coincide
             // or the start is after the partition end. But hopefully start is
             // not after end.
@@ -1236,8 +1194,6 @@ void WindowTransform::appendChunk(Chunk & chunk)
         peer_group_start_row_number = 1;
         peer_group_number = 1;
 
-//        fmt::print(stderr, "reinitialize agg data at start of {}\n",
-//            partition_start);
         // Reinitialize the aggregate function states because the new partition
         // has started.
         for (auto & ws : workspaces)
@@ -1278,10 +1234,6 @@ void WindowTransform::appendChunk(Chunk & chunk)
 
 IProcessor::Status WindowTransform::prepare()
 {
-//    fmt::print(stderr, "prepare, next output {}, not ready row {}, first block {}, hold {} blocks\n",
-//        next_output_block_number, first_not_ready_row, first_block_number,
-//        blocks.size());
-
     if (output.isFinished() || isCancelled())
     {
         // The consumer asked us not to continue (or we decided it ourselves),
@@ -1325,10 +1277,6 @@ IProcessor::Status WindowTransform::prepare()
             }
             output_data.chunk.setColumns(columns, block.rows);
 
-//            fmt::print(stderr, "output block {} as chunk '{}'\n",
-//                next_output_block_number,
-//                output_data.chunk.dumpStructure());
-
             ++next_output_block_number;
 
             output.pushData(std::move(output_data));
@@ -1428,9 +1376,6 @@ void WindowTransform::work()
         std::min(prev_frame_start.block, current_row.block));
     if (first_block_number < first_used_block)
     {
-//        fmt::print(stderr, "will drop blocks from {} to {}\n", first_block_number,
-//            first_used_block);
-
         blocks.erase(blocks.begin(),
             blocks.begin() + (first_used_block - first_block_number));
         first_block_number = first_used_block;

From 38800533483897b7ddb52f3447d52ab9dad2b082 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 23:02:41 +0200
Subject: [PATCH 283/689] Slightly better

---
 src/Processors/Transforms/WindowTransform.cpp | 30 ++++++++-
 src/Processors/Transforms/WindowTransform.h   | 67 +++++--------------
 2 files changed, 42 insertions(+), 55 deletions(-)

diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp
index daeafbca6e4..181cd1d52ff 100644
--- a/src/Processors/Transforms/WindowTransform.cpp
+++ b/src/Processors/Transforms/WindowTransform.cpp
@@ -19,6 +19,30 @@
 #include <DataTypes/DataTypeDateTime64.h>
 
 
+/// See https://fmt.dev/latest/api.html#formatting-user-defined-types
+template <>
+struct fmt::formatter<DB::RowNumber>
+{
+    static constexpr auto parse(format_parse_context & ctx)
+    {
+        const auto * it = ctx.begin();
+        const auto * end = ctx.end();
+
+        /// Only support {}.
+        if (it != end && *it != '}')
+            throw fmt::format_error("Invalid format");
+
+        return it;
+    }
+
+    template <typename FormatContext>
+    auto format(const DB::RowNumber & x, FormatContext & ctx)
+    {
+        return fmt::format_to(ctx.out(), "{}:{}", x.block, x.row);
+    }
+};
+
+
 namespace DB
 {
 
@@ -34,7 +58,7 @@ namespace ErrorCodes
 // Interface for true window functions. It's not much of an interface, they just
 // accept the guts of WindowTransform and do 'something'. Given a small number of
 // true window functions, and the fact that the WindowTransform internals are
-// pretty much well defined in domain terms (e.g. frame boundaries), this is
+// pretty much well-defined in domain terms (e.g. frame boundaries), this is
 // somewhat acceptable.
 class IWindowFunction
 {
@@ -483,9 +507,9 @@ auto WindowTransform::moveRowNumberNoCheck(const RowNumber & _x, int64_t offset)
     return std::tuple<RowNumber, int64_t>{x, offset};
 }
 
-auto WindowTransform::moveRowNumber(const RowNumber & _x, int64_t offset) const
+auto WindowTransform::moveRowNumber(const RowNumber & row_number, int64_t offset) const
 {
-    auto [x, o] = moveRowNumberNoCheck(_x, offset);
+    auto [x, o] = moveRowNumberNoCheck(row_number, offset);
 
 #ifndef NDEBUG
     // Check that it was reversible.
diff --git a/src/Processors/Transforms/WindowTransform.h b/src/Processors/Transforms/WindowTransform.h
index b9894052a96..7fad7f2d437 100644
--- a/src/Processors/Transforms/WindowTransform.h
+++ b/src/Processors/Transforms/WindowTransform.h
@@ -8,6 +8,10 @@
 
 #include <deque>
 
+/// See https://stackoverflow.com/questions/72533435/error-zero-as-null-pointer-constant-while-comparing-template-class-using-spaces
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
+
 
 namespace DB
 {
@@ -51,25 +55,11 @@ struct RowNumber
     uint64_t block = 0;
     uint64_t row = 0;
 
-    bool operator < (const RowNumber & other) const
-    {
-        return block < other.block
-            || (block == other.block && row < other.row);
-    }
-
-    bool operator == (const RowNumber & other) const
-    {
-        return block == other.block && row == other.row;
-    }
-
-    bool operator <= (const RowNumber & other) const
-    {
-        return *this < other || *this == other;
-    }
+    auto operator <=>(const RowNumber &) const = default;
 };
 
-/*
- * Computes several window functions that share the same window. The input must
+
+/* Computes several window functions that share the same window. The input must
  * be sorted by PARTITION BY (in any order), then by ORDER BY.
  * We need to track the following pointers:
  * 1) boundaries of partition -- rows that compare equal w/PARTITION BY.
@@ -188,7 +178,7 @@ public:
         const auto block_rows = blockAt(x).rows;
         assert(x.row < block_rows);
 
-        x.row++;
+        ++x.row;
         if (x.row < block_rows)
         {
             return;
@@ -237,20 +227,16 @@ public:
         return result;
     }
 
-    auto moveRowNumber(const RowNumber & _x, int64_t offset) const;
-    auto moveRowNumberNoCheck(const RowNumber & _x, int64_t offset) const;
+    auto moveRowNumber(const RowNumber & row_number, int64_t offset) const;
+    auto moveRowNumberNoCheck(const RowNumber & row_number, int64_t offset) const;
 
     void assertValid(const RowNumber & x) const
     {
         assert(x.block >= first_block_number);
         if (x.block == first_block_number + blocks.size())
-        {
             assert(x.row == 0);
-        }
         else
-        {
             assert(x.row < blockRowsNumber(x));
-        }
     }
 
     RowNumber blocksEnd() const
@@ -263,8 +249,7 @@ public:
         return RowNumber{first_block_number, 0};
     }
 
-    /*
-     * Data (formerly) inherited from ISimpleTransform, needed for the
+    /* Data (formerly) inherited from ISimpleTransform, needed for the
      * implementation of the IProcessor interface.
      */
     InputPort & input;
@@ -276,8 +261,7 @@ public:
     bool has_output = false;
     Port::Data output_data;
 
-    /*
-     * Data for window transform itself.
+    /* Data for window transform itself.
      */
     Block input_header;
 
@@ -353,34 +337,13 @@ public:
     // Comparison function for RANGE OFFSET frames. We choose the appropriate
     // overload once, based on the type of the ORDER BY column. Choosing it for
     // each row would be slow.
-    int (* compare_values_with_offset) (
+    std::function<int(
         const IColumn * compared_column, size_t compared_row,
         const IColumn * reference_column, size_t reference_row,
         const Field & offset,
-        bool offset_is_preceding);
+        bool offset_is_preceding)> compare_values_with_offset;
 };
 
 }
 
-/// See https://fmt.dev/latest/api.html#formatting-user-defined-types
-template <>
-struct fmt::formatter<DB::RowNumber>
-{
-    static constexpr auto parse(format_parse_context & ctx)
-    {
-        const auto * it = ctx.begin();
-        const auto * end = ctx.end();
-
-        /// Only support {}.
-        if (it != end && *it != '}')
-            throw fmt::format_error("invalid format");
-
-        return it;
-    }
-
-    template <typename FormatContext>
-    auto format(const DB::RowNumber & x, FormatContext & ctx)
-    {
-        return fmt::format_to(ctx.out(), "{}:{}", x.block, x.row);
-    }
-};
+#pragma clang diagnostic pop

From 68fcc39292ed6d515dba3af4ad25148a141ba3b7 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Sun, 7 May 2023 21:09:03 +0000
Subject: [PATCH 284/689] Fix: DISTINCT in order with zero values in non-sorted
 columns

---
 src/Common/HashTable/ClearableHashSet.h           |  6 ++++++
 ...2733_fix_distinct_in_order_bug_49622.reference |  2 ++
 .../02733_fix_distinct_in_order_bug_49622.sql     | 15 +++++++++++++++
 3 files changed, 23 insertions(+)
 create mode 100644 tests/queries/0_stateless/02733_fix_distinct_in_order_bug_49622.reference
 create mode 100644 tests/queries/0_stateless/02733_fix_distinct_in_order_bug_49622.sql

diff --git a/src/Common/HashTable/ClearableHashSet.h b/src/Common/HashTable/ClearableHashSet.h
index 4cbce1a5213..87d02f6e8ac 100644
--- a/src/Common/HashTable/ClearableHashSet.h
+++ b/src/Common/HashTable/ClearableHashSet.h
@@ -88,6 +88,9 @@ public:
     {
         ++this->version;
         this->m_size = 0;
+
+        /// clear ZeroValueStorage
+        this->clearHasZero();
     }
 };
 
@@ -108,6 +111,9 @@ public:
     {
         ++this->version;
         this->m_size = 0;
+
+        /// clear ZeroValueStorage
+        this->clearHasZero();
     }
 };
 
diff --git a/tests/queries/0_stateless/02733_fix_distinct_in_order_bug_49622.reference b/tests/queries/0_stateless/02733_fix_distinct_in_order_bug_49622.reference
new file mode 100644
index 00000000000..19f58f6ca91
--- /dev/null
+++ b/tests/queries/0_stateless/02733_fix_distinct_in_order_bug_49622.reference
@@ -0,0 +1,2 @@
+	1
+	2
diff --git a/tests/queries/0_stateless/02733_fix_distinct_in_order_bug_49622.sql b/tests/queries/0_stateless/02733_fix_distinct_in_order_bug_49622.sql
new file mode 100644
index 00000000000..b9953226c2a
--- /dev/null
+++ b/tests/queries/0_stateless/02733_fix_distinct_in_order_bug_49622.sql
@@ -0,0 +1,15 @@
+set optimize_distinct_in_order=1;
+
+DROP TABLE IF EXISTS test;
+
+CREATE TABLE test
+(
+    `c1` String,
+    `c2` String
+)
+ENGINE = MergeTree
+ORDER BY c1;
+
+INSERT INTO test(c1, c2) VALUES ('1',  ''), ('2', '');
+
+SELECT DISTINCT c2, c1 FROM test FORMAT TSV;

From aa280bcd100111667d4a04b897f6b9ccd129de8c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 7 May 2023 23:12:28 +0200
Subject: [PATCH 285/689] Slightly better

---
 src/Processors/Transforms/WindowTransform.cpp | 61 ++++++++++---------
 src/Processors/Transforms/WindowTransform.h   | 33 +++++-----
 2 files changed, 46 insertions(+), 48 deletions(-)

diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp
index 181cd1d52ff..a785d52bf65 100644
--- a/src/Processors/Transforms/WindowTransform.cpp
+++ b/src/Processors/Transforms/WindowTransform.cpp
@@ -437,26 +437,26 @@ void WindowTransform::advancePartitionEnd()
     assert(!partition_ended && partition_end == blocksEnd());
 }
 
-auto WindowTransform::moveRowNumberNoCheck(const RowNumber & _x, int64_t offset) const
+auto WindowTransform::moveRowNumberNoCheck(const RowNumber & original_row_number, Int64 offset) const
 {
-    RowNumber x = _x;
+    RowNumber moved_row_number = original_row_number;
 
-    if (offset > 0 && x != blocksEnd())
+    if (offset > 0 && moved_row_number != blocksEnd())
     {
         for (;;)
         {
-            assertValid(x);
+            assertValid(moved_row_number);
             assert(offset >= 0);
 
-            const auto block_rows = blockRowsNumber(x);
-            x.row += offset;
-            if (x.row >= block_rows)
+            const auto block_rows = blockRowsNumber(moved_row_number);
+            moved_row_number.row += offset;
+            if (moved_row_number.row >= block_rows)
             {
-                offset = x.row - block_rows;
-                x.row = 0;
-                x.block++;
+                offset = moved_row_number.row - block_rows;
+                moved_row_number.row = 0;
+                ++moved_row_number.block;
 
-                if (x == blocksEnd())
+                if (moved_row_number == blocksEnd())
                 {
                     break;
                 }
@@ -472,54 +472,55 @@ auto WindowTransform::moveRowNumberNoCheck(const RowNumber & _x, int64_t offset)
     {
         for (;;)
         {
-            assertValid(x);
+            assertValid(moved_row_number);
             assert(offset <= 0);
 
             // abs(offset) is less than INT64_MAX, as checked in the parser, so
             // this negation should always work.
             assert(offset >= -INT64_MAX);
-            if (x.row >= static_cast<uint64_t>(-offset))
+            if (moved_row_number.row >= static_cast<UInt64>(-offset))
             {
-                x.row -= -offset;
+                moved_row_number.row -= -offset;
                 offset = 0;
                 break;
             }
 
             // Move to the first row in current block. Note that the offset is
             // negative.
-            offset += x.row;
-            x.row = 0;
+            offset += moved_row_number.row;
+            moved_row_number.row = 0;
 
             // Move to the last row of the previous block, if we are not at the
             // first one. Offset also is incremented by one, because we pass over
             // the first row of this block.
-            if (x.block == first_block_number)
+            if (moved_row_number.block == first_block_number)
             {
                 break;
             }
 
-            --x.block;
+            --moved_row_number.block;
             offset += 1;
-            x.row = blockRowsNumber(x) - 1;
+            moved_row_number.row = blockRowsNumber(moved_row_number) - 1;
         }
     }
 
-    return std::tuple<RowNumber, int64_t>{x, offset};
+    return std::tuple<RowNumber, Int64>{moved_row_number, offset};
 }
 
-auto WindowTransform::moveRowNumber(const RowNumber & row_number, int64_t offset) const
+auto WindowTransform::moveRowNumber(const RowNumber & original_row_number, Int64 offset) const
 {
-    auto [x, o] = moveRowNumberNoCheck(row_number, offset);
+    auto [moved_row_number, offset_after_move] = moveRowNumberNoCheck(original_row_number, offset);
 
 #ifndef NDEBUG
-    // Check that it was reversible.
-    auto [xx, oo] = moveRowNumberNoCheck(x, -(offset - o));
+    /// Check that it was reversible. If we move back, we get the original row number with zero offset.
+    const auto [original_row_number_to_validate, offset_after_move_back]
+        = moveRowNumberNoCheck(moved_row_number, -(offset - offset_after_move));
 
-    assert(xx == _x);
-    assert(oo == 0);
+    assert(original_row_number_to_validate == original_row_number);
+    assert(0 == offset_after_move_back);
 #endif
 
-    return std::tuple<RowNumber, int64_t>{x, o};
+    return std::tuple<RowNumber, Int64>{moved_row_number, offset_after_move};
 }
 
 
@@ -717,7 +718,7 @@ void WindowTransform::advanceFrameEndCurrentRow()
     // We advance until the partition end. It's either in the current block or
     // in the next one, which is also the past-the-end block. Figure out how
     // many rows we have to process.
-    uint64_t rows_end;
+    UInt64 rows_end;
     if (partition_end.row == 0)
     {
         assert(partition_end == blocksEnd());
@@ -2165,7 +2166,7 @@ struct WindowFunctionLagLeadInFrame final : public WindowFunction
         IColumn & to = *current_block.output_columns[function_index];
         const auto & workspace = transform->workspaces[function_index];
 
-        int64_t offset = 1;
+        Int64 offset = 1;
         if (argument_types.size() > 1)
         {
             offset = (*current_block.input_columns[
@@ -2255,7 +2256,7 @@ struct WindowFunctionNthValue final : public WindowFunction
         IColumn & to = *current_block.output_columns[function_index];
         const auto & workspace = transform->workspaces[function_index];
 
-        int64_t offset = (*current_block.input_columns[
+        Int64 offset = (*current_block.input_columns[
                 workspace.argument_column_indices[1]])[
             transform->current_row.row].get<Int64>();
 
diff --git a/src/Processors/Transforms/WindowTransform.h b/src/Processors/Transforms/WindowTransform.h
index 7fad7f2d437..de3e82d15ee 100644
--- a/src/Processors/Transforms/WindowTransform.h
+++ b/src/Processors/Transforms/WindowTransform.h
@@ -38,7 +38,7 @@ struct WindowFunctionWorkspace
 
     // Argument columns. Be careful, this is a per-block cache.
     std::vector<const IColumn *> argument_columns;
-    uint64_t cached_block_number = std::numeric_limits<uint64_t>::max();
+    UInt64 cached_block_number = std::numeric_limits<UInt64>::max();
 };
 
 struct WindowTransformBlock
@@ -52,8 +52,8 @@ struct WindowTransformBlock
 
 struct RowNumber
 {
-    uint64_t block = 0;
-    uint64_t row = 0;
+    UInt64 block = 0;
+    UInt64 row = 0;
 
     auto operator <=>(const RowNumber &) const = default;
 };
@@ -93,19 +93,16 @@ public:
 
     static Block transformHeader(Block header, const ExpressionActionsPtr & expression);
 
-    /*
-     * (former) Implementation of ISimpleTransform.
+    /* (former) Implementation of ISimpleTransform.
      */
     void appendChunk(Chunk & chunk) /*override*/;
 
-    /*
-     * Implementation of IProcessor;
+    /* Implementation of IProcessor;
      */
     Status prepare() override;
     void work() override;
 
-    /*
-     * Implementation details.
+    /* Implementation details.
      */
     void advancePartitionEnd();
 
@@ -136,14 +133,14 @@ public:
         return const_cast<WindowTransform *>(this)->inputAt(x);
     }
 
-    auto & blockAt(const uint64_t block_number)
+    auto & blockAt(const UInt64 block_number)
     {
         assert(block_number >= first_block_number);
         assert(block_number - first_block_number < blocks.size());
         return blocks[block_number - first_block_number];
     }
 
-    const auto & blockAt(const uint64_t block_number) const
+    const auto & blockAt(const UInt64 block_number) const
     {
         return const_cast<WindowTransform *>(this)->blockAt(block_number);
     }
@@ -227,8 +224,8 @@ public:
         return result;
     }
 
-    auto moveRowNumber(const RowNumber & row_number, int64_t offset) const;
-    auto moveRowNumberNoCheck(const RowNumber & row_number, int64_t offset) const;
+    auto moveRowNumber(const RowNumber & original_row_number, Int64 offset) const;
+    auto moveRowNumberNoCheck(const RowNumber & original_row_number, Int64 offset) const;
 
     void assertValid(const RowNumber & x) const
     {
@@ -284,9 +281,9 @@ public:
     // have an always-incrementing index. The index of the first block is in
     // `first_block_number`.
     std::deque<WindowTransformBlock> blocks;
-    uint64_t first_block_number = 0;
+    UInt64 first_block_number = 0;
     // The next block we are going to pass to the consumer.
-    uint64_t next_output_block_number = 0;
+    UInt64 next_output_block_number = 0;
     // The first row for which we still haven't calculated the window functions.
     // Used to determine which resulting blocks we can pass to the consumer.
     RowNumber first_not_ready_row;
@@ -310,9 +307,9 @@ public:
     RowNumber peer_group_start;
 
     // Row and group numbers in partition for calculating rank() and friends.
-    uint64_t current_row_number = 1;
-    uint64_t peer_group_start_row_number = 1;
-    uint64_t peer_group_number = 1;
+    UInt64 current_row_number = 1;
+    UInt64 peer_group_start_row_number = 1;
+    UInt64 peer_group_number = 1;
 
     // The frame is [frame_start, frame_end) if frame_ended && frame_started,
     // and unknown otherwise. Note that when we move to the next row, both the

From ba2e827aa7e85856b6d812bbe002859062bb93c3 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Sun, 7 May 2023 21:31:01 +0000
Subject: [PATCH 286/689] Fix: check if zero storage has zeros before clean up

---
 src/Common/HashTable/ClearableHashSet.h                   | 6 ++++--
 .../0_stateless/02733_fix_distinct_in_order_bug_49622.sql | 8 ++++----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/Common/HashTable/ClearableHashSet.h b/src/Common/HashTable/ClearableHashSet.h
index 87d02f6e8ac..db6a4e487e6 100644
--- a/src/Common/HashTable/ClearableHashSet.h
+++ b/src/Common/HashTable/ClearableHashSet.h
@@ -90,7 +90,8 @@ public:
         this->m_size = 0;
 
         /// clear ZeroValueStorage
-        this->clearHasZero();
+        if (this->hasZero())
+            this->clearHasZero();
     }
 };
 
@@ -113,7 +114,8 @@ public:
         this->m_size = 0;
 
         /// clear ZeroValueStorage
-        this->clearHasZero();
+        if (this->hasZero())
+            this->clearHasZero();
     }
 };
 
diff --git a/tests/queries/0_stateless/02733_fix_distinct_in_order_bug_49622.sql b/tests/queries/0_stateless/02733_fix_distinct_in_order_bug_49622.sql
index b9953226c2a..9501a2c0761 100644
--- a/tests/queries/0_stateless/02733_fix_distinct_in_order_bug_49622.sql
+++ b/tests/queries/0_stateless/02733_fix_distinct_in_order_bug_49622.sql
@@ -1,8 +1,8 @@
 set optimize_distinct_in_order=1;
 
-DROP TABLE IF EXISTS test;
+DROP TABLE IF EXISTS test_string;
 
-CREATE TABLE test
+CREATE TABLE test_string
 (
     `c1` String,
     `c2` String
@@ -10,6 +10,6 @@ CREATE TABLE test
 ENGINE = MergeTree
 ORDER BY c1;
 
-INSERT INTO test(c1, c2) VALUES ('1',  ''), ('2', '');
+INSERT INTO test_string(c1, c2) VALUES ('1',  ''), ('2', '');
 
-SELECT DISTINCT c2, c1 FROM test FORMAT TSV;
+SELECT DISTINCT c2, c1 FROM test_string;

From fe02317d4595ddcea9837dc5a33f047242653642 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Sun, 7 May 2023 22:16:16 +0000
Subject: [PATCH 287/689] Clear zero value storage only if
 need_zero_value_storage

---
 src/Common/HashTable/ClearableHashSet.h | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/src/Common/HashTable/ClearableHashSet.h b/src/Common/HashTable/ClearableHashSet.h
index db6a4e487e6..006d45df7cd 100644
--- a/src/Common/HashTable/ClearableHashSet.h
+++ b/src/Common/HashTable/ClearableHashSet.h
@@ -80,6 +80,8 @@ template <
 class ClearableHashSet
     : public HashTable<Key, ClearableHashTableCell<Key, HashTableCell<Key, Hash, ClearableHashSetState>>, Hash, Grower, Allocator>
 {
+    using Cell = ClearableHashTableCell<Key, HashTableCell<Key, Hash, ClearableHashSetState>>;
+
 public:
     using Base = HashTable<Key, ClearableHashTableCell<Key, HashTableCell<Key, Hash, ClearableHashSetState>>, Hash, Grower, Allocator>;
     using typename Base::LookupResult;
@@ -89,9 +91,12 @@ public:
         ++this->version;
         this->m_size = 0;
 
-        /// clear ZeroValueStorage
-        if (this->hasZero())
-            this->clearHasZero();
+        if constexpr (Cell::need_zero_value_storage)
+        {
+            /// clear ZeroValueStorage
+            if (this->hasZero())
+                this->clearHasZero();
+        }
     }
 };
 
@@ -107,15 +112,20 @@ class ClearableHashSetWithSavedHash : public HashTable<
                                           Grower,
                                           Allocator>
 {
+    using Cell = ClearableHashTableCell<Key, HashSetCellWithSavedHash<Key, Hash, ClearableHashSetState>>;
+
 public:
     void clear()
     {
         ++this->version;
         this->m_size = 0;
 
-        /// clear ZeroValueStorage
-        if (this->hasZero())
-            this->clearHasZero();
+        if constexpr (Cell::need_zero_value_storage)
+        {
+            /// clear ZeroValueStorage
+            if (this->hasZero())
+                this->clearHasZero();
+        }
     }
 };
 

From 1224ac9eda1437f09d1f7eaf649451e3ed03d844 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Mon, 8 May 2023 00:57:13 +0200
Subject: [PATCH 288/689] fix build

---
 src/Databases/DDLDependencyVisitor.cpp        |  2 +-
 .../removeWhereConditionPlaceholder.cpp       | 24 +++++++++++++++++++
 .../removeWhereConditionPlaceholder.h         | 10 ++++++++
 src/Dictionaries/ExternalQueryBuilder.cpp     | 21 +---------------
 src/Dictionaries/ExternalQueryBuilder.h       |  1 -
 5 files changed, 36 insertions(+), 22 deletions(-)
 create mode 100644 src/Databases/removeWhereConditionPlaceholder.cpp
 create mode 100644 src/Databases/removeWhereConditionPlaceholder.h

diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp
index fc0cda03484..cb85119e3b0 100644
--- a/src/Databases/DDLDependencyVisitor.cpp
+++ b/src/Databases/DDLDependencyVisitor.cpp
@@ -1,6 +1,6 @@
 #include <Databases/DDLDependencyVisitor.h>
 #include <Dictionaries/getDictionaryConfigurationFromAST.h>
-#include <Dictionaries/ExternalQueryBuilder.h>
+#include <Databases/removeWhereConditionPlaceholder.h>
 #include <Interpreters/Cluster.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/misc.h>
diff --git a/src/Databases/removeWhereConditionPlaceholder.cpp b/src/Databases/removeWhereConditionPlaceholder.cpp
new file mode 100644
index 00000000000..efe6a5313df
--- /dev/null
+++ b/src/Databases/removeWhereConditionPlaceholder.cpp
@@ -0,0 +1,24 @@
+#include <Databases/removeWhereConditionPlaceholder.h>
+
+namespace DB
+{
+
+std::string removeWhereConditionPlaceholder(const std::string  & query)
+{
+    /** In case UPDATE_FIELD is specified in {condition} for dictionary that must load all data.
+      * Replace {condition} with true_condition for initial dictionary load.
+      * For next dictionary loads {condition} will be updated with UPDATE_FIELD.
+      */
+    static constexpr auto true_condition = "(1 = 1)";
+    auto condition_position = query.find(CONDITION_PLACEHOLDER_TO_REPLACE_VALUE);
+    if (condition_position != std::string::npos)
+    {
+        auto query_copy = query;
+        query_copy.replace(condition_position, CONDITION_PLACEHOLDER_TO_REPLACE_VALUE.size(), true_condition);
+        return query_copy;
+    }
+
+    return query;
+}
+
+}
diff --git a/src/Databases/removeWhereConditionPlaceholder.h b/src/Databases/removeWhereConditionPlaceholder.h
new file mode 100644
index 00000000000..90d849c45fa
--- /dev/null
+++ b/src/Databases/removeWhereConditionPlaceholder.h
@@ -0,0 +1,10 @@
+#include <string>
+
+namespace DB
+{
+
+static constexpr std::string_view CONDITION_PLACEHOLDER_TO_REPLACE_VALUE = "{condition}";
+
+std::string removeWhereConditionPlaceholder(const std::string & query);
+
+}
diff --git a/src/Dictionaries/ExternalQueryBuilder.cpp b/src/Dictionaries/ExternalQueryBuilder.cpp
index 05eaf1576b9..e21b0842e11 100644
--- a/src/Dictionaries/ExternalQueryBuilder.cpp
+++ b/src/Dictionaries/ExternalQueryBuilder.cpp
@@ -6,7 +6,7 @@
 #include <IO/WriteBufferFromString.h>
 #include <IO/WriteHelpers.h>
 #include <Dictionaries/DictionaryStructure.h>
-
+#include <Databases/removeWhereConditionPlaceholder.h>
 
 namespace DB
 {
@@ -24,7 +24,6 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-static constexpr std::string_view CONDITION_PLACEHOLDER_TO_REPLACE_VALUE = "{condition}";
 
 ExternalQueryBuilder::ExternalQueryBuilder(
     const DictionaryStructure & dict_struct_,
@@ -86,24 +85,6 @@ std::string ExternalQueryBuilder::composeLoadAllQuery() const
     return removeWhereConditionPlaceholder(query);
 }
 
-String removeWhereConditionPlaceholder(const String  & query)
-{
-    /** In case UPDATE_FIELD is specified in {condition} for dictionary that must load all data.
-      * Replace {condition} with true_condition for initial dictionary load.
-      * For next dictionary loads {condition} will be updated with UPDATE_FIELD.
-      */
-    static constexpr auto true_condition = "(1 = 1)";
-    auto condition_position = query.find(CONDITION_PLACEHOLDER_TO_REPLACE_VALUE);
-    if (condition_position != std::string::npos)
-    {
-        auto query_copy = query;
-        query_copy.replace(condition_position, CONDITION_PLACEHOLDER_TO_REPLACE_VALUE.size(), true_condition);
-        return query_copy;
-    }
-
-    return query;
-}
-
 void ExternalQueryBuilder::composeLoadAllQuery(WriteBuffer & out) const
 {
     writeString("SELECT ", out);
diff --git a/src/Dictionaries/ExternalQueryBuilder.h b/src/Dictionaries/ExternalQueryBuilder.h
index 9521019d51e..353367b12b6 100644
--- a/src/Dictionaries/ExternalQueryBuilder.h
+++ b/src/Dictionaries/ExternalQueryBuilder.h
@@ -12,7 +12,6 @@ namespace DB
 struct DictionaryStructure;
 class WriteBuffer;
 
-String removeWhereConditionPlaceholder(const String  & query);
 
 /** Builds a query to load data from external database.
   */

From 6f8455efcb366199caf8a08355979fd07a6d94eb Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Mon, 8 May 2023 02:26:29 +0300
Subject: [PATCH 289/689] Update removeWhereConditionPlaceholder.h

---
 src/Databases/removeWhereConditionPlaceholder.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Databases/removeWhereConditionPlaceholder.h b/src/Databases/removeWhereConditionPlaceholder.h
index 90d849c45fa..8a4da3cb9f2 100644
--- a/src/Databases/removeWhereConditionPlaceholder.h
+++ b/src/Databases/removeWhereConditionPlaceholder.h
@@ -1,3 +1,4 @@
+#pragma once
 #include <string>
 
 namespace DB

From 900f8e50e36c4db1b6012a5d86cead664495a7fe Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 8 May 2023 04:08:13 +0200
Subject: [PATCH 290/689] Fix one-off error found by UBSan

---
 base/base/wide_integer_impl.h                            | 9 ++++++++-
 .../0_stateless/02734_big_int_from_float_ubsan.reference | 1 +
 .../0_stateless/02734_big_int_from_float_ubsan.sql       | 1 +
 3 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/02734_big_int_from_float_ubsan.reference
 create mode 100644 tests/queries/0_stateless/02734_big_int_from_float_ubsan.sql

diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h
index ed4570d5e3f..4a80c176829 100644
--- a/base/base/wide_integer_impl.h
+++ b/base/base/wide_integer_impl.h
@@ -314,7 +314,14 @@ struct integer<Bits, Signed>::_impl
 
         const T alpha = t / static_cast<T>(max_int);
 
-        if (alpha <= static_cast<T>(max_int))
+        /** Here we have to use strict comparison.
+          * The max_int is 2^64 - 1.
+          * When casted to floating point type, it will be rounded to the closest representable number,
+          * which is 2^64.
+          * But 2^64 is not representable in uint64_t,
+          * so the maximum representable number will be strictly less.
+          */
+        if (alpha < static_cast<T>(max_int))
             self = static_cast<uint64_t>(alpha);
         else // max(double) / 2^64 will surely contain less than 52 precision bits, so speed up computations.
             set_multiplier<double>(self, static_cast<double>(alpha));
diff --git a/tests/queries/0_stateless/02734_big_int_from_float_ubsan.reference b/tests/queries/0_stateless/02734_big_int_from_float_ubsan.reference
new file mode 100644
index 00000000000..d00491fd7e5
--- /dev/null
+++ b/tests/queries/0_stateless/02734_big_int_from_float_ubsan.reference
@@ -0,0 +1 @@
+1
diff --git a/tests/queries/0_stateless/02734_big_int_from_float_ubsan.sql b/tests/queries/0_stateless/02734_big_int_from_float_ubsan.sql
new file mode 100644
index 00000000000..ebe1394bb55
--- /dev/null
+++ b/tests/queries/0_stateless/02734_big_int_from_float_ubsan.sql
@@ -0,0 +1 @@
+WITH 18 AS precision, toUInt256(-1) AS int, toUInt256(toFloat64(toUInt256(-1))) AS converted SELECT substring(toString(int), 1, precision) = substring(toString(converted), 1, precision) AS res;

From 72573151b6789aeaeb2b4e8774738680e2780306 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 8 May 2023 04:51:35 +0200
Subject: [PATCH 291/689] Revert "fix flaky test"

This reverts commit c5e0869c46663dc49c87a5d8e45386f16132ff86.
---
 tests/queries/0_stateless/01592_long_window_functions1.sql | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/queries/0_stateless/01592_long_window_functions1.sql b/tests/queries/0_stateless/01592_long_window_functions1.sql
index 022d8071ffa..4911b7aa792 100644
--- a/tests/queries/0_stateless/01592_long_window_functions1.sql
+++ b/tests/queries/0_stateless/01592_long_window_functions1.sql
@@ -4,10 +4,6 @@ drop table if exists stack;
 
 set max_insert_threads = 4;
 
--- Temporary disable aggregation in order,
--- because it may fail with UBSan.
-set optimize_aggregation_in_order = 0;
-
 create table stack(item_id Int64, brand_id Int64, rack_id Int64, dt DateTime, expiration_dt DateTime, quantity UInt64)
 Engine = MergeTree 
 partition by toYYYYMM(dt) 

From baf162b26b1e87a975a99ffd77990d157cff86fe Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 8 May 2023 05:15:28 +0200
Subject: [PATCH 292/689] Optimize opentelemetry_span_log

---
 src/Interpreters/OpenTelemetrySpanLog.cpp     | 20 +++++++++----------
 .../FinishAggregatingInOrderAlgorithm.cpp     |  1 -
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp
index 257c64e1df8..610330b8559 100644
--- a/src/Interpreters/OpenTelemetrySpanLog.cpp
+++ b/src/Interpreters/OpenTelemetrySpanLog.cpp
@@ -1,19 +1,14 @@
-#include "OpenTelemetrySpanLog.h"
+#include <Interpreters/OpenTelemetrySpanLog.h>
 
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeDate.h>
 #include <DataTypes/DataTypeDateTime.h>
-#include <DataTypes/DataTypeDateTime64.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeLowCardinality.h>
 #include <DataTypes/DataTypeMap.h>
 #include <DataTypes/DataTypeUUID.h>
 #include <DataTypes/DataTypeEnum.h>
-#include <Interpreters/Context.h>
-
-#include <base/hex.h>
-#include <Common/CurrentThread.h>
-#include <Core/Field.h>
 
 
 namespace DB
@@ -32,11 +27,13 @@ NamesAndTypesList OpenTelemetrySpanLogElement::getNamesAndTypes()
         }
     );
 
+    auto low_cardinality_string = std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>());
+
     return {
         {"trace_id", std::make_shared<DataTypeUUID>()},
         {"span_id", std::make_shared<DataTypeUInt64>()},
         {"parent_span_id", std::make_shared<DataTypeUInt64>()},
-        {"operation_name", std::make_shared<DataTypeString>()},
+        {"operation_name", low_cardinality_string},
         {"kind", std::move(span_kind_type)},
         // DateTime64 is really unwieldy -- there is no "normal" way to convert
         // it to an UInt64 count of microseconds, except:
@@ -51,15 +48,17 @@ NamesAndTypesList OpenTelemetrySpanLogElement::getNamesAndTypes()
         {"start_time_us", std::make_shared<DataTypeUInt64>()},
         {"finish_time_us", std::make_shared<DataTypeUInt64>()},
         {"finish_date", std::make_shared<DataTypeDate>()},
-        {"attribute", std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(), std::make_shared<DataTypeString>())},
+        {"attribute", std::make_shared<DataTypeMap>(low_cardinality_string, std::make_shared<DataTypeString>())},
     };
 }
 
 NamesAndAliases OpenTelemetrySpanLogElement::getNamesAndAliases()
 {
+    auto low_cardinality_string = std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>());
+
     return
     {
-        {"attribute.names", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "mapKeys(attribute)"},
+        {"attribute.names", std::make_shared<DataTypeArray>(low_cardinality_string), "mapKeys(attribute)"},
         {"attribute.values", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "mapValues(attribute)"}
     };
 }
@@ -83,4 +82,3 @@ void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const
 }
 
 }
-
diff --git a/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp b/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp
index 5f9725c804b..a5befca7233 100644
--- a/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp
+++ b/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp
@@ -168,7 +168,6 @@ void FinishAggregatingInOrderAlgorithm::addToAggregation()
         accumulated_bytes += static_cast<size_t>(static_cast<double>(states[i].total_bytes) * current_rows / states[i].num_rows);
         accumulated_rows += current_rows;
 
-
         if (!states[i].isValid())
             inputs_to_update.push_back(i);
     }

From 88d0d94933cf1b4d9e31a297c15607f20d1ed758 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 8 May 2023 07:02:41 +0200
Subject: [PATCH 293/689] Update test

---
 .../test_vertical_merges_from_compact_parts.py               | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
index 5979ad71a2e..3d006caad0d 100644
--- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
+++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py
@@ -111,7 +111,10 @@ def test_vertical_merges_from_compact_parts(start_cluster):
             node_old.contains_in_log("CHECKSUM_DOESNT_MATCH")
             and not node_old.contains_in_log("Different number of files")
         )
-        or (node_new.contains_in_log("CHECKSUM_DOESNT_MATCH"))
+        or (
+            node_new.contains_in_log("CHECKSUM_DOESNT_MATCH")
+            and not node_new.contains_in_log("Different number of files")
+        )
     )
 
     assert node_new.query(check_query.format("all_0_3_3")) == "Vertical\tWide\n"

From a046d8fd4ce44be3f7604b73f9f027e1720d230d Mon Sep 17 00:00:00 2001
From: Vincent <vrfn@users.noreply.github.com>
Date: Mon, 8 May 2023 08:18:52 +0000
Subject: [PATCH 294/689] Reflect that BACKUP now supports ALL in docs

---
 docs/en/operations/backup.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md
index a31a52f509e..6da61833c12 100644
--- a/docs/en/operations/backup.md
+++ b/docs/en/operations/backup.md
@@ -30,7 +30,7 @@ description: In order to effectively mitigate possible human errors, you should
 ```
 
 :::note ALL
-`ALL` is only applicable to the `RESTORE` command.
+`ALL` is only applicable to the `RESTORE` command prior to version 23.4 of Clickhouse.
 :::
 
 ## Background

From 9fad9f234fce73bd5854e7d8a7ffffe6d2cca88a Mon Sep 17 00:00:00 2001
From: jinjunzh <jasper.zhu@intel.com>
Date: Mon, 8 May 2023 15:20:55 -0400
Subject: [PATCH 295/689] add missing idxd-config module for qpl-deflate codec

---
 .gitmodules                                   |  3 ++
 contrib/CMakeLists.txt                        |  1 +
 contrib/idxd-config                           |  1 +
 contrib/idxd-config-cmake/CMakeLists.txt      | 36 +++++++++++++++++++
 .../include}/config.h                         |  0
 contrib/qpl-cmake/CMakeLists.txt              | 27 +++-----------
 src/CMakeLists.txt                            |  4 +++
 7 files changed, 49 insertions(+), 23 deletions(-)
 create mode 160000 contrib/idxd-config
 create mode 100644 contrib/idxd-config-cmake/CMakeLists.txt
 rename contrib/{qpl-cmake/idxd-header => idxd-config-cmake/include}/config.h (100%)

diff --git a/.gitmodules b/.gitmodules
index ed61ddb96ba..6e63474e0f1 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -338,3 +338,6 @@
 [submodule "contrib/isa-l"]
 	path = contrib/isa-l
 	url = https://github.com/ClickHouse/isa-l.git
+[submodule "contrib/idxd-config"]
+	path = contrib/idxd-config
+	url = https://github.com/intel/idxd-config
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 57ce93d45f7..59087d21bb3 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -178,6 +178,7 @@ add_contrib (sqlite-cmake sqlite-amalgamation)
 add_contrib (s2geometry-cmake s2geometry)
 add_contrib (c-ares-cmake c-ares)
 add_contrib (qpl-cmake qpl)
+add_contrib (idxd-config-cmake idxd-config)
 add_contrib (morton-nd-cmake morton-nd)
 if (ARCH_S390X)
     add_contrib(crc32-s390x-cmake crc32-s390x)
diff --git a/contrib/idxd-config b/contrib/idxd-config
new file mode 160000
index 00000000000..a836ce0e420
--- /dev/null
+++ b/contrib/idxd-config
@@ -0,0 +1 @@
+Subproject commit a836ce0e42052a69bffbbc14239ab4097f3b77f1
diff --git a/contrib/idxd-config-cmake/CMakeLists.txt b/contrib/idxd-config-cmake/CMakeLists.txt
new file mode 100644
index 00000000000..f70644d6fea
--- /dev/null
+++ b/contrib/idxd-config-cmake/CMakeLists.txt
@@ -0,0 +1,36 @@
+## accel_config is the utility library required by QPL-Deflate codec for controlling and configuring Intel® In-Memory Analytics Accelerator (Intel® IAA).
+option(ENABLE_ACCEL_CONFIG "Enable accel-config library" ${ENABLE_LIBRARIES})
+
+if (NOT ENABLE_QPL)
+    # Disable accel-config aligned with QPL.
+    set (ENABLE_ACCEL_CONFIG OFF)
+endif ()
+if (NOT ENABLE_ACCEL_CONFIG)
+    message(STATUS "Not using accel-config")
+    return()
+else()
+    message(STATUS "Enable accel-config")
+endif()
+
+set (LIBACCEL_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/idxd-config")
+set (UUID_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl-cmake")
+set (LIBACCEL_HEADER_DIR "${ClickHouse_SOURCE_DIR}/contrib/idxd-config-cmake/include")
+set (SRCS
+    "${LIBACCEL_SOURCE_DIR}/accfg/lib/libaccfg.c"
+    "${LIBACCEL_SOURCE_DIR}/util/log.c"
+    "${LIBACCEL_SOURCE_DIR}/util/sysfs.c"
+)
+
+add_library(_accel-config ${SRCS})
+
+target_compile_options(_accel-config PRIVATE "-D_GNU_SOURCE")
+
+target_include_directories(_accel-config BEFORE
+    PRIVATE ${UUID_DIR}
+    PRIVATE ${LIBACCEL_HEADER_DIR}
+    PRIVATE ${LIBACCEL_SOURCE_DIR})
+
+target_include_directories(_accel-config SYSTEM BEFORE
+    PUBLIC ${LIBACCEL_SOURCE_DIR}/accfg)
+
+add_library(ch_contrib::accel-config ALIAS _accel-config)
diff --git a/contrib/qpl-cmake/idxd-header/config.h b/contrib/idxd-config-cmake/include/config.h
similarity index 100%
rename from contrib/qpl-cmake/idxd-header/config.h
rename to contrib/idxd-config-cmake/include/config.h
diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt
index d2be8add3c7..3b8c2628ec8 100644
--- a/contrib/qpl-cmake/CMakeLists.txt
+++ b/contrib/qpl-cmake/CMakeLists.txt
@@ -10,27 +10,8 @@ if (NOT ENABLE_QPL)
     return()
 endif()
 
-## QPL has build dependency on libaccel-config. Here is to build libaccel-config which is required by QPL.
-## libaccel-config is the utility library for controlling and configuring Intel® In-Memory Analytics Accelerator (Intel® IAA).
-set (LIBACCEL_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/idxd-config")
-set (UUID_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl-cmake")
-set (LIBACCEL_HEADER_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl-cmake/idxd-header")
-set (SRCS
-    "${LIBACCEL_SOURCE_DIR}/accfg/lib/libaccfg.c"
-    "${LIBACCEL_SOURCE_DIR}/util/log.c"
-    "${LIBACCEL_SOURCE_DIR}/util/sysfs.c"
-)
-
-add_library(accel-config ${SRCS})
-
-target_compile_options(accel-config PRIVATE "-D_GNU_SOURCE")
-
-target_include_directories(accel-config BEFORE
-    PRIVATE ${UUID_DIR}
-    PRIVATE ${LIBACCEL_HEADER_DIR}
-    PRIVATE ${LIBACCEL_SOURCE_DIR})
-
 ## QPL build start here.
+set (UUID_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl-cmake")
 set (QPL_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl")
 set (QPL_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl/sources")
 set (QPL_BINARY_DIR "${ClickHouse_BINARY_DIR}/build/contrib/qpl")
@@ -342,12 +323,12 @@ target_compile_definitions(_qpl
         PUBLIC -DENABLE_QPL_COMPRESSION)
 
 target_link_libraries(_qpl
-        PRIVATE accel-config
+        PRIVATE ch_contrib::accel-config
         PRIVATE ch_contrib::isal
         PRIVATE ${CMAKE_DL_LIBS})
 
-add_library (ch_contrib::qpl ALIAS _qpl)
 target_include_directories(_qpl SYSTEM BEFORE
         PUBLIC "${QPL_PROJECT_DIR}/include"
-        PUBLIC "${LIBACCEL_SOURCE_DIR}/accfg"
         PUBLIC ${UUID_DIR})
+
+add_library (ch_contrib::qpl ALIAS _qpl)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 76a67ade99c..b22c2546aef 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -544,6 +544,10 @@ if (TARGET ch_contrib::qpl)
 dbms_target_link_libraries(PUBLIC ch_contrib::qpl)
 endif ()
 
+if (TARGET ch_contrib::accel-config)
+    dbms_target_link_libraries(PUBLIC ch_contrib::accel-config)
+endif ()
+
 target_link_libraries(clickhouse_common_io PUBLIC boost::context)
 dbms_target_link_libraries(PUBLIC boost::context)
 

From 17cb445c1b21fa928b87fb1cd1b399aca9f09f4f Mon Sep 17 00:00:00 2001
From: jinjunzh <jasper.zhu@intel.com>
Date: Mon, 8 May 2023 16:43:37 -0400
Subject: [PATCH 296/689] refine content of cmake for qpl

---
 .gitmodules                              |  3 ---
 contrib/CMakeLists.txt                   | 15 +++++++++++++--
 contrib/idxd-config-cmake/CMakeLists.txt | 13 -------------
 contrib/qpl-cmake/CMakeLists.txt         | 12 ------------
 4 files changed, 13 insertions(+), 30 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index 6e63474e0f1..ed61ddb96ba 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -338,6 +338,3 @@
 [submodule "contrib/isa-l"]
 	path = contrib/isa-l
 	url = https://github.com/ClickHouse/isa-l.git
-[submodule "contrib/idxd-config"]
-	path = contrib/idxd-config
-	url = https://github.com/intel/idxd-config
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 59087d21bb3..901df1b2432 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -177,8 +177,19 @@ endif()
 add_contrib (sqlite-cmake sqlite-amalgamation)
 add_contrib (s2geometry-cmake s2geometry)
 add_contrib (c-ares-cmake c-ares)
-add_contrib (qpl-cmake qpl)
-add_contrib (idxd-config-cmake idxd-config)
+
+if (OS_LINUX AND ARCH_AMD64 AND (ENABLE_AVX2 OR ENABLE_AVX512))
+    option (ENABLE_QPL "Enable Intel® Query Processing Library" ${ENABLE_LIBRARIES})
+elseif(ENABLE_QPL)
+    message (${RECONFIGURE_MESSAGE_LEVEL} "QPL library is only supported on x86_64 arch with avx2/avx512 support")
+endif()
+if (ENABLE_QPL)
+    add_contrib (idxd-config-cmake idxd-config)
+    add_contrib (qpl-cmake qpl) # requires: idxd-config
+else()
+    message(STATUS "Not using QPL")
+endif ()
+
 add_contrib (morton-nd-cmake morton-nd)
 if (ARCH_S390X)
     add_contrib(crc32-s390x-cmake crc32-s390x)
diff --git a/contrib/idxd-config-cmake/CMakeLists.txt b/contrib/idxd-config-cmake/CMakeLists.txt
index f70644d6fea..030252ec8e6 100644
--- a/contrib/idxd-config-cmake/CMakeLists.txt
+++ b/contrib/idxd-config-cmake/CMakeLists.txt
@@ -1,17 +1,4 @@
 ## accel_config is the utility library required by QPL-Deflate codec for controlling and configuring Intel® In-Memory Analytics Accelerator (Intel® IAA).
-option(ENABLE_ACCEL_CONFIG "Enable accel-config library" ${ENABLE_LIBRARIES})
-
-if (NOT ENABLE_QPL)
-    # Disable accel-config aligned with QPL.
-    set (ENABLE_ACCEL_CONFIG OFF)
-endif ()
-if (NOT ENABLE_ACCEL_CONFIG)
-    message(STATUS "Not using accel-config")
-    return()
-else()
-    message(STATUS "Enable accel-config")
-endif()
-
 set (LIBACCEL_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/idxd-config")
 set (UUID_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl-cmake")
 set (LIBACCEL_HEADER_DIR "${ClickHouse_SOURCE_DIR}/contrib/idxd-config-cmake/include")
diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt
index 3b8c2628ec8..034106b51b6 100644
--- a/contrib/qpl-cmake/CMakeLists.txt
+++ b/contrib/qpl-cmake/CMakeLists.txt
@@ -1,16 +1,4 @@
 ## The Intel® QPL provides high performance implementations of data processing functions for existing hardware accelerator, and/or software path in case if hardware accelerator is not available.
-if (OS_LINUX AND ARCH_AMD64 AND (ENABLE_AVX2 OR ENABLE_AVX512))
-    option (ENABLE_QPL "Enable Intel® Query Processing Library" ${ENABLE_LIBRARIES})
-elseif(ENABLE_QPL)
-    message (${RECONFIGURE_MESSAGE_LEVEL} "QPL library is only supported on x86_64 arch with avx2/avx512 support")
-endif()
-
-if (NOT ENABLE_QPL)
-    message(STATUS "Not using QPL")
-    return()
-endif()
-
-## QPL build start here.
 set (UUID_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl-cmake")
 set (QPL_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl")
 set (QPL_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl/sources")

From d2d291fe893b42603598ade04bc081494e5def7d Mon Sep 17 00:00:00 2001
From: jinjunzh <jasper.zhu@intel.com>
Date: Mon, 8 May 2023 16:46:27 -0400
Subject: [PATCH 297/689] resolved conflict with master

---
 contrib/idxd-config | 1 -
 1 file changed, 1 deletion(-)
 delete mode 160000 contrib/idxd-config

diff --git a/contrib/idxd-config b/contrib/idxd-config
deleted file mode 160000
index a836ce0e420..00000000000
--- a/contrib/idxd-config
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit a836ce0e42052a69bffbbc14239ab4097f3b77f1

From fce2e089c695dde657d65c303b452aa344ea3990 Mon Sep 17 00:00:00 2001
From: sanjam <sanjamkumar360@gmail.com>
Date: Mon, 8 May 2023 10:49:25 +0000
Subject: [PATCH 298/689] temp fix for t64 on s390x

---
 src/Compression/CompressionCodecT64.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/Compression/CompressionCodecT64.cpp b/src/Compression/CompressionCodecT64.cpp
index d5bace020be..3506c087b54 100644
--- a/src/Compression/CompressionCodecT64.cpp
+++ b/src/Compression/CompressionCodecT64.cpp
@@ -378,6 +378,13 @@ void transpose(const T * src, char * dst, UInt32 num_bits, UInt32 tail = 64)
 
 /// UInt64[N] transposed matrix -> UIntX[64]
 template <typename T, bool full = false>
+#if defined(__s390x__)
+
+/* Compiler Bug for S390x :- https://github.com/llvm/llvm-project/issues/62572
+ * Please remove this after the fix is backported
+ */
+        __attribute__((noinline))
+#endif
 void reverseTranspose(const char * src, T * buf, UInt32 num_bits, UInt32 tail = 64)
 {
     UInt64 matrix[64] = {};

From b5af1e10cbfb85bb76383d9aa8bdf09547e2bbe3 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Mon, 8 May 2023 13:37:03 +0200
Subject: [PATCH 299/689] Update getThreadPoolReader.cpp

---
 src/Disks/IO/getThreadPoolReader.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/Disks/IO/getThreadPoolReader.cpp b/src/Disks/IO/getThreadPoolReader.cpp
index a28501f4dbd..deb8f66106c 100644
--- a/src/Disks/IO/getThreadPoolReader.cpp
+++ b/src/Disks/IO/getThreadPoolReader.cpp
@@ -21,12 +21,7 @@ namespace ErrorCodes
 
 IAsynchronousReader & getThreadPoolReader(FilesystemReaderType type)
 {
-#ifndef CLICKHOUSE_PROGRAM_STANDALONE_BUILD
-    auto context = Context::getGlobalContextInstance();
-    if (!context)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context not initialized");
-    return context->getThreadPoolReader(type);
-#else
+#ifdef CLICKHOUSE_PROGRAM_STANDALONE_BUILD
     const auto & config = Poco::Util::Application::instance().config();
     switch (type)
     {
@@ -46,6 +41,11 @@ IAsynchronousReader & getThreadPoolReader(FilesystemReaderType type)
             return *synchronous_local_fs_reader;
         }
     }
+#else
+    auto context = Context::getGlobalContextInstance();
+    if (!context)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context not initialized");
+    return context->getThreadPoolReader(type);
 #endif
 }
 

From 0612f5d7820e70921748353efcea600cab0581ab Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Mon, 8 May 2023 11:50:21 +0000
Subject: [PATCH 300/689] fix sparse columns after reload

---
 .../Serializations/SerializationInfo.cpp      | 23 +++++++++++++++----
 .../Serializations/SerializationInfo.h        |  8 +++++--
 src/Storages/MergeTree/IMergeTreeDataPart.cpp |  4 ++--
 src/Storages/MergeTree/checkDataPart.cpp      |  5 ++--
 .../02733_sparse_columns_reload.reference     |  2 ++
 .../02733_sparse_columns_reload.sql           | 18 +++++++++++++++
 6 files changed, 50 insertions(+), 10 deletions(-)
 create mode 100644 tests/queries/0_stateless/02733_sparse_columns_reload.reference
 create mode 100644 tests/queries/0_stateless/02733_sparse_columns_reload.sql

diff --git a/src/DataTypes/Serializations/SerializationInfo.cpp b/src/DataTypes/Serializations/SerializationInfo.cpp
index 4e5790ad58d..4e9b9905454 100644
--- a/src/DataTypes/Serializations/SerializationInfo.cpp
+++ b/src/DataTypes/Serializations/SerializationInfo.cpp
@@ -246,7 +246,8 @@ void SerializationInfoByName::writeJSON(WriteBuffer & out) const
     return writeString(oss.str(), out);
 }
 
-void SerializationInfoByName::readJSON(ReadBuffer & in)
+SerializationInfoByName SerializationInfoByName::readJSON(
+    const NamesAndTypesList & columns, const Settings & settings, ReadBuffer & in)
 {
     String json_str;
     readString(json_str, in);
@@ -262,8 +263,13 @@ void SerializationInfoByName::readJSON(ReadBuffer & in)
             "Unknown version of serialization infos ({}). Should be less or equal than {}",
             object->getValue<size_t>(KEY_VERSION), SERIALIZATION_INFO_VERSION);
 
+    SerializationInfoByName infos;
     if (object->has(KEY_COLUMNS))
     {
+        std::unordered_map<std::string_view, const IDataType *> column_type_by_name;
+        for (const auto & [name, type] : columns)
+            column_type_by_name.emplace(name, type.get());
+
         auto array = object->getArray(KEY_COLUMNS);
         for (const auto & elem : *array)
         {
@@ -271,13 +277,22 @@ void SerializationInfoByName::readJSON(ReadBuffer & in)
 
             if (!elem_object->has(KEY_NAME))
                 throw Exception(ErrorCodes::CORRUPTED_DATA,
-                    "Missed field '{}' in SerializationInfo of columns", KEY_NAME);
+                    "Missed field '{}' in serialization infos", KEY_NAME);
 
             auto name = elem_object->getValue<String>(KEY_NAME);
-            if (auto it = find(name); it != end())
-                it->second->fromJSON(*elem_object);
+            auto it = column_type_by_name.find(name);
+
+            if (it == column_type_by_name.end())
+                throw Exception(ErrorCodes::CORRUPTED_DATA,
+                    "Found unexpected column '{}' in serialization infos", name);
+
+            auto info = it->second->createSerializationInfo(settings);
+            info->fromJSON(*elem_object);
+            infos.emplace(name, std::move(info));
         }
     }
+
+    return infos;
 }
 
 }
diff --git a/src/DataTypes/Serializations/SerializationInfo.h b/src/DataTypes/Serializations/SerializationInfo.h
index 22a9d62d312..3d8f4f1d00c 100644
--- a/src/DataTypes/Serializations/SerializationInfo.h
+++ b/src/DataTypes/Serializations/SerializationInfo.h
@@ -96,8 +96,10 @@ using MutableSerializationInfos = std::vector<MutableSerializationInfoPtr>;
 class SerializationInfoByName : public std::map<String, MutableSerializationInfoPtr>
 {
 public:
+    using Settings = SerializationInfo::Settings;
+
     SerializationInfoByName() = default;
-    SerializationInfoByName(const NamesAndTypesList & columns, const SerializationInfo::Settings & settings);
+    SerializationInfoByName(const NamesAndTypesList & columns, const Settings & settings);
 
     void add(const Block & block);
     void add(const SerializationInfoByName & other);
@@ -108,7 +110,9 @@ public:
     void replaceData(const SerializationInfoByName & other);
 
     void writeJSON(WriteBuffer & out) const;
-    void readJSON(ReadBuffer & in);
+
+    static SerializationInfoByName readJSON(
+        const NamesAndTypesList & columns, const Settings & settings, ReadBuffer & in);
 };
 
 }
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
index d3bc3cc70d9..c9930e61e98 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
@@ -1341,11 +1341,11 @@ void IMergeTreeDataPart::loadColumns(bool require)
         .choose_kind = false,
     };
 
-    SerializationInfoByName infos(loaded_columns, settings);
+    SerializationInfoByName infos;
     if (metadata_manager->exists(SERIALIZATION_FILE_NAME))
     {
         auto in = metadata_manager->read(SERIALIZATION_FILE_NAME);
-        infos.readJSON(*in);
+        infos = SerializationInfoByName::readJSON(loaded_columns, settings, *in);
     }
 
     int32_t loaded_metadata_version;
diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp
index de31258b2f9..00710ed3ed6 100644
--- a/src/Storages/MergeTree/checkDataPart.cpp
+++ b/src/Storages/MergeTree/checkDataPart.cpp
@@ -94,12 +94,13 @@ IMergeTreeDataPart::Checksums checkDataPart(
     };
 
     auto ratio_of_defaults = data_part->storage.getSettings()->ratio_of_defaults_for_sparse_serialization;
-    SerializationInfoByName serialization_infos(columns_txt, SerializationInfo::Settings{ratio_of_defaults, false});
+    SerializationInfoByName serialization_infos;
 
     if (data_part_storage.exists(IMergeTreeDataPart::SERIALIZATION_FILE_NAME))
     {
         auto serialization_file = data_part_storage.readFile(IMergeTreeDataPart::SERIALIZATION_FILE_NAME, {}, std::nullopt, std::nullopt);
-        serialization_infos.readJSON(*serialization_file);
+        SerializationInfo::Settings settings{ratio_of_defaults, false};
+        serialization_infos = SerializationInfoByName::readJSON(columns_txt, settings, *serialization_file);
     }
 
     auto get_serialization = [&serialization_infos](const auto & column)
diff --git a/tests/queries/0_stateless/02733_sparse_columns_reload.reference b/tests/queries/0_stateless/02733_sparse_columns_reload.reference
new file mode 100644
index 00000000000..7ab314964ee
--- /dev/null
+++ b/tests/queries/0_stateless/02733_sparse_columns_reload.reference
@@ -0,0 +1,2 @@
+100000
+100000
diff --git a/tests/queries/0_stateless/02733_sparse_columns_reload.sql b/tests/queries/0_stateless/02733_sparse_columns_reload.sql
new file mode 100644
index 00000000000..d4b48274107
--- /dev/null
+++ b/tests/queries/0_stateless/02733_sparse_columns_reload.sql
@@ -0,0 +1,18 @@
+DROP TABLE IF EXISTS t_sparse_reload;
+
+CREATE TABLE t_sparse_reload (id UInt64, v UInt64)
+ENGINE = MergeTree ORDER BY id
+SETTINGS ratio_of_defaults_for_sparse_serialization = 0.95;
+
+INSERT INTO t_sparse_reload SELECT number, 0 FROM numbers(100000);
+
+SELECT count() FROM t_sparse_reload WHERE NOT ignore(*);
+
+ALTER TABLE t_sparse_reload MODIFY SETTING ratio_of_defaults_for_sparse_serialization = 1.0;
+
+DETACH TABLE t_sparse_reload;
+ATTACH TABLE t_sparse_reload;
+
+SELECT count() FROM t_sparse_reload WHERE NOT ignore(*);
+
+DROP TABLE t_sparse_reload;

From ee4ce7db470de74921a59604837b097b6d94f057 Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Mon, 8 May 2023 12:33:24 +0000
Subject: [PATCH 301/689] Fix style

---
 tests/integration/test_ssl_cert_authentication/test.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py
index 641680b07d3..ec7111099c9 100644
--- a/tests/integration/test_ssl_cert_authentication/test.py
+++ b/tests/integration/test_ssl_cert_authentication/test.py
@@ -167,9 +167,7 @@ def execute_query_https(
     retries = 10
     while True:
         try:
-            url = (
-            f"https://{instance.ip_address}:{HTTPS_PORT}/?query={urllib.parse.quote(query)}"
-            )
+            url = f"https://{instance.ip_address}:{HTTPS_PORT}/?query={urllib.parse.quote(query)}"
             request = urllib.request.Request(url)
             request.add_header("X-ClickHouse-User", user)
             if enable_ssl_auth:

From 1e3cd4ed6d39961addf82ab75d5fb2f661801ae5 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Mon, 8 May 2023 12:39:01 +0000
Subject: [PATCH 302/689] close pipe before join

---
 src/Interpreters/TraceCollector.cpp | 47 ++++++++++++++---------------
 src/Interpreters/TraceCollector.h   |  1 -
 2 files changed, 22 insertions(+), 26 deletions(-)

diff --git a/src/Interpreters/TraceCollector.cpp b/src/Interpreters/TraceCollector.cpp
index bedc27f7d15..0fc9aeb170b 100644
--- a/src/Interpreters/TraceCollector.cpp
+++ b/src/Interpreters/TraceCollector.cpp
@@ -34,40 +34,37 @@ TraceCollector::~TraceCollector()
 {
     try
     {
-        if (!thread.joinable())
-            LOG_ERROR(&Poco::Logger::get("TraceCollector"), "TraceCollector thread is malformed and cannot be joined");
+        if (thread.joinable())
+        {
+            /** Sends TraceCollector stop message
+            *
+            * Each sequence of data for TraceCollector thread starts with a boolean flag.
+            * If this flag is true, TraceCollector must stop reading trace_pipe and exit.
+            * This function sends flag with a true value to stop TraceCollector gracefully.
+            */
+            WriteBufferFromFileDescriptor out(TraceSender::pipe.fds_rw[1]);
+            writeChar(true, out);
+            out.next();
+        }
         else
-            stop();
+            LOG_ERROR(&Poco::Logger::get("TraceCollector"), "TraceCollector thread is malformed and cannot be joined");
+    }
+    catch (...)
+    {
+        tryLogCurrentException("TraceCollector");
+    }
 
+    try
+    {
         TraceSender::pipe.close();
     }
     catch (...)
     {
         tryLogCurrentException("TraceCollector");
     }
-}
 
-
-/** Sends TraceCollector stop message
-  *
-  * Each sequence of data for TraceCollector thread starts with a boolean flag.
-  * If this flag is true, TraceCollector must stop reading trace_pipe and exit.
-  * This function sends flag with a true value to stop TraceCollector gracefully.
-  */
-void TraceCollector::stop()
-{
-    try
-    {
-        WriteBufferFromFileDescriptor out(TraceSender::pipe.fds_rw[1]);
-        writeChar(true, out);
-        out.next();
-    }
-    catch (...)
-    {
-        tryLogCurrentException("TraceCollector");
-    }
-
-    thread.join();
+    if (thread.joinable())
+        thread.join();
 }
 
 
diff --git a/src/Interpreters/TraceCollector.h b/src/Interpreters/TraceCollector.h
index 40fa854b791..d5bd09f1757 100644
--- a/src/Interpreters/TraceCollector.h
+++ b/src/Interpreters/TraceCollector.h
@@ -24,7 +24,6 @@ private:
     ThreadFromGlobalPool thread;
 
     void run();
-    void stop();
 };
 
 }

From 14dc7d988f0258621b1181a564c8e50c8f30c127 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Mon, 8 May 2023 14:49:11 +0200
Subject: [PATCH 303/689] fix

---
 src/Databases/removeWhereConditionPlaceholder.cpp | 6 +-----
 src/Databases/removeWhereConditionPlaceholder.h   | 4 ++++
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/Databases/removeWhereConditionPlaceholder.cpp b/src/Databases/removeWhereConditionPlaceholder.cpp
index efe6a5313df..fb147be26d0 100644
--- a/src/Databases/removeWhereConditionPlaceholder.cpp
+++ b/src/Databases/removeWhereConditionPlaceholder.cpp
@@ -3,12 +3,8 @@
 namespace DB
 {
 
-std::string removeWhereConditionPlaceholder(const std::string  & query)
+std::string removeWhereConditionPlaceholder(const std::string & query)
 {
-    /** In case UPDATE_FIELD is specified in {condition} for dictionary that must load all data.
-      * Replace {condition} with true_condition for initial dictionary load.
-      * For next dictionary loads {condition} will be updated with UPDATE_FIELD.
-      */
     static constexpr auto true_condition = "(1 = 1)";
     auto condition_position = query.find(CONDITION_PLACEHOLDER_TO_REPLACE_VALUE);
     if (condition_position != std::string::npos)
diff --git a/src/Databases/removeWhereConditionPlaceholder.h b/src/Databases/removeWhereConditionPlaceholder.h
index 8a4da3cb9f2..7f0e23d42cf 100644
--- a/src/Databases/removeWhereConditionPlaceholder.h
+++ b/src/Databases/removeWhereConditionPlaceholder.h
@@ -6,6 +6,10 @@ namespace DB
 
 static constexpr std::string_view CONDITION_PLACEHOLDER_TO_REPLACE_VALUE = "{condition}";
 
+/** In case UPDATE_FIELD is specified in {condition} for dictionary that must load all data.
+  * Replace {condition} with true_condition for initial dictionary load.
+  * For next dictionary loads {condition} will be updated with UPDATE_FIELD.
+  */
 std::string removeWhereConditionPlaceholder(const std::string & query);
 
 }

From 49ecba63af61313d4419a80e827a1ba22f163838 Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Mon, 8 May 2023 14:51:04 +0200
Subject: [PATCH 304/689] Removed setStageForCluster and added option all_hosts
 to set stage for cluster

---
 src/Backups/BackupCoordinationLocal.cpp     |  4 --
 src/Backups/BackupCoordinationLocal.h       |  3 +-
 src/Backups/BackupCoordinationRemote.cpp    | 11 ++----
 src/Backups/BackupCoordinationRemote.h      |  3 +-
 src/Backups/BackupCoordinationStageSync.cpp | 43 +++++++++++----------
 src/Backups/BackupCoordinationStageSync.h   |  3 +-
 src/Backups/BackupsWorker.cpp               |  6 +--
 src/Backups/IBackupCoordination.h           |  3 +-
 src/Backups/IRestoreCoordination.h          |  3 +-
 src/Backups/RestoreCoordinationLocal.cpp    |  4 --
 src/Backups/RestoreCoordinationLocal.h      |  3 +-
 src/Backups/RestoreCoordinationRemote.cpp   | 11 ++----
 src/Backups/RestoreCoordinationRemote.h     |  3 +-
 13 files changed, 40 insertions(+), 60 deletions(-)

diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp
index 5b7ee37618b..27e0f173cf3 100644
--- a/src/Backups/BackupCoordinationLocal.cpp
+++ b/src/Backups/BackupCoordinationLocal.cpp
@@ -19,10 +19,6 @@ void BackupCoordinationLocal::setStage(const String &, const String &)
 {
 }
 
-void BackupCoordinationLocal::setStageForCluster(const String &)
-{
-}
-
 void BackupCoordinationLocal::setError(const Exception &)
 {
 }
diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h
index f1ffa8e8517..a7b05fbb83c 100644
--- a/src/Backups/BackupCoordinationLocal.h
+++ b/src/Backups/BackupCoordinationLocal.h
@@ -22,8 +22,7 @@ public:
     BackupCoordinationLocal(bool plain_backup_);
     ~BackupCoordinationLocal() override;
 
-    void setStage(const String & new_stage, const String & message) override;
-    void setStageForCluster(const String & new_stage) override; /// Sets stage for cluster
+    void setStage(const String & new_stage, const String & message = "") override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp
index af88b15d622..27e7d23ce5f 100644
--- a/src/Backups/BackupCoordinationRemote.cpp
+++ b/src/Backups/BackupCoordinationRemote.cpp
@@ -254,17 +254,14 @@ void BackupCoordinationRemote::removeAllNodes()
 
 void BackupCoordinationRemote::setStage(const String & new_stage, const String & message)
 {
-    stage_sync->set(current_host, new_stage, message);
-}
-
-void BackupCoordinationRemote::setStageForCluster(const String & new_stage)
-{
-    stage_sync->setStageForCluster(new_stage);
+    if (is_internal)
+        stage_sync->set(current_host, new_stage, message);
+    else
+        stage_sync->set(current_host, new_stage, /* message */ "", /* all_hosts */ true);
 }
 
 void BackupCoordinationRemote::setError(const Exception & exception)
 {
-    stage_sync->setStageForCluster(Stage::ERROR);
     stage_sync->setError(current_host, exception);
 }
 
diff --git a/src/Backups/BackupCoordinationRemote.h b/src/Backups/BackupCoordinationRemote.h
index c659cb0d459..5671079fa27 100644
--- a/src/Backups/BackupCoordinationRemote.h
+++ b/src/Backups/BackupCoordinationRemote.h
@@ -33,8 +33,7 @@ public:
 
     ~BackupCoordinationRemote() override;
 
-    void setStage(const String & new_stage, const String & message) override;
-    void setStageForCluster(const String & new_stage) override; /// Sets stage for cluster
+    void setStage(const String & new_stage, const String & message = "") override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
diff --git a/src/Backups/BackupCoordinationStageSync.cpp b/src/Backups/BackupCoordinationStageSync.cpp
index 5cbeec0ec76..3d8c283f084 100644
--- a/src/Backups/BackupCoordinationStageSync.cpp
+++ b/src/Backups/BackupCoordinationStageSync.cpp
@@ -8,11 +8,13 @@
 #include <IO/ReadHelpers.h>
 #include <IO/WriteBufferFromString.h>
 #include <IO/WriteHelpers.h>
-
+#include <Backups/BackupCoordinationStage.h>
 
 namespace DB
 {
 
+namespace Stage = BackupCoordinationStage;
+
 namespace ErrorCodes
 {
     extern const int FAILED_TO_SYNC_BACKUP_OR_RESTORE;
@@ -42,7 +44,7 @@ void BackupCoordinationStageSync::createRootNodes()
     });
 }
 
-void BackupCoordinationStageSync::set(const String & current_host, const String & new_stage, const String & message)
+void BackupCoordinationStageSync::set(const String & current_host, const String & new_stage, const String & message, const bool & all_hosts)
 {
     auto holder = with_retries.createRetriesControlHolder("set");
     holder.retries_ctl.retryLoop(
@@ -50,29 +52,24 @@ void BackupCoordinationStageSync::set(const String & current_host, const String
     {
         with_retries.renewZooKeeper(zookeeper);
 
-        /// Make an ephemeral node so the initiator can track if the current host is still working.
-        String alive_node_path = zookeeper_path + "/alive|" + current_host;
-        auto code = zookeeper->tryCreate(alive_node_path, "", zkutil::CreateMode::Ephemeral);
-        if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNODEEXISTS)
-            throw zkutil::KeeperException(code, alive_node_path);
-
-        zookeeper->createIfNotExists(zookeeper_path + "/started|" + current_host, "");
-        zookeeper->createIfNotExists(zookeeper_path + "/current|" + current_host + "|" + new_stage, message);
-    });
-}
-
-void BackupCoordinationStageSync::setStageForCluster(const String & new_stage)
-{
-    auto holder = with_retries.createRetriesControlHolder("setStageForCluster");
-    holder.retries_ctl.retryLoop(
-        [&, &zookeeper = holder.faulty_zookeeper]()
+        if (all_hosts)
         {
-            with_retries.renewZooKeeper(zookeeper);
-            zookeeper->trySet(zookeeper_path, new_stage);
             auto code = zookeeper->trySet(zookeeper_path, new_stage);
             if (code != Coordination::Error::ZOK)
                 throw zkutil::KeeperException(code, zookeeper_path);
-        });
+        }
+        else
+        {
+            /// Make an ephemeral node so the initiator can track if the current host is still working.
+            String alive_node_path = zookeeper_path + "/alive|" + current_host;
+            auto code = zookeeper->tryCreate(alive_node_path, "", zkutil::CreateMode::Ephemeral);
+            if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNODEEXISTS)
+                throw zkutil::KeeperException(code, alive_node_path);
+
+            zookeeper->createIfNotExists(zookeeper_path + "/started|" + current_host, "");
+            zookeeper->createIfNotExists(zookeeper_path + "/current|" + current_host + "|" + new_stage, message);
+        }
+    });
 }
 
 void BackupCoordinationStageSync::setError(const String & current_host, const Exception & exception)
@@ -87,6 +84,10 @@ void BackupCoordinationStageSync::setError(const String & current_host, const Ex
         writeStringBinary(current_host, buf);
         writeException(exception, buf, true);
         zookeeper->createIfNotExists(zookeeper_path + "/error", buf.str());
+
+        auto code = zookeeper->trySet(zookeeper_path, Stage::ERROR);
+        if (code != Coordination::Error::ZOK)
+            throw zkutil::KeeperException(code, zookeeper_path);
     });
 }
 
diff --git a/src/Backups/BackupCoordinationStageSync.h b/src/Backups/BackupCoordinationStageSync.h
index 9dde4e3095f..2efaec46b3a 100644
--- a/src/Backups/BackupCoordinationStageSync.h
+++ b/src/Backups/BackupCoordinationStageSync.h
@@ -15,8 +15,7 @@ public:
         Poco::Logger * log_);
 
     /// Sets the stage of the current host and signal other hosts if there were other hosts waiting for that.
-    void set(const String & current_host, const String & new_stage, const String & message);
-    void setStageForCluster(const String & new_stage);
+    void set(const String & current_host, const String & new_stage, const String & message, const bool & all_hosts = false);
     void setError(const String & current_host, const Exception & exception);
 
     /// Sets the stage of the current host and waits until all hosts come to the same stage.
diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp
index de05cc2b092..720ca994a40 100644
--- a/src/Backups/BackupsWorker.cpp
+++ b/src/Backups/BackupsWorker.cpp
@@ -368,7 +368,7 @@ void BackupsWorker::doBackup(
 
             /// Wait until all the hosts have written their backup entries.
             backup_coordination->waitForStage(Stage::COMPLETED);
-            backup_coordination->setStageForCluster(Stage::COMPLETED);
+            backup_coordination->setStage(Stage::COMPLETED);
         }
         else
         {
@@ -386,7 +386,7 @@ void BackupsWorker::doBackup(
             writeBackupEntries(backup, std::move(backup_entries), backup_id, backup_coordination, backup_settings.internal);
 
             /// We have written our backup entries, we need to tell other hosts (they could be waiting for it).
-            backup_coordination->setStage(Stage::COMPLETED, "");
+            backup_coordination->setStage(Stage::COMPLETED);
         }
 
         size_t num_files = 0;
@@ -709,7 +709,7 @@ void BackupsWorker::doRestore(
 
             /// Wait until all the hosts have written their backup entries.
             restore_coordination->waitForStage(Stage::COMPLETED);
-            restore_coordination->setStageForCluster(Stage::COMPLETED);
+            restore_coordination->setStage(Stage::COMPLETED);
         }
         else
         {
diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h
index 6caae1dd741..68a13ab7846 100644
--- a/src/Backups/IBackupCoordination.h
+++ b/src/Backups/IBackupCoordination.h
@@ -21,8 +21,7 @@ public:
     virtual ~IBackupCoordination() = default;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    virtual void setStage(const String & new_stage, const String & message) = 0;
-    virtual void setStageForCluster(const String & new_stage) = 0;
+    virtual void setStage(const String & new_stage, const String & message = "") = 0;
     virtual void setError(const Exception & exception) = 0;
     virtual Strings waitForStage(const String & stage_to_wait) = 0;
     virtual Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) = 0;
diff --git a/src/Backups/IRestoreCoordination.h b/src/Backups/IRestoreCoordination.h
index a5c8db84c86..b4df9491c4c 100644
--- a/src/Backups/IRestoreCoordination.h
+++ b/src/Backups/IRestoreCoordination.h
@@ -18,8 +18,7 @@ public:
     virtual ~IRestoreCoordination() = default;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    virtual void setStage(const String & new_stage, const String & message) = 0;
-    virtual void setStageForCluster(const String & new_stage) = 0; /// Sets stage for cluster
+    virtual void setStage(const String & new_stage, const String & message = "") = 0;
     virtual void setError(const Exception & exception) = 0;
     virtual Strings waitForStage(const String & stage_to_wait) = 0;
     virtual Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) = 0;
diff --git a/src/Backups/RestoreCoordinationLocal.cpp b/src/Backups/RestoreCoordinationLocal.cpp
index 513204c931c..068c4fe7e52 100644
--- a/src/Backups/RestoreCoordinationLocal.cpp
+++ b/src/Backups/RestoreCoordinationLocal.cpp
@@ -15,10 +15,6 @@ void RestoreCoordinationLocal::setStage(const String &, const String &)
 {
 }
 
-void RestoreCoordinationLocal::setStageForCluster(const String &)
-{
-}
-
 void RestoreCoordinationLocal::setError(const Exception &)
 {
 }
diff --git a/src/Backups/RestoreCoordinationLocal.h b/src/Backups/RestoreCoordinationLocal.h
index 0e4f4f01846..2240a25ef3d 100644
--- a/src/Backups/RestoreCoordinationLocal.h
+++ b/src/Backups/RestoreCoordinationLocal.h
@@ -19,8 +19,7 @@ public:
     ~RestoreCoordinationLocal() override;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    void setStage(const String & new_stage, const String & message) override;
-    void setStageForCluster(const String & new_stage) override;
+    void setStage(const String & new_stage, const String & message = "") override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp
index a3541614f36..c4ecee4aaa6 100644
--- a/src/Backups/RestoreCoordinationRemote.cpp
+++ b/src/Backups/RestoreCoordinationRemote.cpp
@@ -93,17 +93,14 @@ void RestoreCoordinationRemote::createRootNodes()
 
 void RestoreCoordinationRemote::setStage(const String & new_stage, const String & message)
 {
-    stage_sync->set(current_host, new_stage, message);
-}
-
-void RestoreCoordinationRemote::setStageForCluster(const String & new_stage)
-{
-    stage_sync->setStageForCluster(new_stage);
+    if (is_internal)
+        stage_sync->set(current_host, new_stage, message);
+    else
+        stage_sync->set(current_host, new_stage, /* message */ "", /* all_hosts */ true);
 }
 
 void RestoreCoordinationRemote::setError(const Exception & exception)
 {
-    stage_sync->setStageForCluster(Stage::ERROR);
     stage_sync->setError(current_host, exception);
 }
 
diff --git a/src/Backups/RestoreCoordinationRemote.h b/src/Backups/RestoreCoordinationRemote.h
index 947d08a66e5..989b1c1b727 100644
--- a/src/Backups/RestoreCoordinationRemote.h
+++ b/src/Backups/RestoreCoordinationRemote.h
@@ -26,8 +26,7 @@ public:
     ~RestoreCoordinationRemote() override;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    void setStage(const String & new_stage, const String & message) override;
-    void setStageForCluster(const String & new_stage) override;
+    void setStage(const String & new_stage, const String & message = "") override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;

From 713ffb3bda09e4f72f17c5029d8dd7e73dd1d2a5 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Mon, 8 May 2023 13:22:31 +0000
Subject: [PATCH 305/689] fix race in Context::createCopy

---
 src/Interpreters/Context.cpp                  |  4 +-
 src/Interpreters/tests/gtest_context_race.cpp | 49 +++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)
 create mode 100644 src/Interpreters/tests/gtest_context_race.cpp

diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index 31c0f2fc87a..ee2fc0c5c4e 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -669,13 +669,15 @@ SharedContextHolder Context::createShared()
 
 ContextMutablePtr Context::createCopy(const ContextPtr & other)
 {
+    auto lock = other->getLock();
     return std::shared_ptr<Context>(new Context(*other));
 }
 
 ContextMutablePtr Context::createCopy(const ContextWeakPtr & other)
 {
     auto ptr = other.lock();
-    if (!ptr) throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't copy an expired context");
+    if (!ptr)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't copy an expired context");
     return createCopy(ptr);
 }
 
diff --git a/src/Interpreters/tests/gtest_context_race.cpp b/src/Interpreters/tests/gtest_context_race.cpp
new file mode 100644
index 00000000000..32922305435
--- /dev/null
+++ b/src/Interpreters/tests/gtest_context_race.cpp
@@ -0,0 +1,49 @@
+#include <Interpreters/Context.h>
+#include <Common/tests/gtest_global_context.h>
+#include <gtest/gtest.h>
+
+using namespace DB;
+
+template <typename Ptr>
+void run(Ptr && context)
+{
+    for (size_t i = 0; i < 100; ++i)
+    {
+        std::thread t1([context]
+        {
+            if constexpr (std::is_same_v<ContextWeakPtr, Ptr>)
+                context.lock()->getAsyncReadCounters();
+            else
+                context->getAsyncReadCounters();
+        });
+
+        std::thread t2([context]
+        {
+            Context::createCopy(context);
+        });
+
+        t1.join();
+        t2.join();
+    }
+}
+
+TEST(Context, MutableRace)
+{
+    auto context = Context::createCopy(getContext().context);
+    context->makeQueryContext();
+    run<ContextMutablePtr>(std::move(context));
+}
+
+TEST(Context, ConstRace)
+{
+    auto context = Context::createCopy(getContext().context);
+    context->makeQueryContext();
+    run<ContextPtr>(std::move(context));
+}
+
+TEST(Context, WeakRace)
+{
+    auto context = Context::createCopy(getContext().context);
+    context->makeQueryContext();
+    run<ContextWeakPtr>(std::move(context));
+}

From 8482cf01c268bd9d8153fa5e19444a03f7f19c07 Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <440544+davenger@users.noreply.github.com>
Date: Mon, 8 May 2023 15:27:10 +0200
Subject: [PATCH 306/689] Check that lengths are also equal

Slightly modified the test to make it more clear
---
 .../02734_big_int_from_float_ubsan.reference           |  2 +-
 .../0_stateless/02734_big_int_from_float_ubsan.sql     | 10 +++++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02734_big_int_from_float_ubsan.reference b/tests/queries/0_stateless/02734_big_int_from_float_ubsan.reference
index d00491fd7e5..9972842f982 100644
--- a/tests/queries/0_stateless/02734_big_int_from_float_ubsan.reference
+++ b/tests/queries/0_stateless/02734_big_int_from_float_ubsan.reference
@@ -1 +1 @@
-1
+1	1
diff --git a/tests/queries/0_stateless/02734_big_int_from_float_ubsan.sql b/tests/queries/0_stateless/02734_big_int_from_float_ubsan.sql
index ebe1394bb55..9fbf54c1a4d 100644
--- a/tests/queries/0_stateless/02734_big_int_from_float_ubsan.sql
+++ b/tests/queries/0_stateless/02734_big_int_from_float_ubsan.sql
@@ -1 +1,9 @@
-WITH 18 AS precision, toUInt256(-1) AS int, toUInt256(toFloat64(toUInt256(-1))) AS converted SELECT substring(toString(int), 1, precision) = substring(toString(converted), 1, precision) AS res;
+WITH
+    18 AS precision,
+    toUInt256(-1) AS int,
+    toUInt256(toFloat64(int)) AS converted,
+    toString(int) AS int_str,
+    toString(converted) AS converted_str
+SELECT
+    length(int_str) = length(converted_str) AS have_same_length,
+    substring(int_str, 1, precision) = substring(converted_str, 1, precision) AS have_same_prefix

From 92e57817a21529409d659c7c759a62a82b0a3bde Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Mon, 8 May 2023 16:14:08 +0200
Subject: [PATCH 307/689] Support `dictionary` table function for
 `RegExpTreeDictionary`

---
 src/Core/Settings.h                           |  1 -
 src/Dictionaries/RegExpTreeDictionary.cpp     | 53 ++++++++++++++++---
 src/Dictionaries/RegExpTreeDictionary.h       |  7 ++-
 .../ExternalDictionariesLoader.cpp            | 30 ++++++++++-
 src/Interpreters/ExternalDictionariesLoader.h |  3 ++
 .../TableFunctionDictionary.cpp               |  4 +-
 ...4_regexp_dictionary_table_source.reference |  9 ++++
 .../02504_regexp_dictionary_table_source.sql  |  6 ++-
 8 files changed, 94 insertions(+), 19 deletions(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index ed5ad0278a2..8bcb38d5c5f 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -969,7 +969,6 @@ class IColumn;
     M(Bool, output_format_bson_string_as_string, false, "Use BSON String type instead of Binary for String columns.", 0) \
     M(Bool, input_format_bson_skip_fields_with_unsupported_types_in_schema_inference, false, "Skip fields with unsupported types while schema inference for format BSON.", 0) \
     \
-    M(Bool, regexp_dict_allow_other_sources, false, "Allow regexp_tree dictionary to use sources other than yaml source.", 0) \
     M(Bool, regexp_dict_allow_hyperscan, true, "Allow regexp_tree dictionary using Hyperscan library.", 0) \
     \
     M(Bool, dictionary_use_async_executor, false, "Execute a pipeline for reading from a dictionary with several threads. It's supported only by DIRECT dictionary with CLICKHOUSE source.", 0) \
diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp
index 8263c87d0e6..79fefde23e1 100644
--- a/src/Dictionaries/RegExpTreeDictionary.cpp
+++ b/src/Dictionaries/RegExpTreeDictionary.cpp
@@ -31,6 +31,7 @@
 
 #include <re2_st/stringpiece.h>
 
+#include "Processors/Sources/BlocksListSource.h"
 #include "config.h"
 
 #if USE_VECTORSCAN
@@ -117,6 +118,7 @@ struct RegExpTreeDictionary::RegexTreeNode
     {
         Field field;
         std::vector<StringPiece> pieces;
+        String original_value;
 
         constexpr bool containsBackRefs() const { return !pieces.empty(); }
     };
@@ -208,12 +210,12 @@ void RegExpTreeDictionary::initRegexNodes(Block & block)
                 auto string_pieces = createStringPieces(value, num_captures, regex, logger);
                 if (!string_pieces.empty())
                 {
-                    node->attributes[name_] = RegexTreeNode::AttributeValue{.field = values[j], .pieces = std::move(string_pieces)};
+                    node->attributes[name_] = RegexTreeNode::AttributeValue{.field = values[j], .pieces = std::move(string_pieces), .original_value = value};
                 }
                 else
                 {
-                    Field field = parseStringToField(values[j].safeGet<String>(), attr.type);
-                    node->attributes[name_] = RegexTreeNode::AttributeValue{.field = std::move(field)};
+                    Field field = parseStringToField(value, attr.type);
+                    node->attributes[name_] = RegexTreeNode::AttributeValue{.field = std::move(field), .original_value = value};
                 }
             }
         }
@@ -651,6 +653,47 @@ std::unordered_map<String, ColumnPtr> RegExpTreeDictionary::match(
     return result;
 }
 
+Pipe RegExpTreeDictionary::read(const Names & , size_t max_block_size, size_t) const
+{
+    auto it = regex_nodes.begin();
+    size_t block_size = 0;
+    BlocksList result;
+    for (;;)
+    {
+        Block block;
+        auto col_id = std::make_shared<DataTypeUInt64>()->createColumn();
+        auto col_pid = std::make_shared<DataTypeUInt64>()->createColumn();
+        auto col_regex = std::make_shared<DataTypeString>()->createColumn();
+        auto col_keys = std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())->createColumn();
+        auto col_values = std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())->createColumn();
+        for (;it != regex_nodes.end() && block_size < max_block_size; it++, block_size++)
+        {
+            col_id->insert(it->first);
+            const auto & node = it->second;
+            col_pid->insert(node->parent_id);
+            col_regex->insert(node->regex);
+            std::vector<Field> keys, values;
+            for (const auto & [key, attr] : node->attributes)
+            {
+                keys.push_back(key);
+                values.push_back(attr.original_value);
+            }
+            col_keys->insert(Array(keys.begin(), keys.end()));
+            col_values->insert(Array(values.begin(), values.end()));
+        }
+        block.insert(ColumnWithTypeAndName(std::move(col_id),std::make_shared<DataTypeUInt64>(),kId));
+        block.insert(ColumnWithTypeAndName(std::move(col_pid),std::make_shared<DataTypeUInt64>(),kParentId));
+        block.insert(ColumnWithTypeAndName(std::move(col_regex),std::make_shared<DataTypeString>(),kRegExp));
+        block.insert(ColumnWithTypeAndName(std::move(col_keys),std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),kKeys));
+        block.insert(ColumnWithTypeAndName(std::move(col_values),std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),kValues));
+        result.push_back(std::move(block));
+        if (it == regex_nodes.end())
+            break;
+        block_size = 0;
+    }
+    return Pipe(std::make_shared<BlocksListSource>(std::move(result)));
+}
+
 Columns RegExpTreeDictionary::getColumns(
     const Strings & attribute_names,
     const DataTypes & result_types,
@@ -717,10 +760,6 @@ void registerDictionaryRegExpTree(DictionaryFactory & factory)
         const auto dict_id = StorageID::fromDictionaryConfig(config, config_prefix);
 
         auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix);
-        if (!context->getSettings().regexp_dict_allow_other_sources && typeid_cast<YAMLRegExpTreeDictionarySource *>(source_ptr.get()) == nullptr)
-            throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION,
-                            "regexp_tree dictionary doesn't accept sources other than yaml source. "
-                            "To active it, please set regexp_dict_allow_other_sources=true");
 
         return std::make_unique<RegExpTreeDictionary>(dict_id, dict_struct, std::move(source_ptr), configuration, context->getSettings().regexp_dict_allow_hyperscan);
     };
diff --git a/src/Dictionaries/RegExpTreeDictionary.h b/src/Dictionaries/RegExpTreeDictionary.h
index 4e8e20bba2d..893d1fb7a0b 100644
--- a/src/Dictionaries/RegExpTreeDictionary.h
+++ b/src/Dictionaries/RegExpTreeDictionary.h
@@ -22,6 +22,8 @@
 #include <Dictionaries/DictionaryStructure.h>
 #include <Dictionaries/IDictionary.h>
 
+#include <Storages/ColumnsDescription.h>
+
 namespace DB
 {
 
@@ -91,10 +93,7 @@ public:
         throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Dictionary {} does not support method `hasKeys`", name);
     }
 
-    Pipe read(const Names &, size_t, size_t) const override
-    {
-        throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Dictionary {} does not support method `read`", name);
-    }
+    Pipe read(const Names &, size_t, size_t) const override;
 
     ColumnPtr getColumn(
         const std::string & attribute_name,
diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp
index 080878c7d86..2129b3bbec6 100644
--- a/src/Interpreters/ExternalDictionariesLoader.cpp
+++ b/src/Interpreters/ExternalDictionariesLoader.cpp
@@ -4,9 +4,11 @@
 #include <Dictionaries/DictionaryFactory.h>
 #include <Dictionaries/DictionaryStructure.h>
 #include <Databases/IDatabase.h>
+#include <DataTypes/DataTypeArray.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypesNumber.h>
 #include <Storages/IStorage.h>
-
-#include "config.h"
+#include <Storages/StorageDictionary.h>
 
 #if USE_MYSQL
 #   include <mysqlxx/PoolFactory.h>
@@ -67,6 +69,30 @@ void ExternalDictionariesLoader::reloadDictionary(const std::string & dictionary
     loadOrReload(resolved_dictionary_name);
 }
 
+ColumnsDescription ExternalDictionariesLoader::getActualTableStructure(const std::string & dictionary_name, ContextPtr query_context) const
+{
+    /// if it is regexp tree dictionary, we have a special table structure.
+    std::string resolved_name = resolveDictionaryName(dictionary_name, query_context->getCurrentDatabase());
+    auto load_result = load(resolved_name);
+    if (load_result)
+    {
+        const auto dictionary = std::static_pointer_cast<const IDictionary>(load_result);
+        if (dictionary->getTypeName() == "RegExpTree")
+        {
+            return ColumnsDescription(NamesAndTypesList({
+                {"id", std::make_shared<DataTypeUInt64>()},
+                {"parent_id", std::make_shared<DataTypeUInt64>()},
+                {"regexp", std::make_shared<DataTypeString>()},
+                {"keys", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
+                {"values", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())}
+            }));
+        }
+    }
+    /// otherwise, we get table structure by dictionary structure.
+    auto dictionary_structure = getDictionaryStructure(dictionary_name, query_context);
+    return ColumnsDescription(StorageDictionary::getNamesAndTypes(dictionary_structure));
+}
+
 DictionaryStructure ExternalDictionariesLoader::getDictionaryStructure(const std::string & dictionary_name, ContextPtr query_context) const
 {
     std::string resolved_name = resolveDictionaryName(dictionary_name, query_context->getCurrentDatabase());
diff --git a/src/Interpreters/ExternalDictionariesLoader.h b/src/Interpreters/ExternalDictionariesLoader.h
index 659820d02ac..5635a307816 100644
--- a/src/Interpreters/ExternalDictionariesLoader.h
+++ b/src/Interpreters/ExternalDictionariesLoader.h
@@ -4,6 +4,7 @@
 #include <Interpreters/Context_fwd.h>
 #include <Interpreters/ExternalLoader.h>
 #include <Common/quoteString.h>
+#include "Storages/ColumnsDescription.h"
 
 #include <memory>
 
@@ -29,6 +30,8 @@ public:
 
     QualifiedTableName qualifyDictionaryNameWithDatabase(const std::string & dictionary_name, ContextPtr context) const;
 
+    ColumnsDescription getActualTableStructure(const std::string & dictionary_name, ContextPtr context) const;
+
     DictionaryStructure getDictionaryStructure(const std::string & dictionary_name, ContextPtr context) const;
 
     void assertDictionaryStructureExists(const std::string & dictionary_name, ContextPtr context) const;
diff --git a/src/TableFunctions/TableFunctionDictionary.cpp b/src/TableFunctions/TableFunctionDictionary.cpp
index 54c23cfb64b..5231e39dd8c 100644
--- a/src/TableFunctions/TableFunctionDictionary.cpp
+++ b/src/TableFunctions/TableFunctionDictionary.cpp
@@ -42,10 +42,8 @@ void TableFunctionDictionary::parseArguments(const ASTPtr & ast_function, Contex
 ColumnsDescription TableFunctionDictionary::getActualTableStructure(ContextPtr context) const
 {
     const ExternalDictionariesLoader & external_loader = context->getExternalDictionariesLoader();
-    auto dictionary_structure = external_loader.getDictionaryStructure(dictionary_name, context);
-    auto result = ColumnsDescription(StorageDictionary::getNamesAndTypes(dictionary_structure));
 
-    return result;
+    return external_loader.getActualTableStructure(dictionary_name, context);
 }
 
 StoragePtr TableFunctionDictionary::executeImpl(
diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_table_source.reference b/tests/queries/0_stateless/02504_regexp_dictionary_table_source.reference
index 28b98cfabf3..86a74291b07 100644
--- a/tests/queries/0_stateless/02504_regexp_dictionary_table_source.reference
+++ b/tests/queries/0_stateless/02504_regexp_dictionary_table_source.reference
@@ -1,3 +1,9 @@
+1	0	Linux/(\\d+[\\.\\d]*).+tlinux	['version','name']	['\\1','TencentOS']
+2	0	(\\d+)/tclwebkit(\\d+[\\.\\d]*)	['comment','version','name']	['test $1 and $2','$1','Andriod']
+3	2	33/tclwebkit	['version']	['13']
+4	2	3[12]/tclwebkit	['version']	['12']
+5	2	3[12]/tclwebkit	['version']	['11']
+6	2	3[12]/tclwebkit	['version']	['10']
 ('TencentOS',101,'nothing')
 ('Andriod',13,'test 33 and 11.10')
 ('',NULL,'nothing')
@@ -33,3 +39,6 @@
 ('Andriod',43)
 ('Andriod',44)
 ('Andriod1',33,'matched 3')
+1	0	(\\d+)/tclwebkit	['version','name']	['$1','Andriod']
+2	0	33/tclwebkit	['comment','version']	['matched 3','13']
+3	1	33/tclwebkit	['name']	['Andriod1']
diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql b/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql
index a8f1fb17a45..e7937b347a0 100644
--- a/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql
+++ b/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql
@@ -31,8 +31,9 @@ create dictionary regexp_dict1
 PRIMARY KEY(regexp)
 SOURCE(CLICKHOUSE(QUERY concat('select * from ', currentDatabase() , '.regexp_dictionary_source_table')))
 LIFETIME(0)
-LAYOUT(regexp_tree)
-SETTINGS(regexp_dict_allow_other_sources = true);
+LAYOUT(regexp_tree);
+
+select * from dictionary(regexp_dict1);
 
 select dictGet('regexp_dict1', ('name', 'version', 'comment'), 'Linux/101.tlinux');
 select dictGet('regexp_dict1', ('name', 'version', 'comment'), '33/tclwebkit11.10x');
@@ -79,6 +80,7 @@ select dictGet(regexp_dict1, ('name', 'version', 'comment'), '33/tclwebkit');
 truncate table regexp_dictionary_source_table;
 SYSTEM RELOAD dictionary regexp_dict1; -- { serverError 489 }
 
+select * from dictionary(regexp_dict1);
 
 DROP TABLE IF EXISTS regexp_dictionary_source_table;
 DROP TABLE IF EXISTS needle_table;

From 71c96ceb6138512a2cd303ee196a9aed2cd18526 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Mon, 8 May 2023 16:21:42 +0200
Subject: [PATCH 308/689] Update src/Interpreters/ExternalDictionariesLoader.h

---
 src/Interpreters/ExternalDictionariesLoader.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/ExternalDictionariesLoader.h b/src/Interpreters/ExternalDictionariesLoader.h
index 5635a307816..717aefddd8f 100644
--- a/src/Interpreters/ExternalDictionariesLoader.h
+++ b/src/Interpreters/ExternalDictionariesLoader.h
@@ -4,7 +4,7 @@
 #include <Interpreters/Context_fwd.h>
 #include <Interpreters/ExternalLoader.h>
 #include <Common/quoteString.h>
-#include "Storages/ColumnsDescription.h"
+#include <Storages/ColumnsDescription.h>
 
 #include <memory>
 

From 82897d83fc7d405a08a005e982dcc04bc337714c Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Mon, 8 May 2023 14:22:52 +0000
Subject: [PATCH 309/689] Add CI check with enabled analyzer in master

---
 .github/workflows/master.yml | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 1182481c897..f0741b5465f 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -1341,6 +1341,40 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
+  FunctionalStatelessTestReleaseAnalyzer:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/stateless_analyzer
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Stateless tests (release, analyzer)
+          REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse
+          KILL_TIMEOUT=10800
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Functional test
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
   FunctionalStatelessTestAarch64:
     needs: [BuilderDebAarch64]
     runs-on: [self-hosted, func-tester-aarch64]

From 4b3b9f6ba614a5d605553cf312a7d155f16cfe49 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Mon, 8 May 2023 18:53:59 +0300
Subject: [PATCH 310/689] Disable 01710_projection_aggregation_in_order.sql
 (#49667)

* Update 01710_projection_aggregation_in_order.sql

* Update 02516_projections_with_rollup.sql

* Update 02516_projections_with_rollup.sql
---
 .../0_stateless/01710_projection_aggregation_in_order.sql      | 2 ++
 tests/queries/0_stateless/02516_projections_with_rollup.sql    | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql b/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql
index c7ed91eb19b..f9de1275dbd 100644
--- a/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql
+++ b/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql
@@ -1,3 +1,5 @@
+-- Tags: disabled
+-- FIXME https://github.com/ClickHouse/ClickHouse/issues/49552
 -- Test that check the correctness of the result for optimize_aggregation_in_order and projections,
 -- not that this optimization will take place.
 
diff --git a/tests/queries/0_stateless/02516_projections_with_rollup.sql b/tests/queries/0_stateless/02516_projections_with_rollup.sql
index e670fbb7827..038caf59264 100644
--- a/tests/queries/0_stateless/02516_projections_with_rollup.sql
+++ b/tests/queries/0_stateless/02516_projections_with_rollup.sql
@@ -1,3 +1,6 @@
+-- Tags: disabled
+-- FIXME https://github.com/ClickHouse/ClickHouse/issues/49552
+
 DROP TABLE IF EXISTS video_log;
 DROP TABLE IF EXISTS video_log_result__fuzz_0;
 DROP TABLE IF EXISTS rng;

From b320527158e3318dd28d9eaca4b31178d8a05a34 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 8 May 2023 16:33:39 +0000
Subject: [PATCH 311/689] Fix assert in SpanHolder::finish() with fibers

---
 contrib/boost                            |  2 +-
 contrib/boost-cmake/CMakeLists.txt       | 23 ++++++++++
 src/CMakeLists.txt                       |  3 ++
 src/Common/OpenTelemetryTraceContext.cpp | 55 +++++++++++++++---------
 4 files changed, 62 insertions(+), 21 deletions(-)

diff --git a/contrib/boost b/contrib/boost
index 8fe7b3326ef..d6c95434acb 160000
--- a/contrib/boost
+++ b/contrib/boost
@@ -1 +1 @@
-Subproject commit 8fe7b3326ef482ee6ecdf5a4f698f2b8c2780f98
+Subproject commit d6c95434acbb1a02d0b9de52bf4f37cac6c00328
diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt
index c9a759eab9c..ae20568f386 100644
--- a/contrib/boost-cmake/CMakeLists.txt
+++ b/contrib/boost-cmake/CMakeLists.txt
@@ -151,6 +151,7 @@ add_library (_boost_context ${SRCS_CONTEXT})
 add_library (boost::context ALIAS _boost_context)
 target_include_directories (_boost_context PRIVATE ${LIBRARY_DIR})
 
+
 if (SANITIZE OR BOOST_USE_UCONTEXT)
     target_compile_definitions(_boost_context PUBLIC BOOST_USE_UCONTEXT)
 endif()
@@ -161,6 +162,28 @@ elseif (SANITIZE STREQUAL "thread")
     target_compile_definitions(_boost_context PUBLIC BOOST_USE_TSAN)
 endif()
 
+# fiber
+
+set (SRCS_FIBER
+    "${LIBRARY_DIR}/libs/fiber/src/context.cpp"
+    "${LIBRARY_DIR}/libs/fiber/src/fiber.cpp"
+    "${LIBRARY_DIR}/libs/fiber/src/barrier.cpp"
+    "${LIBRARY_DIR}/libs/fiber/src/condition_variable.cpp"
+    "${LIBRARY_DIR}/libs/fiber/src/future.cpp"
+    "${LIBRARY_DIR}/libs/fiber/src/mutex.cpp"
+    "${LIBRARY_DIR}/libs/fiber/src/properties.cpp"
+    "${LIBRARY_DIR}/libs/fiber/src/recursive_mutex.cpp"
+    "${LIBRARY_DIR}/libs/fiber/src/recursive_timed_mutex.cpp"
+    "${LIBRARY_DIR}/libs/fiber/src/scheduler.cpp"
+    "${LIBRARY_DIR}/libs/fiber/src/timed_mutex.cpp"
+    "${LIBRARY_DIR}/libs/fiber/src/waker.cpp"
+    "${LIBRARY_DIR}/libs/fiber/src/algo/round_robin.cpp"
+)
+
+add_library (_boost_fiber ${SRCS_FIBER})
+add_library (boost::fiber ALIAS _boost_fiber)
+target_include_directories (_boost_fiber PRIVATE ${LIBRARY_DIR})
+
 # coroutine
 
 set (SRCS_COROUTINE
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 76a67ade99c..c69ac885154 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -547,6 +547,9 @@ endif ()
 target_link_libraries(clickhouse_common_io PUBLIC boost::context)
 dbms_target_link_libraries(PUBLIC boost::context)
 
+target_link_libraries(clickhouse_common_io PUBLIC boost::fiber)
+dbms_target_link_libraries(PUBLIC boost::fiber)
+
 if (ENABLE_NLP)
     dbms_target_link_libraries (PUBLIC ch_contrib::stemmer)
     dbms_target_link_libraries (PUBLIC ch_contrib::wnb)
diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp
index 0d89c581318..1c75bd3efaf 100644
--- a/src/Common/OpenTelemetryTraceContext.cpp
+++ b/src/Common/OpenTelemetryTraceContext.cpp
@@ -6,13 +6,26 @@
 #include <base/hex.h>
 #include <Core/Settings.h>
 #include <IO/Operators.h>
+#include <boost/fiber/fss.hpp>
 
 namespace DB
 {
 namespace OpenTelemetry
 {
 
-thread_local TracingContextOnThread current_thread_trace_context;
+static TracingContextOnThread & getCurrentThreadTraceContext()
+{
+    static boost::fibers::fiber_specific_ptr<TracingContextOnThread> current_thread_trace_context;
+    
+    auto * ptr = current_thread_trace_context.get();
+    if (unlikely(!ptr))
+    {
+        ptr = new TracingContextOnThread();
+        current_thread_trace_context.reset(ptr);
+    }
+    return *ptr;
+}
+
 
 bool Span::addAttribute(std::string_view name, UInt64 value) noexcept
 {
@@ -104,7 +117,7 @@ bool Span::addAttributeImpl(std::string_view name, std::string_view value) noexc
 
 SpanHolder::SpanHolder(std::string_view _operation_name, SpanKind _kind)
 {
-    if (!current_thread_trace_context.isTraceEnabled())
+    if (!getCurrentThreadTraceContext().isTraceEnabled())
     {
         return;
     }
@@ -112,8 +125,8 @@ SpanHolder::SpanHolder(std::string_view _operation_name, SpanKind _kind)
     /// Use try-catch to make sure the ctor is exception safe.
     try
     {
-        this->trace_id = current_thread_trace_context.trace_id;
-        this->parent_span_id = current_thread_trace_context.span_id;
+        this->trace_id = getCurrentThreadTraceContext().trace_id;
+        this->parent_span_id = getCurrentThreadTraceContext().span_id;
         this->span_id = thread_local_rng(); // create a new id for this span
         this->operation_name = _operation_name;
         this->kind = _kind;
@@ -132,7 +145,7 @@ SpanHolder::SpanHolder(std::string_view _operation_name, SpanKind _kind)
     }
 
     /// Set current span as parent of other spans created later on this thread.
-    current_thread_trace_context.span_id = this->span_id;
+    getCurrentThreadTraceContext().span_id = this->span_id;
 }
 
 void SpanHolder::finish() noexcept
@@ -141,12 +154,12 @@ void SpanHolder::finish() noexcept
         return;
 
     // First of all, restore old value of current span.
-    assert(current_thread_trace_context.span_id == span_id);
-    current_thread_trace_context.span_id = parent_span_id;
+    assert(getCurrentThreadTraceContext().span_id == span_id);
+    getCurrentThreadTraceContext().span_id = parent_span_id;
 
     try
     {
-        auto log = current_thread_trace_context.span_log.lock();
+        auto log = getCurrentThreadTraceContext().span_log.lock();
 
         /// The log might be disabled, check it before use
         if (log)
@@ -269,7 +282,7 @@ void TracingContext::serialize(WriteBuffer & buf) const
 
 const TracingContextOnThread & CurrentContext()
 {
-    return current_thread_trace_context;
+    return getCurrentThreadTraceContext();
 }
 
 void TracingContextOnThread::reset() noexcept
@@ -291,7 +304,7 @@ TracingContextHolder::TracingContextHolder(
     /// If any exception is raised during the construction, the tracing is not enabled on current thread.
     try
     {
-        if (current_thread_trace_context.isTraceEnabled())
+        if (getCurrentThreadTraceContext().isTraceEnabled())
         {
             ///
             /// This is not the normal case,
@@ -304,15 +317,15 @@ TracingContextHolder::TracingContextHolder(
             /// So this branch ensures this class can be instantiated multiple times on one same thread safely.
             ///
             this->is_context_owner = false;
-            this->root_span.trace_id = current_thread_trace_context.trace_id;
-            this->root_span.parent_span_id = current_thread_trace_context.span_id;
+            this->root_span.trace_id = getCurrentThreadTraceContext().trace_id;
+            this->root_span.parent_span_id = getCurrentThreadTraceContext().span_id;
             this->root_span.span_id = thread_local_rng();
             this->root_span.operation_name = _operation_name;
             this->root_span.start_time_us
                 = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
 
             /// Set the root span as parent of other spans created on current thread
-            current_thread_trace_context.span_id = this->root_span.span_id;
+            getCurrentThreadTraceContext().span_id = this->root_span.span_id;
             return;
         }
 
@@ -356,10 +369,10 @@ TracingContextHolder::TracingContextHolder(
     }
 
     /// Set up trace context on current thread only when the root span is successfully initialized.
-    current_thread_trace_context = _parent_trace_context;
-    current_thread_trace_context.span_id = this->root_span.span_id;
-    current_thread_trace_context.trace_flags = TRACE_FLAG_SAMPLED;
-    current_thread_trace_context.span_log = _span_log;
+    getCurrentThreadTraceContext() = _parent_trace_context;
+    getCurrentThreadTraceContext().span_id = this->root_span.span_id;
+    getCurrentThreadTraceContext().trace_flags = TRACE_FLAG_SAMPLED;
+    getCurrentThreadTraceContext().span_log = _span_log;
 }
 
 TracingContextHolder::~TracingContextHolder()
@@ -371,7 +384,7 @@ TracingContextHolder::~TracingContextHolder()
 
     try
     {
-        auto shared_span_log = current_thread_trace_context.span_log.lock();
+        auto shared_span_log = getCurrentThreadTraceContext().span_log.lock();
         if (shared_span_log)
         {
             try
@@ -402,12 +415,14 @@ TracingContextHolder::~TracingContextHolder()
     if (this->is_context_owner)
     {
         /// Clear the context on current thread
-        current_thread_trace_context.reset();
+        getCurrentThreadTraceContext().reset();
     }
     else
     {
-        current_thread_trace_context.span_id = this->root_span.parent_span_id;
+        getCurrentThreadTraceContext().span_id = this->root_span.parent_span_id;
     }
+
+
 }
 
 }

From 5527d43a5d09f9a9e75d3f9b94bd8ef1bec9980a Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 8 May 2023 16:51:17 +0000
Subject: [PATCH 312/689] Use only needed src files

---
 contrib/boost-cmake/CMakeLists.txt | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt
index ae20568f386..6c722c42e7d 100644
--- a/contrib/boost-cmake/CMakeLists.txt
+++ b/contrib/boost-cmake/CMakeLists.txt
@@ -167,15 +167,7 @@ endif()
 set (SRCS_FIBER
     "${LIBRARY_DIR}/libs/fiber/src/context.cpp"
     "${LIBRARY_DIR}/libs/fiber/src/fiber.cpp"
-    "${LIBRARY_DIR}/libs/fiber/src/barrier.cpp"
-    "${LIBRARY_DIR}/libs/fiber/src/condition_variable.cpp"
-    "${LIBRARY_DIR}/libs/fiber/src/future.cpp"
-    "${LIBRARY_DIR}/libs/fiber/src/mutex.cpp"
-    "${LIBRARY_DIR}/libs/fiber/src/properties.cpp"
-    "${LIBRARY_DIR}/libs/fiber/src/recursive_mutex.cpp"
-    "${LIBRARY_DIR}/libs/fiber/src/recursive_timed_mutex.cpp"
     "${LIBRARY_DIR}/libs/fiber/src/scheduler.cpp"
-    "${LIBRARY_DIR}/libs/fiber/src/timed_mutex.cpp"
     "${LIBRARY_DIR}/libs/fiber/src/waker.cpp"
     "${LIBRARY_DIR}/libs/fiber/src/algo/round_robin.cpp"
 )

From c961e3706ed1b4028f7420bd2fa8ac96126f378d Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 8 May 2023 16:52:57 +0000
Subject: [PATCH 313/689] Clean up

---
 contrib/boost-cmake/CMakeLists.txt       | 1 -
 src/Common/OpenTelemetryTraceContext.cpp | 4 +---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt
index 6c722c42e7d..c8be40be1d4 100644
--- a/contrib/boost-cmake/CMakeLists.txt
+++ b/contrib/boost-cmake/CMakeLists.txt
@@ -151,7 +151,6 @@ add_library (_boost_context ${SRCS_CONTEXT})
 add_library (boost::context ALIAS _boost_context)
 target_include_directories (_boost_context PRIVATE ${LIBRARY_DIR})
 
-
 if (SANITIZE OR BOOST_USE_UCONTEXT)
     target_compile_definitions(_boost_context PUBLIC BOOST_USE_UCONTEXT)
 endif()
diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp
index 1c75bd3efaf..86ce30941a3 100644
--- a/src/Common/OpenTelemetryTraceContext.cpp
+++ b/src/Common/OpenTelemetryTraceContext.cpp
@@ -16,7 +16,7 @@ namespace OpenTelemetry
 static TracingContextOnThread & getCurrentThreadTraceContext()
 {
     static boost::fibers::fiber_specific_ptr<TracingContextOnThread> current_thread_trace_context;
-    
+
     auto * ptr = current_thread_trace_context.get();
     if (unlikely(!ptr))
     {
@@ -421,8 +421,6 @@ TracingContextHolder::~TracingContextHolder()
     {
         getCurrentThreadTraceContext().span_id = this->root_span.parent_span_id;
     }
-
-
 }
 
 }

From c2a9b63f1f76aecdc611e69a6f6cf33cef45a2c9 Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Mon, 8 May 2023 19:30:25 +0200
Subject: [PATCH 314/689] Fix flaky 02684_bson.sql

---
 tests/queries/0_stateless/02684_bson.sql | Bin 9049 -> 9071 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/tests/queries/0_stateless/02684_bson.sql b/tests/queries/0_stateless/02684_bson.sql
index 577bd4ffd27dcad1ece67fe85757c10cf5528e56..cab5600eff0d8da33636fb948a5d1fb0d6d74048 100644
GIT binary patch
delta 33
ocmccV_TFtnn6jjfg0rKapMOYvkgKChyrYj#yo+Oqqe3kg0LFm|c>n+a

delta 11
ScmaFwcGGP`m@>0MEf)YE&;%#|


From 80692caa4c30eee1e2bf974547700f661619cd50 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Mon, 8 May 2023 19:33:53 +0200
Subject: [PATCH 315/689] Update src/Access/Common/AccessType.h

Co-authored-by: Sergei Trifonov <svtrifonov@gmail.com>
---
 src/Access/Common/AccessType.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h
index 1024d904c03..3ce55b98b1f 100644
--- a/src/Access/Common/AccessType.h
+++ b/src/Access/Common/AccessType.h
@@ -185,7 +185,7 @@ enum class AccessType
     M(SYSTEM_FLUSH, "", GROUP, SYSTEM) \
     M(SYSTEM_THREAD_FUZZER, "SYSTEM START THREAD FUZZER, SYSTEM STOP THREAD FUZZER, START THREAD FUZZER, STOP THREAD FUZZER", GLOBAL, SYSTEM) \
     M(SYSTEM_UNFREEZE, "SYSTEM UNFREEZE", GLOBAL, SYSTEM) \
-    M(SYSTEM_FAILPOINT, "SYSTEM ENABLE/DISABLE FAILPOINT", GLOBAL, SYSTEM) \
+    M(SYSTEM_FAILPOINT, "SYSTEM ENABLE FAILPOINT, SYSTEM DISABLE FAILPOINT", GLOBAL, SYSTEM) \
     M(SYSTEM, "", GROUP, ALL) /* allows to execute SYSTEM {SHUTDOWN|RELOAD CONFIG|...} */ \
     \
     M(dictGet, "dictHas, dictGetHierarchy, dictIsIn", DICTIONARY, ALL) /* allows to execute functions dictGet(), dictHas(), dictGetHierarchy(), dictIsIn() */\

From e7fff74797c97e0b1e9325726911513c7d201ccd Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Mon, 8 May 2023 17:45:42 +0000
Subject: [PATCH 316/689] Cleanup

---
 src/Interpreters/Cache/FileSegment.h |  2 +-
 src/Interpreters/Cache/Guards.h      | 22 ++++------------------
 src/Interpreters/Cache/Metadata.cpp  | 14 +++-----------
 3 files changed, 8 insertions(+), 30 deletions(-)

diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h
index 60883631177..163a15fcfda 100644
--- a/src/Interpreters/Cache/FileSegment.h
+++ b/src/Interpreters/Cache/FileSegment.h
@@ -61,7 +61,7 @@ struct CreateFileSegmentSettings
         : kind(kind_), unbounded(unbounded_) {}
 };
 
-class FileSegment : private boost::noncopyable, public std::enable_shared_from_this<FileSegment>
+class FileSegment : private boost::noncopyable
 {
 friend struct LockedKey;
 friend class FileCache; /// Because of reserved_size in tryReserve().
diff --git a/src/Interpreters/Cache/Guards.h b/src/Interpreters/Cache/Guards.h
index 0e06495bd82..66382d82003 100644
--- a/src/Interpreters/Cache/Guards.h
+++ b/src/Interpreters/Cache/Guards.h
@@ -1,8 +1,6 @@
 #pragma once
 #include <mutex>
-#include <Interpreters/Cache/FileCache_fwd.h>
 #include <boost/noncopyable.hpp>
-#include <map>
 
 namespace DB
 {
@@ -63,10 +61,7 @@ namespace DB
  */
 struct CacheGuard : private boost::noncopyable
 {
-    struct Lock : public std::unique_lock<std::mutex>
-    {
-        explicit Lock(std::mutex & mutex_) : std::unique_lock<std::mutex>(mutex_) {}
-    };
+    using Lock = std::unique_lock<std::mutex>;
 
     Lock lock() { return Lock(mutex); }
     std::mutex mutex;
@@ -77,10 +72,7 @@ struct CacheGuard : private boost::noncopyable
  */
 struct CacheMetadataGuard : private boost::noncopyable
 {
-    struct Lock : public std::unique_lock<std::mutex>
-    {
-        explicit Lock(std::mutex & mutex_) : std::unique_lock<std::mutex>(mutex_) {}
-    };
+    using Lock = std::unique_lock<std::mutex>;
 
     Lock lock() { return Lock(mutex); }
     std::mutex mutex;
@@ -91,10 +83,7 @@ struct CacheMetadataGuard : private boost::noncopyable
  */
 struct KeyGuard : private boost::noncopyable
 {
-    struct Lock : public std::unique_lock<std::mutex>
-    {
-        explicit Lock(std::mutex & mutex_) : std::unique_lock<std::mutex>(mutex_) {}
-    };
+    using Lock = std::unique_lock<std::mutex>;
 
     Lock lock() { return Lock(mutex); }
     std::mutex mutex;
@@ -105,10 +94,7 @@ struct KeyGuard : private boost::noncopyable
  */
 struct FileSegmentGuard : private boost::noncopyable
 {
-    struct Lock : public std::unique_lock<std::mutex>
-    {
-        explicit Lock(std::mutex & mutex_) : std::unique_lock<std::mutex>(mutex_) {}
-    };
+    using Lock = std::unique_lock<std::mutex>;
 
     Lock lock() { return Lock(mutex); }
     std::mutex mutex;
diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp
index 02ec7fdbefa..1aa13d4e98c 100644
--- a/src/Interpreters/Cache/Metadata.cpp
+++ b/src/Interpreters/Cache/Metadata.cpp
@@ -208,10 +208,9 @@ LockedKeyPtr CacheMetadata::lockKeyMetadata(
         chassert(key_not_found_policy == KeyNotFoundPolicy::CREATE_EMPTY);
     }
 
-    /// Not we are at a case:
-    /// key_state == KeyMetadata::KeyState::REMOVED
-    /// and KeyNotFoundPolicy == CREATE_EMPTY
-    /// Retry.
+    /// Now we are at the case when the key was removed (key_state == KeyMetadata::KeyState::REMOVED)
+    /// but we need to return empty key (key_not_found_policy == KeyNotFoundPolicy::CREATE_EMPTY)
+    /// Retry
     return lockKeyMetadata(key, key_not_found_policy);
 }
 
@@ -241,13 +240,6 @@ void CacheMetadata::doCleanup()
 {
     auto lock = guard.lock();
 
-    /// Let's mention this case.
-    /// This metadata cleanup is delayed so what is we marked key as deleted and
-    /// put it to deletion queue, but then the same key was added to cache before
-    /// we actually performed this delayed removal?
-    /// In this case it will work fine because on each attempt to add any key to cache
-    /// we perform this delayed removal.
-
     FileCacheKey cleanup_key;
     while (cleanup_queue->tryPop(cleanup_key))
     {

From 7250e7fec863286b010484dd0a13a0812168ff60 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Mon, 8 May 2023 19:58:43 +0200
Subject: [PATCH 317/689] correctly update log pointer during replica recovery

---
 src/Databases/DatabaseReplicated.cpp | 2 +-
 src/Databases/DatabaseReplicated.h   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index 0c12b78738d..2827ec0ce77 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -726,7 +726,7 @@ static UUID getTableUUIDIfReplicated(const String & metadata, ContextPtr context
     return create.uuid;
 }
 
-void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeeper, UInt32 our_log_ptr, UInt32 max_log_ptr)
+void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeeper, UInt32 our_log_ptr, UInt32 & max_log_ptr)
 {
     is_recovering = true;
     SCOPE_EXIT({ is_recovering = false; });
diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h
index b3397a832f2..1da181de030 100644
--- a/src/Databases/DatabaseReplicated.h
+++ b/src/Databases/DatabaseReplicated.h
@@ -102,7 +102,7 @@ private:
 
     void checkQueryValid(const ASTPtr & query, ContextPtr query_context) const;
 
-    void recoverLostReplica(const ZooKeeperPtr & current_zookeeper, UInt32 our_log_ptr, UInt32 max_log_ptr);
+    void recoverLostReplica(const ZooKeeperPtr & current_zookeeper, UInt32 our_log_ptr, UInt32 & max_log_ptr);
     std::map<String, String> tryGetConsistentMetadataSnapshot(const ZooKeeperPtr & zookeeper, UInt32 & max_log_ptr);
 
     ASTPtr parseQueryFromMetadataInZooKeeper(const String & node_name, const String & query);

From bb68a1582c9788660c67097b422be8b6ba705be8 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Mon, 8 May 2023 18:12:22 +0000
Subject: [PATCH 318/689] Enable distinct in order after fix #49636

---
 src/Core/Settings.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 1bea2c26392..c4af1cf73ec 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -734,7 +734,7 @@ class IColumn;
     M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \
     M(UInt64, grace_hash_join_initial_buckets, 1, "Initial number of grace hash join buckets", 0) \
     M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \
-    M(Bool, optimize_distinct_in_order, false, "This optimization has a bug and it is disabled. Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
+    M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
     M(Bool, optimize_sorting_by_input_stream_properties, true, "Optimize sorting by sorting properties of input stream", 0) \
     M(UInt64, insert_keeper_max_retries, 20, "Max retries for keeper operations during insert", 0) \
     M(UInt64, insert_keeper_retry_initial_backoff_ms, 100, "Initial backoff timeout for keeper operations during insert", 0) \

From 903021e896e88f8c5c144caf3de55df1cd523a0f Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Mon, 8 May 2023 18:29:47 +0000
Subject: [PATCH 319/689] Fix review comment

---
 src/Interpreters/Cache/Guards.h | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/Interpreters/Cache/Guards.h b/src/Interpreters/Cache/Guards.h
index 66382d82003..09586b55c61 100644
--- a/src/Interpreters/Cache/Guards.h
+++ b/src/Interpreters/Cache/Guards.h
@@ -61,7 +61,12 @@ namespace DB
  */
 struct CacheGuard : private boost::noncopyable
 {
-    using Lock = std::unique_lock<std::mutex>;
+    /// struct is used (not keyword `using`) to make CacheGuard::Lock non-interchangable with other guards locks
+    /// so, we wouldn't be able to pass CacheGuard::Lock to a function which accepts KeyGuard::Lock, for example
+    struct Lock : public std::unique_lock<std::mutex>
+    {
+        explicit Lock(std::mutex & mutex_) : std::unique_lock<std::mutex>(mutex_) {}
+    };
 
     Lock lock() { return Lock(mutex); }
     std::mutex mutex;
@@ -72,7 +77,10 @@ struct CacheGuard : private boost::noncopyable
  */
 struct CacheMetadataGuard : private boost::noncopyable
 {
-    using Lock = std::unique_lock<std::mutex>;
+    struct Lock : public std::unique_lock<std::mutex>
+    {
+        explicit Lock(std::mutex & mutex_) : std::unique_lock<std::mutex>(mutex_) {}
+    };
 
     Lock lock() { return Lock(mutex); }
     std::mutex mutex;
@@ -83,7 +91,10 @@ struct CacheMetadataGuard : private boost::noncopyable
  */
 struct KeyGuard : private boost::noncopyable
 {
-    using Lock = std::unique_lock<std::mutex>;
+    struct Lock : public std::unique_lock<std::mutex>
+    {
+        explicit Lock(std::mutex & mutex_) : std::unique_lock<std::mutex>(mutex_) {}
+    };
 
     Lock lock() { return Lock(mutex); }
     std::mutex mutex;
@@ -94,7 +105,10 @@ struct KeyGuard : private boost::noncopyable
  */
 struct FileSegmentGuard : private boost::noncopyable
 {
-    using Lock = std::unique_lock<std::mutex>;
+    struct Lock : public std::unique_lock<std::mutex>
+    {
+        explicit Lock(std::mutex & mutex_) : std::unique_lock<std::mutex>(mutex_) {}
+    };
 
     Lock lock() { return Lock(mutex); }
     std::mutex mutex;

From c525d132727298ccdc7f9c651a79f3c287e72bc8 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 8 May 2023 22:35:09 +0200
Subject: [PATCH 320/689] Enable back thread fuzzuer under TSan

Since initial issue got resolved [1].

  [1]: https://github.com/google/sanitizers/issues/1540

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docker/test/stress/run.sh | 38 +++++++++++++++++---------------------
 1 file changed, 17 insertions(+), 21 deletions(-)

diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh
index bfad2c9a7c5..4926967d2d2 100644
--- a/docker/test/stress/run.sh
+++ b/docker/test/stress/run.sh
@@ -20,31 +20,27 @@ install_packages package_folder
 
 # Thread Fuzzer allows to check more permutations of possible thread scheduling
 # and find more potential issues.
-# Temporarily disable ThreadFuzzer with tsan because of https://github.com/google/sanitizers/issues/1540
-is_tsan_build=$(clickhouse local -q "select value like '% -fsanitize=thread %' from system.build_options where name='CXX_FLAGS'")
-if [ "$is_tsan_build" -eq "0" ]; then
-    export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
-    export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
-    export THREAD_FUZZER_SLEEP_TIME_US=100000
+export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
+export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
+export THREAD_FUZZER_SLEEP_TIME_US=100000
 
-    export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
-    export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
-    export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_MIGRATE_PROBABILITY=1
-    export THREAD_FUZZER_pthread_mutex_unlock_AFTER_MIGRATE_PROBABILITY=1
+export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
+export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
+export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_MIGRATE_PROBABILITY=1
+export THREAD_FUZZER_pthread_mutex_unlock_AFTER_MIGRATE_PROBABILITY=1
 
-    export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
-    export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
-    export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
-    export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
-    export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
+export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
+export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
+export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
+export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
+export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
 
-    export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
-    export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
-    export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
+export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
+export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
+export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
 
-    export THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY=0.01
-    export THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY=0.01
-fi
+export THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY=0.01
+export THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY=0.01
 
 export ZOOKEEPER_FAULT_INJECTION=1
 # Initial run without S3 to create system.*_log on local file system to make it

From cf987db8f67016441e963b22b89beba9da756e6a Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 8 May 2023 22:38:02 +0200
Subject: [PATCH 321/689] Disable .debug_aranges workaround under ThinLTO for
 clang/llvm 16

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 cmake/tools.cmake | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/cmake/tools.cmake b/cmake/tools.cmake
index 8e6da1051bc..802907c9dda 100644
--- a/cmake/tools.cmake
+++ b/cmake/tools.cmake
@@ -70,13 +70,15 @@ if (LINKER_NAME)
     if (NOT LLD_PATH)
         message (FATAL_ERROR "Using linker ${LINKER_NAME} but can't find its path.")
     endif ()
-    if (COMPILER_CLANG)
-        # This a temporary quirk to emit .debug_aranges with ThinLTO, can be removed after upgrade to clang-16
+    # This a temporary quirk to emit .debug_aranges with ThinLTO, it is only the case clang/llvm <16
+    if (COMPILER_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16)
         set (LLD_WRAPPER "${CMAKE_CURRENT_BINARY_DIR}/ld.lld")
         configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ld.lld.in" "${LLD_WRAPPER}" @ONLY)
 
         set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LLD_WRAPPER}")
-    endif ()
+    else ()
+        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LLD_PATH}")
+    endif()
 
 endif ()
 

From e2780813442503a527f5f2a2f70a61486984d587 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 8 May 2023 22:39:49 +0200
Subject: [PATCH 322/689] Revert "Merge pull request #43195 from
 ClickHouse/workaround-for-llvm-bug"

With clang 16 this workaround should not be required anymore.

This reverts commit 992729f3c0cb00611f0e2c082cf996d6ea718a22, reversing
changes made to d885b3cc4c5514e4e8a090e085041e589060a03a.
---
 cmake/sanitize.cmake | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake
index bf5eddf09f5..0dd7fda4b45 100644
--- a/cmake/sanitize.cmake
+++ b/cmake/sanitize.cmake
@@ -10,9 +10,7 @@ set (SAN_FLAGS "${SAN_FLAGS} -g -fno-omit-frame-pointer -DSANITIZER")
 
 if (SANITIZE)
     if (SANITIZE STREQUAL "address")
-        # LLVM-15 has a bug in Address Sanitizer, preventing the usage of 'sanitize-address-use-after-scope',
-        # see https://github.com/llvm/llvm-project/issues/58633
-        set (ASAN_FLAGS "-fsanitize=address -fno-sanitize-address-use-after-scope")
+        set (ASAN_FLAGS "-fsanitize=address -fsanitize-address-use-after-scope")
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${ASAN_FLAGS}")
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${ASAN_FLAGS}")
 

From 82ce47ce5636c72a399996097c1167148708dd43 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Mon, 8 May 2023 23:02:36 +0200
Subject: [PATCH 323/689] address comments

---
 src/Common/FailPoint.cpp                      | 141 +++++++-----------
 src/Common/FailPoint.h                        |   8 +-
 src/Interpreters/InterpreterSystemQuery.cpp   |   4 +-
 .../MergeTree/ReplicatedMergeTreeSink.cpp     |   4 +-
 .../02718_insert_meet_hardware_error.sql      |   4 +-
 5 files changed, 61 insertions(+), 100 deletions(-)

diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp
index 1048b0ecd2c..85cc6853fdc 100644
--- a/src/Common/FailPoint.cpp
+++ b/src/Common/FailPoint.cpp
@@ -2,6 +2,7 @@
 #include <Common/FailPoint.h>
 
 #include <boost/core/noncopyable.hpp>
+#include <chrono>
 #include <condition_variable>
 #include <mutex>
 #include <optional>
@@ -24,50 +25,38 @@ static struct InitFiu
 } init_fiu;
 #endif
 
-#define APPLY_FOR_FAILPOINTS_ONCE(M) \
-    M(rmt_commit_zk_fail_after_op)
-
-#define APPLY_FOR_FAILPOINTS(M) \
-    M(dummy_failpoint)
-
-#define APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M) \
-    M(dummy_pausable_failpoint_once)
-
-#define APPLY_FOR_PAUSEABLE_FAILPOINTS(M) \
-    M(dummy_pausable_failpoint)
+#define APPLY_FOR_FAILPOINTS(ONCE, REGULAR, PAUSEABLE_ONCE, PAUSEABLE) \
+    ONCE(replicated_merge_tree_commit_zk_fail_after_op) \
+    REGULAR(dummy_failpoint) \
+    PAUSEABLE_ONCE(dummy_pausable_failpoint_once) \
+    PAUSEABLE(dummy_pausable_failpoint)
 
 namespace FailPoints
 {
 #define M(NAME) extern const char(NAME)[] = #NAME "";
-APPLY_FOR_FAILPOINTS_ONCE(M)
-APPLY_FOR_FAILPOINTS(M)
-APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M)
-APPLY_FOR_PAUSEABLE_FAILPOINTS(M)
+APPLY_FOR_FAILPOINTS(M, M, M, M)
 #undef M
 }
 
-std::unordered_map<String, std::any> FailPointHelper::fail_point_val;
-std::unordered_map<String, std::shared_ptr<FailPointChannel>> FailPointHelper::fail_point_wait_channels;
+std::unordered_map<String, std::shared_ptr<FailPointChannel>> FailPointInjection::fail_point_wait_channels;
+std::mutex FailPointInjection::mu;
 class FailPointChannel : private boost::noncopyable
 {
 public:
-    // wake up all waiting threads when destroy
-    ~FailPointChannel() { notifyAll(); }
-
     explicit FailPointChannel(UInt64 timeout_)
-        : timeout(timeout_)
+        : timeout_ms(timeout_)
     {}
     FailPointChannel()
-        : timeout(0)
+        : timeout_ms(0)
     {}
 
     void wait()
     {
         std::unique_lock lock(m);
-        if (timeout == 0)
+        if (timeout_ms == 0)
             cv.wait(lock);
         else
-            cv.wait_for(lock, std::chrono::seconds(timeout));
+            cv.wait_for(lock, std::chrono::milliseconds(timeout_ms));
     }
 
     void notifyAll()
@@ -77,95 +66,69 @@ public:
     }
 
 private:
-    UInt64 timeout;
+    UInt64 timeout_ms;
     std::mutex m;
     std::condition_variable cv;
 };
 
-void FailPointHelper::enablePauseFailPoint(const String & fail_point_name, UInt64 time)
+void FailPointInjection::enablePauseFailPoint(const String & fail_point_name, UInt64 time_ms)
 {
 #define SUB_M(NAME, flags)                                                                                  \
     if (fail_point_name == FailPoints::NAME)                                                                \
     {                                                                                                       \
         /* FIU_ONETIME -- Only fail once; the point of failure will be automatically disabled afterwards.*/ \
         fiu_enable(FailPoints::NAME, 1, nullptr, flags);                                                    \
-        fail_point_wait_channels.try_emplace(FailPoints::NAME, std::make_shared<FailPointChannel>(time));   \
+        std::lock_guard lock(mu);                                                                           \
+        fail_point_wait_channels.try_emplace(FailPoints::NAME, std::make_shared<FailPointChannel>(time_ms));   \
         return;                                                                                             \
     }
-
-#define M(NAME) SUB_M(NAME, FIU_ONETIME)
-    APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M)
-#undef M
-
-#define M(NAME) SUB_M(NAME, 0)
-    APPLY_FOR_PAUSEABLE_FAILPOINTS(M)
-#undef M
+#define ONCE(NAME)
+#define REGULAR(NAME)
+#define PAUSEABLE_ONCE(NAME) SUB_M(NAME, FIU_ONETIME)
+#define PAUSEABLE(NAME) SUB_M(NAME, 0)
+    APPLY_FOR_FAILPOINTS(ONCE, REGULAR, PAUSEABLE_ONCE, PAUSEABLE)
 #undef SUB_M
+#undef ONCE
+#undef REGULAR
+#undef PAUSEABLE_ONCE
+#undef PAUSEABLE
 
     throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find fail point {}", fail_point_name);
 }
 
-void FailPointHelper::enableFailPoint(const String & fail_point_name, std::optional<std::any> v)
+void FailPointInjection::enableFailPoint(const String & fail_point_name)
 {
 #if FIU_ENABLE
-#define SUB_M(NAME, flags)                                                                                  \
-    if (fail_point_name == FailPoints::NAME)                                                                \
-    {                                                                                                       \
-        /* FIU_ONETIME -- Only fail once; the point of failure will be automatically disabled afterwards.*/ \
-        fiu_enable(FailPoints::NAME, 1, nullptr, flags);                                                    \
-        if (v.has_value())                                                                                  \
-        {                                                                                                   \
-            fail_point_val.try_emplace(FailPoints::NAME, v.value());                                        \
-        }                                                                                                   \
-        return;                                                                                             \
+#define SUB_M(NAME, flags, pause)                                                                               \
+    if (fail_point_name == FailPoints::NAME)                                                                    \
+    {                                                                                                           \
+        /* FIU_ONETIME -- Only fail once; the point of failure will be automatically disabled afterwards.*/     \
+        fiu_enable(FailPoints::NAME, 1, nullptr, flags);                                                        \
+        if (pause)                                                                                               \
+        {                                                                                                       \
+            std::lock_guard lock(mu);                                                                           \
+            fail_point_wait_channels.try_emplace(FailPoints::NAME, std::make_shared<FailPointChannel>());       \
+        }                                                                                                       \
+        return;                                                                                                 \
     }
-
-#define M(NAME) SUB_M(NAME, FIU_ONETIME)
-    APPLY_FOR_FAILPOINTS_ONCE(M)
-#undef M
-#define M(NAME) SUB_M(NAME, 0)
-    APPLY_FOR_FAILPOINTS(M)
-#undef M
-#undef SUB_M
-
-#define SUB_M(NAME, flags)                                                                                  \
-    if (fail_point_name == FailPoints::NAME)                                                                \
-    {                                                                                                       \
-        /* FIU_ONETIME -- Only fail once; the point of failure will be automatically disabled afterwards.*/ \
-        fiu_enable(FailPoints::NAME, 1, nullptr, flags);                                                    \
-        fail_point_wait_channels.try_emplace(FailPoints::NAME, std::make_shared<FailPointChannel>());       \
-        if (v.has_value())                                                                                  \
-        {                                                                                                   \
-            fail_point_val.try_emplace(FailPoints::NAME, v.value());                                        \
-        }                                                                                                   \
-        return;                                                                                             \
-    }
-
-#define M(NAME) SUB_M(NAME, FIU_ONETIME)
-    APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M)
-#undef M
-
-#define M(NAME) SUB_M(NAME, 0)
-    APPLY_FOR_PAUSEABLE_FAILPOINTS(M)
-#undef M
+#define ONCE(NAME) SUB_M(NAME, FIU_ONETIME, 0)
+#define REGULAR(NAME) SUB_M(NAME, 0, 0)
+#define PAUSEABLE_ONCE(NAME) SUB_M(NAME, FIU_ONETIME, 1)
+#define PAUSEABLE(NAME) SUB_M(NAME, 0, 1)
+    APPLY_FOR_FAILPOINTS(ONCE, REGULAR, PAUSEABLE_ONCE, PAUSEABLE)
 #undef SUB_M
+#undef ONCE
+#undef REGULAR
+#undef PAUSEABLE_ONCE
+#undef PAUSEABLE
 
 #endif
     throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find fail point {}", fail_point_name);
 }
 
-std::optional<std::any>
-FailPointHelper::getFailPointVal(const String & fail_point_name)
-{
-    if (auto iter = fail_point_val.find(fail_point_name); iter != fail_point_val.end())
-    {
-        return iter->second;
-    }
-    return std::nullopt;
-}
-
-void FailPointHelper::disableFailPoint(const String & fail_point_name)
+void FailPointInjection::disableFailPoint(const String & fail_point_name)
 {
+    std::lock_guard lock(mu);
     if (auto iter = fail_point_wait_channels.find(fail_point_name); iter != fail_point_wait_channels.end())
     {
         /// can not rely on deconstruction to do the notify_all things, because
@@ -173,12 +136,12 @@ void FailPointHelper::disableFailPoint(const String & fail_point_name)
         iter->second->notifyAll();
         fail_point_wait_channels.erase(iter);
     }
-    fail_point_val.erase(fail_point_name);
     fiu_disable(fail_point_name.c_str());
 }
 
-void FailPointHelper::wait(const String & fail_point_name)
+void FailPointInjection::wait(const String & fail_point_name)
 {
+    std::lock_guard lock(mu);
     if (auto iter = fail_point_wait_channels.find(fail_point_name); iter == fail_point_wait_channels.end())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Can not find channel for fail point {}", fail_point_name);
     else
@@ -186,6 +149,6 @@ void FailPointHelper::wait(const String & fail_point_name)
         auto ptr = iter->second;
         ptr->wait();
     }
-}
+};
 
 }
diff --git a/src/Common/FailPoint.h b/src/Common/FailPoint.h
index 86dbf70d9c2..c7505766907 100644
--- a/src/Common/FailPoint.h
+++ b/src/Common/FailPoint.h
@@ -27,12 +27,10 @@ namespace DB
 #define FAIL_POINT_PAUSE(fail_point) fiu_do_on(fail_point, FailPointHelper::wait(fail_point);)
 
 class FailPointChannel;
-class FailPointHelper
+class FailPointInjection
 {
 public:
-    static void enableFailPoint(const String & fail_point_name, std::optional<std::any> v = std::nullopt);
-
-    static std::optional<std::any> getFailPointVal(const String & fail_point_name);
+    static void enableFailPoint(const String & fail_point_name);
 
     static void enablePauseFailPoint(const String & fail_point_name, UInt64 time);
 
@@ -41,7 +39,7 @@ public:
     static void wait(const String & fail_point_name);
 
 private:
+    static std::mutex mu;
     static std::unordered_map<String, std::shared_ptr<FailPointChannel>> fail_point_wait_channels;
-    static std::unordered_map<String, std::any> fail_point_val;
 };
 }
diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp
index 8d69782756e..2185226be7e 100644
--- a/src/Interpreters/InterpreterSystemQuery.cpp
+++ b/src/Interpreters/InterpreterSystemQuery.cpp
@@ -586,13 +586,13 @@ BlockIO InterpreterSystemQuery::execute()
         case Type::ENABLE_FAILPOINT:
         {
             getContext()->checkAccess(AccessType::SYSTEM_FAILPOINT);
-            FailPointHelper::enableFailPoint(query.fail_point_name);
+            FailPointInjection::enableFailPoint(query.fail_point_name);
             break;
         }
         case Type::DISABLE_FAILPOINT:
         {
             getContext()->checkAccess(AccessType::SYSTEM_FAILPOINT);
-            FailPointHelper::disableFailPoint(query.fail_point_name);
+            FailPointInjection::disableFailPoint(query.fail_point_name);
             break;
         }
         default:
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index 4109194398d..9f97835612b 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -23,7 +23,7 @@ namespace DB
 
 namespace FailPoints
 {
-    extern const char rmt_commit_zk_fail_after_op[];
+    extern const char replicated_merge_tree_commit_zk_fail_after_op[];
 }
 
 namespace ErrorCodes
@@ -946,7 +946,7 @@ std::vector<String> ReplicatedMergeTreeSinkImpl<async_insert>::commitPart(
 
         ThreadFuzzer::maybeInjectSleep();
 
-        fiu_do_on(FailPoints::rmt_commit_zk_fail_after_op,
+        fiu_do_on(FailPoints::replicated_merge_tree_commit_zk_fail_after_op,
         {
             if (!zookeeper->fault_policy)
             {
diff --git a/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql b/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql
index 72d22d9d5b2..65354e0624a 100644
--- a/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql
+++ b/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql
@@ -9,12 +9,12 @@ ORDER BY (KeyID);
 
 insert into t_hardware_error values (1), (2), (3), (4), (5); 
 
-system enable failpoint rmt_commit_zk_fail_after_op;
+system enable failpoint replicated_merge_tree_commit_zk_fail_after_op;
 
 insert into t_hardware_error values (6), (7), (8), (9), (10); 
 
 select count() from t_hardware_error;
 
-system disable failpoint rmt_commit_zk_fail_after_op;
+system disable failpoint replicated_commit_zk_fail_after_op;
 
 DROP TABLE t_hardware_error NO DELAY;

From 70516f2f451fd9bdb82f6e3ee689772fce589a8e Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Mon, 8 May 2023 23:00:20 +0000
Subject: [PATCH 324/689] close pipe in run() if exception happens

---
 src/Interpreters/TraceCollector.cpp | 128 +++++++++++++++-------------
 src/Interpreters/TraceCollector.h   |   2 +
 2 files changed, 72 insertions(+), 58 deletions(-)

diff --git a/src/Interpreters/TraceCollector.cpp b/src/Interpreters/TraceCollector.cpp
index 0fc9aeb170b..b8ef7044781 100644
--- a/src/Interpreters/TraceCollector.cpp
+++ b/src/Interpreters/TraceCollector.cpp
@@ -29,6 +29,17 @@ TraceCollector::TraceCollector(std::shared_ptr<TraceLog> trace_log_)
     thread = ThreadFromGlobalPool(&TraceCollector::run, this);
 }
 
+void TraceCollector::tryClosePipe()
+{
+    try
+    {
+        TraceSender::pipe.close();
+    }
+    catch (...)
+    {
+        tryLogCurrentException("TraceCollector");
+    }
+}
 
 TraceCollector::~TraceCollector()
 {
@@ -54,14 +65,7 @@ TraceCollector::~TraceCollector()
         tryLogCurrentException("TraceCollector");
     }
 
-    try
-    {
-        TraceSender::pipe.close();
-    }
-    catch (...)
-    {
-        tryLogCurrentException("TraceCollector");
-    }
+    tryClosePipe();
 
     if (thread.joinable())
         thread.join();
@@ -74,60 +78,68 @@ void TraceCollector::run()
 
     ReadBufferFromFileDescriptor in(TraceSender::pipe.fds_rw[0]);
 
-    while (true)
+    try
     {
-        char is_last;
-        readChar(is_last, in);
-        if (is_last)
-            break;
-
-        std::string query_id;
-        UInt8 query_id_size = 0;
-        readBinary(query_id_size, in);
-        query_id.resize(query_id_size);
-        in.readStrict(query_id.data(), query_id_size);
-
-        UInt8 trace_size = 0;
-        readIntBinary(trace_size, in);
-
-        Array trace;
-        trace.reserve(trace_size);
-
-        for (size_t i = 0; i < trace_size; ++i)
+        while (true)
         {
-            uintptr_t addr = 0;
-            readPODBinary(addr, in);
-            trace.emplace_back(static_cast<UInt64>(addr));
-        }
-
-        TraceType trace_type;
-        readPODBinary(trace_type, in);
-
-        UInt64 thread_id;
-        readPODBinary(thread_id, in);
-
-        Int64 size;
-        readPODBinary(size, in);
-
-        ProfileEvents::Event event;
-        readPODBinary(event, in);
-
-        ProfileEvents::Count increment;
-        readPODBinary(increment, in);
-
-        if (trace_log)
-        {
-            // time and time_in_microseconds are both being constructed from the same timespec so that the
-            // times will be equal up to the precision of a second.
-            struct timespec ts;
-            clock_gettime(CLOCK_REALTIME, &ts);
-
-            UInt64 time = static_cast<UInt64>(ts.tv_sec * 1000000000LL + ts.tv_nsec);
-            UInt64 time_in_microseconds = static_cast<UInt64>((ts.tv_sec * 1000000LL) + (ts.tv_nsec / 1000));
-            TraceLogElement element{time_t(time / 1000000000), time_in_microseconds, time, trace_type, thread_id, query_id, trace, size, event, increment};
-            trace_log->add(element);
+            char is_last;
+            readChar(is_last, in);
+            if (is_last)
+                break;
+
+            std::string query_id;
+            UInt8 query_id_size = 0;
+            readBinary(query_id_size, in);
+            query_id.resize(query_id_size);
+            in.readStrict(query_id.data(), query_id_size);
+
+            UInt8 trace_size = 0;
+            readIntBinary(trace_size, in);
+
+            Array trace;
+            trace.reserve(trace_size);
+
+            for (size_t i = 0; i < trace_size; ++i)
+            {
+                uintptr_t addr = 0;
+                readPODBinary(addr, in);
+                trace.emplace_back(static_cast<UInt64>(addr));
+            }
+
+            TraceType trace_type;
+            readPODBinary(trace_type, in);
+
+            UInt64 thread_id;
+            readPODBinary(thread_id, in);
+
+            Int64 size;
+            readPODBinary(size, in);
+
+            ProfileEvents::Event event;
+            readPODBinary(event, in);
+
+            ProfileEvents::Count increment;
+            readPODBinary(increment, in);
+
+            if (trace_log)
+            {
+                // time and time_in_microseconds are both being constructed from the same timespec so that the
+                // times will be equal up to the precision of a second.
+                struct timespec ts;
+                clock_gettime(CLOCK_REALTIME, &ts);
+
+                UInt64 time = static_cast<UInt64>(ts.tv_sec * 1000000000LL + ts.tv_nsec);
+                UInt64 time_in_microseconds = static_cast<UInt64>((ts.tv_sec * 1000000LL) + (ts.tv_nsec / 1000));
+                TraceLogElement element{time_t(time / 1000000000), time_in_microseconds, time, trace_type, thread_id, query_id, trace, size, event, increment};
+                trace_log->add(element);
+            }
         }
     }
+    catch (...)
+    {
+        tryClosePipe();
+        throw;
+    }
 }
 
 }
diff --git a/src/Interpreters/TraceCollector.h b/src/Interpreters/TraceCollector.h
index d5bd09f1757..382e7511ac6 100644
--- a/src/Interpreters/TraceCollector.h
+++ b/src/Interpreters/TraceCollector.h
@@ -23,6 +23,8 @@ private:
     std::shared_ptr<TraceLog> trace_log;
     ThreadFromGlobalPool thread;
 
+    void tryClosePipe();
+
     void run();
 };
 

From a7e04b7576d5dc5cc2b7feb4a043eb4361d9eeb9 Mon Sep 17 00:00:00 2001
From: fhbai <fhbai@tencent.com>
Date: Tue, 9 May 2023 11:36:15 +0800
Subject: [PATCH 325/689] fix return type

---
 src/Functions/array/arrayDotProduct.cpp       | 59 +++++++++++--------
 src/Functions/array/arrayScalarProduct.h      | 22 +++++--
 .../0_stateless/02708_dot_product.reference   |  4 ++
 .../queries/0_stateless/02708_dot_product.sql | 23 +++++++-
 4 files changed, 76 insertions(+), 32 deletions(-)

diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp
index 0d21bae90e3..7aa9f1d49c7 100644
--- a/src/Functions/array/arrayDotProduct.cpp
+++ b/src/Functions/array/arrayDotProduct.cpp
@@ -1,7 +1,12 @@
 #include <DataTypes/DataTypesNumber.h>
 #include <Functions/FunctionFactory.h>
 #include <DataTypes/getLeastSupertype.h>
-#include "arrayScalarProduct.h"
+#include <Core/Types_fwd.h>
+#include <DataTypes/Serializations/ISerialization.h>
+#include <Functions/castTypeToEither.h>
+#include <Functions/array/arrayScalarProduct.h>
+#include <base/types.h>
+#include <Functions/FunctionBinaryArithmetic.h>
 
 
 namespace DB
@@ -20,31 +25,33 @@ struct NameArrayDotProduct
 class ArrayDotProductImpl
 {
 public:
-    static DataTypePtr getReturnType(const DataTypePtr & left_type, const DataTypePtr & right_type)
+    static DataTypePtr getReturnType(const DataTypePtr & left, const DataTypePtr & right)
     {
-        const auto & common_type = getLeastSupertype(DataTypes{left_type, right_type});
-        switch (common_type->getTypeId())
-        {
-            case TypeIndex::UInt8:
-            case TypeIndex::UInt16:
-            case TypeIndex::UInt32:
-            case TypeIndex::Int8:
-            case TypeIndex::Int16:
-            case TypeIndex::Int32:
-            case TypeIndex::UInt64:
-            case TypeIndex::Int64:
-            case TypeIndex::Float64:
-                return std::make_shared<DataTypeFloat64>();
-            case TypeIndex::Float32:
-                return std::make_shared<DataTypeFloat32>();
-           default:
-                throw Exception(
-                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "Arguments of function {} has nested type {}. "
-                    "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.",
-                    std::string(NameArrayDotProduct::name),
-                    common_type->getName());
-        }
+        using Types = TypeList<DataTypeFloat32, DataTypeFloat64,
+                               DataTypeUInt8, DataTypeUInt16, DataTypeUInt32, DataTypeUInt64,
+                               DataTypeInt8, DataTypeInt16, DataTypeInt32, DataTypeInt64>;
+
+        DataTypePtr result_type;
+        bool valid = castTypeToEither(Types{}, left.get(), [&](const auto & left_) {
+            return castTypeToEither(Types{}, right.get(), [&](const auto & right_) {
+                using LeftDataType = typename std::decay_t<decltype(left_)>::FieldType;
+                using RightDataType = typename std::decay_t<decltype(right_)>::FieldType;
+                using ResultType = typename NumberTraits::ResultOfAdditionMultiplication<LeftDataType, RightDataType>::Type;
+                if (std::is_same_v<LeftDataType, Float32> && std::is_same_v<RightDataType, Float32>)
+                    result_type = std::make_shared<DataTypeFloat32>();
+                else
+                    result_type = std::make_shared<DataTypeFromFieldType<ResultType>>();
+                return true;
+            });
+        });
+
+        if (!valid)
+            throw Exception(
+                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                "Arguments of function {} "
+                "only support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.",
+                std::string(NameArrayDotProduct::name));
+        return result_type;
     }
 
     template <typename ResultType, typename T, typename U>
@@ -67,6 +74,6 @@ REGISTER_FUNCTION(ArrayDotProduct)
     factory.registerFunction<FunctionArrayDotProduct>();
 }
 
-/// These functions are used by TupleOrArrayFunction in Function/vectorFunctions.cpp
+// These functions are used by TupleOrArrayFunction in Function/vectorFunctions.cpp
 FunctionPtr createFunctionArrayDotProduct(ContextPtr context_) { return FunctionArrayDotProduct::create(context_); }
 }
diff --git a/src/Functions/array/arrayScalarProduct.h b/src/Functions/array/arrayScalarProduct.h
index 5c36f2492c6..374a2d8a194 100644
--- a/src/Functions/array/arrayScalarProduct.h
+++ b/src/Functions/array/arrayScalarProduct.h
@@ -6,6 +6,7 @@
 #include <Functions/FunctionHelpers.h>
 #include <Functions/IFunction.h>
 #include <Interpreters/Context_fwd.h>
+#include <Core/TypeId.h>
 
 
 namespace DB
@@ -154,12 +155,23 @@ public:
     {
         switch (result_type->getTypeId())
         {
-            case TypeIndex::Float32:
-                return executeWithResultType<Float32>(arguments);
-                break;
-            case TypeIndex::Float64:
-                return executeWithResultType<Float64>(arguments);
+        #define SUPPORTED_TYPE(type) \
+            case TypeIndex::type: \
+                return executeWithResultType<type>(arguments); \
                 break;
+
+            SUPPORTED_TYPE(UInt8)
+            SUPPORTED_TYPE(UInt16)
+            SUPPORTED_TYPE(UInt32)
+            SUPPORTED_TYPE(UInt64)
+            SUPPORTED_TYPE(Int8)
+            SUPPORTED_TYPE(Int16)
+            SUPPORTED_TYPE(Int32)
+            SUPPORTED_TYPE(Int64)
+            SUPPORTED_TYPE(Float32)
+            SUPPORTED_TYPE(Float64)
+        #undef SUPPORTED_TYPE
+
             default:
                 throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected result type {}", result_type->getName());
         }
diff --git a/tests/queries/0_stateless/02708_dot_product.reference b/tests/queries/0_stateless/02708_dot_product.reference
index 7106b870fab..45e53871aa2 100644
--- a/tests/queries/0_stateless/02708_dot_product.reference
+++ b/tests/queries/0_stateless/02708_dot_product.reference
@@ -4,7 +4,11 @@
 376.5
 230
 0
+0
 Float64
 Float32
 Float64
 Float64
+UInt16
+UInt64
+Int64
diff --git a/tests/queries/0_stateless/02708_dot_product.sql b/tests/queries/0_stateless/02708_dot_product.sql
index 46450ae6394..e94cb577bf4 100644
--- a/tests/queries/0_stateless/02708_dot_product.sql
+++ b/tests/queries/0_stateless/02708_dot_product.sql
@@ -1,13 +1,16 @@
 SELECT dotProduct([12, 2.22, 302], [1.32, 231.2, 11.1]);
+
 SELECT scalarProduct([12, 2.22, 302], [1.32, 231.2, 11.1]);
+
 SELECT arrayDotProduct([12, 2.22, 302], [1.32, 231.2, 11.1]);
 
 SELECT dotProduct([1.3, 2, 3, 4, 5], [222, 12, 5.3, 2, 8]);
 
 SELECT dotProduct([1, 1, 1, 1, 1], [222, 12, 0, -12, 8]);
 
-SELECT round(dotProduct([-1, 2, 3.002], [2, 3.4, 4]) - dotProduct((-1, 2, 3.002), (2, 3.4, 4)), 2);
+SELECT round(dotProduct([12345678901234567], [1]) - dotProduct(tuple(12345678901234567), tuple(1)), 2);
 
+SELECT round(dotProduct([-1, 2, 3.002], [2, 3.4, 4]) - dotProduct((-1, 2, 3.002), (2, 3.4, 4)), 2);
 
 DROP TABLE IF EXISTS product_fp64_fp64;
 CREATE TABLE product_fp64_fp64 (x Array(Float64), y Array(Float64)) engine = MergeTree() order by x;
@@ -32,3 +35,21 @@ CREATE TABLE product_uint8_fp64 (x Array(UInt8), y Array(Float64)) engine = Merg
 INSERT INTO TABLE product_uint8_fp64 (x, y) values ([1, 2], [3, 4]);
 SELECT toTypeName(dotProduct(x, y)) from product_uint8_fp64;
 DROP TABLE product_uint8_fp64;
+
+DROP TABLE IF EXISTS product_uint8_uint8;
+CREATE TABLE product_uint8_uint8 (x Array(UInt8), y Array(UInt8)) engine = MergeTree() order by x;
+INSERT INTO TABLE product_uint8_uint8 (x, y) values ([1, 2], [3, 4]);
+SELECT toTypeName(dotProduct(x, y)) from product_uint8_uint8;
+DROP TABLE product_uint8_uint8;
+
+DROP TABLE IF EXISTS product_uint64_uint64;
+CREATE TABLE product_uint64_uint64 (x Array(UInt64), y Array(UInt64)) engine = MergeTree() order by x;
+INSERT INTO TABLE product_uint64_uint64 (x, y) values ([1, 2], [3, 4]);
+SELECT toTypeName(dotProduct(x, y)) from product_uint64_uint64;
+DROP TABLE product_uint64_uint64;
+
+DROP TABLE IF EXISTS product_int32_uint64;
+CREATE TABLE product_int32_uint64 (x Array(Int32), y Array(UInt64)) engine = MergeTree() order by x;
+INSERT INTO TABLE product_int32_uint64 (x, y) values ([1, 2], [3, 4]);
+SELECT toTypeName(dotProduct(x, y)) from product_int32_uint64;
+DROP TABLE product_int32_uint64;

From 79398f612f4a94cee996d6256155c5e229ee090b Mon Sep 17 00:00:00 2001
From: FFFFFFFHHHHHHH <916677625@qq.com>
Date: Tue, 9 May 2023 11:50:38 +0800
Subject: [PATCH 326/689] fix style

---
 src/Functions/array/arrayDotProduct.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp
index 7aa9f1d49c7..e3c80775f1b 100644
--- a/src/Functions/array/arrayDotProduct.cpp
+++ b/src/Functions/array/arrayDotProduct.cpp
@@ -32,8 +32,10 @@ public:
                                DataTypeInt8, DataTypeInt16, DataTypeInt32, DataTypeInt64>;
 
         DataTypePtr result_type;
-        bool valid = castTypeToEither(Types{}, left.get(), [&](const auto & left_) {
-            return castTypeToEither(Types{}, right.get(), [&](const auto & right_) {
+        bool valid = castTypeToEither(Types{}, left.get(), [&](const auto & left_)
+        {
+            return castTypeToEither(Types{}, right.get(), [&](const auto & right_)
+            {
                 using LeftDataType = typename std::decay_t<decltype(left_)>::FieldType;
                 using RightDataType = typename std::decay_t<decltype(right_)>::FieldType;
                 using ResultType = typename NumberTraits::ResultOfAdditionMultiplication<LeftDataType, RightDataType>::Type;

From 82536feddee07f56fbada7a17cc78969eb799141 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 10:25:50 +0200
Subject: [PATCH 327/689] docker: install libclang-16-dev for woboq code
 browser

Otherwise you will get the following error:

    CMake Error at /usr/lib/llvm-16/lib/cmake/clang/ClangTargets.cmake:792 (message):
      The imported target "clangBasic" references the file

         "/usr/lib/llvm-16/lib/libclangBasic.a"

      but this file does not exist.  Possible reasons include:

      * The file was deleted, renamed, or moved to another location.

      * An install or uninstall procedure did not complete successfully.

      * The installation package was faulty and contained

         "/usr/lib/llvm-16/lib/cmake/clang/ClangTargets.cmake"

      but not all the files it references.

    Call Stack (most recent call first):
      /usr/lib/llvm-16/lib/cmake/clang/ClangConfig.cmake:20 (include)
      generator/CMakeLists.txt:7 (Find_Package)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docker/test/util/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile
index 92ae9ed02b2..a49278e960b 100644
--- a/docker/test/util/Dockerfile
+++ b/docker/test/util/Dockerfile
@@ -52,6 +52,7 @@ RUN apt-get update \
         lld-${LLVM_VERSION} \
         llvm-${LLVM_VERSION} \
         llvm-${LLVM_VERSION}-dev \
+        libclang-${LLVM_VERSION}-dev \
         moreutils \
         nasm \
         ninja-build \

From 08b8b7e5879d30c2ed079d4b1a79d33fc80d2caf Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 10:30:02 +0200
Subject: [PATCH 328/689] docker: remove symlink for libclang (not required
 anymore)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docker/test/codebrowser/Dockerfile | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile
index 804a665cfa8..b71b73af8bc 100644
--- a/docker/test/codebrowser/Dockerfile
+++ b/docker/test/codebrowser/Dockerfile
@@ -10,28 +10,13 @@ RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
 
 RUN apt-get update && apt-get --yes --allow-unauthenticated install libclang-${LLVM_VERSION}-dev libmlir-${LLVM_VERSION}-dev
 
-# libclang-15-dev does not contain proper symlink:
-#
-# This is what cmake will search for:
-#
-#     # readlink -f /usr/lib/llvm-15/lib/libclang-15.so.1
-#     /usr/lib/x86_64-linux-gnu/libclang-15.so.1
-#
-# This is what exists:
-#
-#     # ls -l /usr/lib/x86_64-linux-gnu/libclang-15*
-#     lrwxrwxrwx 1 root root       16 Sep  5 13:31 /usr/lib/x86_64-linux-gnu/libclang-15.so -> libclang-15.so.1
-#     lrwxrwxrwx 1 root root       21 Sep  5 13:31 /usr/lib/x86_64-linux-gnu/libclang-15.so.15 -> libclang-15.so.15.0.0
-#     -rw-r--r-- 1 root root 31835760 Sep  5 13:31 /usr/lib/x86_64-linux-gnu/libclang-15.so.15.0.0
-#
 ARG TARGETARCH
 RUN arch=${TARGETARCH:-amd64} \
     && case $arch in \
         amd64) rarch=x86_64 ;; \
         arm64) rarch=aarch64 ;; \
         *) exit 1 ;; \
-    esac \
-    && ln -rsf /usr/lib/$rarch-linux-gnu/libclang-16.so.15 /usr/lib/$rarch-linux-gnu/libclang-16.so.1
+    esac
 
 # repo versions doesn't work correctly with C++17
 # also we push reports to s3, so we add index.html to subfolder urls

From bf17c819c31ddcec0e358a8f82fa5f5dcfe14d46 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 10:31:55 +0200
Subject: [PATCH 329/689] docker: fix path to clang builtin headers for woboq
 code browser

By default woboq uses $version.$minor.$patch, while clang simply uses
$version:

    CMake Error at generator/CMakeLists.txt:77 (message):
      Could not find any clang builtins headers in
      /usr/lib/llvm-16/lib/clang/16.0.4/include

    # ls -d /usr/lib/llvm-16/lib/clang/16/include
    /usr/lib/llvm-16/lib/clang/16/include

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docker/test/codebrowser/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile
index b71b73af8bc..6ceaa9e1bee 100644
--- a/docker/test/codebrowser/Dockerfile
+++ b/docker/test/codebrowser/Dockerfile
@@ -23,7 +23,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b
 RUN git clone https://github.com/ClickHouse/woboq_codebrowser \
   && cd woboq_codebrowser \
-  && cmake . -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-${LLVM_VERSION} -DCMAKE_C_COMPILER=clang-${LLVM_VERSION} \
+  && cmake . -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-${LLVM_VERSION} -DCMAKE_C_COMPILER=clang-${LLVM_VERSION} -DCLANG_BUILTIN_HEADERS_DIR=/usr/lib/llvm-${LLVM_VERSION}/lib/clang/${LLVM_VERSION}/include \
   && ninja \
   && cd .. \
   && rm -rf woboq_codebrowser

From 8cfe0eb3f2009878ad68465b6949aed494dc2dd1 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 7 May 2023 19:31:47 +0200
Subject: [PATCH 330/689] Do not change CMAKE_EXE_LINKER_FLAGS_INIT in
 toolchains

Otherwise check in PreLoad.cmake will throw.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 cmake/linux/toolchain-riscv64.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/linux/toolchain-riscv64.cmake b/cmake/linux/toolchain-riscv64.cmake
index ea57c3b2c42..7f876f88d72 100644
--- a/cmake/linux/toolchain-riscv64.cmake
+++ b/cmake/linux/toolchain-riscv64.cmake
@@ -21,7 +21,7 @@ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 
-set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=bfd")
+set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=bfd")
 
 # Currently, lld does not work with the error:
 # ld.lld: error: section size decrease is too large

From 6ae41be0ccce9bcdb16cb2a059ac315215a8d9e8 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 7 May 2023 19:32:12 +0200
Subject: [PATCH 331/689] Add riscv64 support for boringssl

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 contrib/boringssl-cmake/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/contrib/boringssl-cmake/CMakeLists.txt b/contrib/boringssl-cmake/CMakeLists.txt
index 828919476a7..51137f6d04e 100644
--- a/contrib/boringssl-cmake/CMakeLists.txt
+++ b/contrib/boringssl-cmake/CMakeLists.txt
@@ -111,6 +111,8 @@ elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "mips")
   set(ARCH "generic")
 elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "ppc64le")
   set(ARCH "ppc64le")
+elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "riscv64")
+  set(ARCH "riscv64")
 else()
   message(FATAL_ERROR "Unknown processor:" ${CMAKE_SYSTEM_PROCESSOR})
 endif()

From 4c7849a9c8d154d5c3142933320f8cd635532b6d Mon Sep 17 00:00:00 2001
From: Duyet Le <5009534+duyet@users.noreply.github.com>
Date: Tue, 9 May 2023 17:08:49 +0700
Subject: [PATCH 332/689] Fix incorrect docs for `use_query_cache`

---
 docs/en/operations/settings/settings.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index f21dff9fbb7..0a4b9a6bd9d 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -1410,8 +1410,8 @@ and [enable_writes_to_query_cache](#enable-writes-to-query-cache) control in mor
 
 Possible values:
 
-- 0 - Yes
-- 1 - No
+- 0 - Disabled
+- 1 - Enabled
 
 Default value: `0`.
 

From f28834f60ba9a95d6ef16d5c4f84392791868b0d Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 9 May 2023 12:12:35 +0200
Subject: [PATCH 333/689] fix fast test

---
 tests/queries/0_stateless/01271_show_privileges.reference | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference
index 4f7d4a575f3..ec245d8b9e0 100644
--- a/tests/queries/0_stateless/01271_show_privileges.reference
+++ b/tests/queries/0_stateless/01271_show_privileges.reference
@@ -136,7 +136,7 @@ SYSTEM FLUSH LOGS	['FLUSH LOGS']	GLOBAL	SYSTEM FLUSH
 SYSTEM FLUSH	[]	\N	SYSTEM
 SYSTEM THREAD FUZZER	['SYSTEM START THREAD FUZZER','SYSTEM STOP THREAD FUZZER','START THREAD FUZZER','STOP THREAD FUZZER']	GLOBAL	SYSTEM
 SYSTEM UNFREEZE	['SYSTEM UNFREEZE']	GLOBAL	SYSTEM
-SYSTEM FAILPOINT	['SYSTEM ENABLE/DISABLE FAILPOINT']	GLOBAL	SYSTEM
+SYSTEM FAILPOINT	['SYSTEM ENABLE FAILPOINT','SYSTEM DISABLE FAILPOINT']	GLOBAL	SYSTEM
 SYSTEM	[]	\N	ALL
 dictGet	['dictHas','dictGetHierarchy','dictIsIn']	DICTIONARY	ALL
 displaySecretsInShowAndSelect	[]	GLOBAL	ALL

From a794177dc3c018530aa5feb86a2070e55e86a41b Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 12:20:50 +0200
Subject: [PATCH 334/689] Use __powerpc64__ over __ppc64__ (clang 16 stops
 declaring __ppc64__)

From the release notes [1]:

    To match GCC, __ppc64__ is no longer defined on PowerPC64 targets. Use __powerpc64__ instead.

  [1]: https://releases.llvm.org/16.0.0/tools/clang/docs/ReleaseNotes.html

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Common/atomicRename.cpp       | 2 +-
 src/Common/waitForPid.cpp         | 2 +-
 src/Disks/IO/ThreadPoolReader.cpp | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Common/atomicRename.cpp b/src/Common/atomicRename.cpp
index 987e94a4378..69e077e38f5 100644
--- a/src/Common/atomicRename.cpp
+++ b/src/Common/atomicRename.cpp
@@ -42,7 +42,7 @@ namespace ErrorCodes
         #define __NR_renameat2 316
     #elif defined(__aarch64__)
         #define __NR_renameat2 276
-    #elif defined(__ppc64__)
+    #elif defined(__powerpc64__)
         #define __NR_renameat2 357
     #elif defined(__riscv)
         #define __NR_renameat2 276
diff --git a/src/Common/waitForPid.cpp b/src/Common/waitForPid.cpp
index 8f2be169c13..0ec10811354 100644
--- a/src/Common/waitForPid.cpp
+++ b/src/Common/waitForPid.cpp
@@ -41,7 +41,7 @@ enum PollPidResult
         #define SYS_pidfd_open 434
     #elif defined(__aarch64__)
         #define SYS_pidfd_open 434
-    #elif defined(__ppc64__)
+    #elif defined(__powerpc64__)
         #define SYS_pidfd_open 434
     #elif defined(__riscv)
         #define SYS_pidfd_open 434
diff --git a/src/Disks/IO/ThreadPoolReader.cpp b/src/Disks/IO/ThreadPoolReader.cpp
index de57fa157da..effa19bc1af 100644
--- a/src/Disks/IO/ThreadPoolReader.cpp
+++ b/src/Disks/IO/ThreadPoolReader.cpp
@@ -32,7 +32,7 @@
         #define SYS_preadv2 327
     #elif defined(__aarch64__)
         #define SYS_preadv2 286
-    #elif defined(__ppc64__)
+    #elif defined(__powerpc64__)
         #define SYS_preadv2 380
     #elif defined(__riscv)
         #define SYS_preadv2 286

From 6e9985a94d88d49169a48551ba8910c2b373d0de Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 12:22:14 +0200
Subject: [PATCH 335/689] Remove removed in clang 16
 -fexperimental-new-pass-manager

"The -fexperimental-new-pass-manager and -fno-legacy-pass-manager flags
have been removed. These flags have been silently ignored since Clang
15."

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 CMakeLists.txt | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5ff48ceacc2..c09acbec7f1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -342,13 +342,6 @@ if (COMPILER_CLANG)
 
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstrict-vtable-pointers")
 
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16)
-        # Set new experimental pass manager, it's a performance, build time and binary size win.
-        # Can be removed after https://reviews.llvm.org/D66490 merged and released to at least two versions of clang.
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexperimental-new-pass-manager")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fexperimental-new-pass-manager")
-    endif ()
-
     # We cannot afford to use LTO when compiling unit tests, and it's not enough
     # to only supply -fno-lto at the final linking stage. So we disable it
     # completely.

From 780b5a6f9722596fc7804c7546a36fdc52299344 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Tue, 9 May 2023 12:34:25 +0200
Subject: [PATCH 336/689] Fix a wrong built generator removal, use `depth=1`

---
 docker/test/codebrowser/Dockerfile | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile
index b76b8234c81..2b788ab134a 100644
--- a/docker/test/codebrowser/Dockerfile
+++ b/docker/test/codebrowser/Dockerfile
@@ -36,12 +36,10 @@ RUN arch=${TARGETARCH:-amd64} \
 # repo versions doesn't work correctly with C++17
 # also we push reports to s3, so we add index.html to subfolder urls
 # https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b
-RUN git clone https://github.com/ClickHouse/woboq_codebrowser \
-  && cd woboq_codebrowser \
+RUN git clone --depth=1 https://github.com/ClickHouse/woboq_codebrowser /woboq_codebrowser \
+  && cd /woboq_codebrowser \
   && cmake . -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-${LLVM_VERSION} -DCMAKE_C_COMPILER=clang-${LLVM_VERSION} \
-  && ninja \
-  && cd .. \
-  && rm -rf woboq_codebrowser
+  && ninja
 
 ENV CODEGEN=/woboq_codebrowser/generator/codebrowser_generator
 ENV CODEINDEX=/woboq_codebrowser/indexgenerator/codebrowser_indexgenerator

From 459181568c6b2e39b3b76d84f9c141a33707ccb9 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Tue, 9 May 2023 11:54:25 +0000
Subject: [PATCH 337/689] remove not used ErrorCodes

---
 src/Common/ErrorCodes.cpp | 75 ---------------------------------------
 1 file changed, 75 deletions(-)

diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp
index d8a83378e26..e9dc5649245 100644
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@@ -29,21 +29,14 @@
     M(13, SIZES_OF_COLUMNS_IN_TUPLE_DOESNT_MATCH) \
     M(15, DUPLICATE_COLUMN) \
     M(16, NO_SUCH_COLUMN_IN_TABLE) \
-    M(17, DELIMITER_IN_STRING_LITERAL_DOESNT_MATCH) \
-    M(18, CANNOT_INSERT_ELEMENT_INTO_CONSTANT_COLUMN) \
     M(19, SIZE_OF_FIXED_STRING_DOESNT_MATCH) \
     M(20, NUMBER_OF_COLUMNS_DOESNT_MATCH) \
-    M(21, CANNOT_READ_ALL_DATA_FROM_TAB_SEPARATED_INPUT) \
-    M(22, CANNOT_PARSE_ALL_VALUE_FROM_TAB_SEPARATED_INPUT) \
     M(23, CANNOT_READ_FROM_ISTREAM) \
     M(24, CANNOT_WRITE_TO_OSTREAM) \
     M(25, CANNOT_PARSE_ESCAPE_SEQUENCE) \
     M(26, CANNOT_PARSE_QUOTED_STRING) \
     M(27, CANNOT_PARSE_INPUT_ASSERTION_FAILED) \
     M(28, CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER) \
-    M(29, CANNOT_PRINT_INTEGER) \
-    M(30, CANNOT_READ_SIZE_OF_COMPRESSED_CHUNK) \
-    M(31, CANNOT_READ_COMPRESSED_CHUNK) \
     M(32, ATTEMPT_TO_READ_AFTER_EOF) \
     M(33, CANNOT_READ_ALL_DATA) \
     M(34, TOO_MANY_ARGUMENTS_FOR_FUNCTION) \
@@ -57,7 +50,6 @@
     M(42, NUMBER_OF_ARGUMENTS_DOESNT_MATCH) \
     M(43, ILLEGAL_TYPE_OF_ARGUMENT) \
     M(44, ILLEGAL_COLUMN) \
-    M(45, ILLEGAL_NUMBER_OF_RESULT_COLUMNS) \
     M(46, UNKNOWN_FUNCTION) \
     M(47, UNKNOWN_IDENTIFIER) \
     M(48, NOT_IMPLEMENTED) \
@@ -66,20 +58,14 @@
     M(51, EMPTY_LIST_OF_COLUMNS_QUERIED) \
     M(52, COLUMN_QUERIED_MORE_THAN_ONCE) \
     M(53, TYPE_MISMATCH) \
-    M(54, STORAGE_DOESNT_ALLOW_PARAMETERS) \
     M(55, STORAGE_REQUIRES_PARAMETER) \
     M(56, UNKNOWN_STORAGE) \
     M(57, TABLE_ALREADY_EXISTS) \
     M(58, TABLE_METADATA_ALREADY_EXISTS) \
     M(59, ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER) \
     M(60, UNKNOWN_TABLE) \
-    M(61, ONLY_FILTER_COLUMN_IN_BLOCK) \
     M(62, SYNTAX_ERROR) \
     M(63, UNKNOWN_AGGREGATE_FUNCTION) \
-    M(64, CANNOT_READ_AGGREGATE_FUNCTION_FROM_TEXT) \
-    M(65, CANNOT_WRITE_AGGREGATE_FUNCTION_AS_TEXT) \
-    M(66, NOT_A_COLUMN) \
-    M(67, ILLEGAL_KEY_OF_AGGREGATION) \
     M(68, CANNOT_GET_SIZE_OF_FIELD) \
     M(69, ARGUMENT_OUT_OF_BOUND) \
     M(70, CANNOT_CONVERT_TYPE) \
@@ -109,16 +95,11 @@
     M(94, CANNOT_MERGE_DIFFERENT_AGGREGATED_DATA_VARIANTS) \
     M(95, CANNOT_READ_FROM_SOCKET) \
     M(96, CANNOT_WRITE_TO_SOCKET) \
-    M(97, CANNOT_READ_ALL_DATA_FROM_CHUNKED_INPUT) \
-    M(98, CANNOT_WRITE_TO_EMPTY_BLOCK_OUTPUT_STREAM) \
     M(99, UNKNOWN_PACKET_FROM_CLIENT) \
     M(100, UNKNOWN_PACKET_FROM_SERVER) \
     M(101, UNEXPECTED_PACKET_FROM_CLIENT) \
     M(102, UNEXPECTED_PACKET_FROM_SERVER) \
-    M(103, RECEIVED_DATA_FOR_WRONG_QUERY_ID) \
     M(104, TOO_SMALL_BUFFER_SIZE) \
-    M(105, CANNOT_READ_HISTORY) \
-    M(106, CANNOT_APPEND_HISTORY) \
     M(107, FILE_DOESNT_EXIST) \
     M(108, NO_DATA_TO_INSERT) \
     M(109, CANNOT_BLOCK_SIGNAL) \
@@ -137,7 +118,6 @@
     M(123, UNKNOWN_TYPE_OF_AST_NODE) \
     M(124, INCORRECT_ELEMENT_OF_SET) \
     M(125, INCORRECT_RESULT_OF_SCALAR_SUBQUERY) \
-    M(126, CANNOT_GET_RETURN_TYPE) \
     M(127, ILLEGAL_INDEX) \
     M(128, TOO_LARGE_ARRAY_SIZE) \
     M(129, FUNCTION_IS_SPECIAL) \
@@ -149,30 +129,17 @@
     M(137, UNKNOWN_ELEMENT_IN_CONFIG) \
     M(138, EXCESSIVE_ELEMENT_IN_CONFIG) \
     M(139, NO_ELEMENTS_IN_CONFIG) \
-    M(140, ALL_REQUESTED_COLUMNS_ARE_MISSING) \
     M(141, SAMPLING_NOT_SUPPORTED) \
     M(142, NOT_FOUND_NODE) \
-    M(143, FOUND_MORE_THAN_ONE_NODE) \
-    M(144, FIRST_DATE_IS_BIGGER_THAN_LAST_DATE) \
     M(145, UNKNOWN_OVERFLOW_MODE) \
-    M(146, QUERY_SECTION_DOESNT_MAKE_SENSE) \
-    M(147, NOT_FOUND_FUNCTION_ELEMENT_FOR_AGGREGATE) \
-    M(148, NOT_FOUND_RELATION_ELEMENT_FOR_CONDITION) \
-    M(149, NOT_FOUND_RHS_ELEMENT_FOR_CONDITION) \
-    M(150, EMPTY_LIST_OF_ATTRIBUTES_PASSED) \
-    M(151, INDEX_OF_COLUMN_IN_SORT_CLAUSE_IS_OUT_OF_RANGE) \
     M(152, UNKNOWN_DIRECTION_OF_SORTING) \
     M(153, ILLEGAL_DIVISION) \
-    M(154, AGGREGATE_FUNCTION_NOT_APPLICABLE) \
-    M(155, UNKNOWN_RELATION) \
     M(156, DICTIONARIES_WAS_NOT_LOADED) \
-    M(157, ILLEGAL_OVERFLOW_MODE) \
     M(158, TOO_MANY_ROWS) \
     M(159, TIMEOUT_EXCEEDED) \
     M(160, TOO_SLOW) \
     M(161, TOO_MANY_COLUMNS) \
     M(162, TOO_DEEP_SUBQUERIES) \
-    M(163, TOO_DEEP_PIPELINE) \
     M(164, READONLY) \
     M(165, TOO_MANY_TEMPORARY_COLUMNS) \
     M(166, TOO_MANY_TEMPORARY_NON_CONST_COLUMNS) \
@@ -183,20 +150,14 @@
     M(172, CANNOT_CREATE_DIRECTORY) \
     M(173, CANNOT_ALLOCATE_MEMORY) \
     M(174, CYCLIC_ALIASES) \
-    M(176, CHUNK_NOT_FOUND) \
-    M(177, DUPLICATE_CHUNK_NAME) \
-    M(178, MULTIPLE_ALIASES_FOR_EXPRESSION) \
     M(179, MULTIPLE_EXPRESSIONS_FOR_ALIAS) \
     M(180, THERE_IS_NO_PROFILE) \
     M(181, ILLEGAL_FINAL) \
     M(182, ILLEGAL_PREWHERE) \
     M(183, UNEXPECTED_EXPRESSION) \
     M(184, ILLEGAL_AGGREGATION) \
-    M(185, UNSUPPORTED_MYISAM_BLOCK_TYPE) \
     M(186, UNSUPPORTED_COLLATION_LOCALE) \
     M(187, COLLATION_COMPARISON_FAILED) \
-    M(188, UNKNOWN_ACTION) \
-    M(189, TABLE_MUST_NOT_BE_CREATED_MANUALLY) \
     M(190, SIZES_OF_ARRAYS_DONT_MATCH) \
     M(191, SET_SIZE_LIMIT_EXCEEDED) \
     M(192, UNKNOWN_USER) \
@@ -204,15 +165,12 @@
     M(194, REQUIRED_PASSWORD) \
     M(195, IP_ADDRESS_NOT_ALLOWED) \
     M(196, UNKNOWN_ADDRESS_PATTERN_TYPE) \
-    M(197, SERVER_REVISION_IS_TOO_OLD) \
     M(198, DNS_ERROR) \
     M(199, UNKNOWN_QUOTA) \
-    M(200, QUOTA_DOESNT_ALLOW_KEYS) \
     M(201, QUOTA_EXCEEDED) \
     M(202, TOO_MANY_SIMULTANEOUS_QUERIES) \
     M(203, NO_FREE_CONNECTION) \
     M(204, CANNOT_FSYNC) \
-    M(205, NESTED_TYPE_TOO_DEEP) \
     M(206, ALIAS_REQUIRED) \
     M(207, AMBIGUOUS_IDENTIFIER) \
     M(208, EMPTY_NESTED_TABLE) \
@@ -229,7 +187,6 @@
     M(219, DATABASE_NOT_EMPTY) \
     M(220, DUPLICATE_INTERSERVER_IO_ENDPOINT) \
     M(221, NO_SUCH_INTERSERVER_IO_ENDPOINT) \
-    M(222, ADDING_REPLICA_TO_NON_EMPTY_TABLE) \
     M(223, UNEXPECTED_AST_STRUCTURE) \
     M(224, REPLICA_IS_ALREADY_ACTIVE) \
     M(225, NO_ZOOKEEPER) \
@@ -253,9 +210,7 @@
     M(243, NOT_ENOUGH_SPACE) \
     M(244, UNEXPECTED_ZOOKEEPER_ERROR) \
     M(246, CORRUPTED_DATA) \
-    M(247, INCORRECT_MARK) \
     M(248, INVALID_PARTITION_VALUE) \
-    M(250, NOT_ENOUGH_BLOCK_NUMBERS) \
     M(251, NO_SUCH_REPLICA) \
     M(252, TOO_MANY_PARTS) \
     M(253, REPLICA_ALREADY_EXISTS) \
@@ -271,8 +226,6 @@
     M(264, INCOMPATIBLE_TYPE_OF_JOIN) \
     M(265, NO_AVAILABLE_REPLICA) \
     M(266, MISMATCH_REPLICAS_DATA_SOURCES) \
-    M(267, STORAGE_DOESNT_SUPPORT_PARALLEL_REPLICAS) \
-    M(268, CPUID_ERROR) \
     M(269, INFINITE_LOOP) \
     M(270, CANNOT_COMPRESS) \
     M(271, CANNOT_DECOMPRESS) \
@@ -295,9 +248,7 @@
     M(290, LIMIT_EXCEEDED) \
     M(291, DATABASE_ACCESS_DENIED) \
     M(293, MONGODB_CANNOT_AUTHENTICATE) \
-    M(294, INVALID_BLOCK_EXTRA_INFO) \
     M(295, RECEIVED_EMPTY_DATA) \
-    M(296, NO_REMOTE_SHARD_FOUND) \
     M(297, SHARD_HAS_NO_CONNECTIONS) \
     M(298, CANNOT_PIPE) \
     M(299, CANNOT_FORK) \
@@ -311,13 +262,10 @@
     M(307, TOO_MANY_BYTES) \
     M(308, UNEXPECTED_NODE_IN_ZOOKEEPER) \
     M(309, FUNCTION_CANNOT_HAVE_PARAMETERS) \
-    M(317, INVALID_SHARD_WEIGHT) \
     M(318, INVALID_CONFIG_PARAMETER) \
     M(319, UNKNOWN_STATUS_OF_INSERT) \
     M(321, VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE) \
-    M(335, BARRIER_TIMEOUT) \
     M(336, UNKNOWN_DATABASE_ENGINE) \
-    M(337, DDL_GUARD_IS_ACTIVE) \
     M(341, UNFINISHED) \
     M(342, METADATA_MISMATCH) \
     M(344, SUPPORT_IS_DISABLED) \
@@ -325,14 +273,10 @@
     M(346, CANNOT_CONVERT_CHARSET) \
     M(347, CANNOT_LOAD_CONFIG) \
     M(349, CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN) \
-    M(350, INCOMPATIBLE_SOURCE_TABLES) \
-    M(351, AMBIGUOUS_TABLE_NAME) \
     M(352, AMBIGUOUS_COLUMN_NAME) \
     M(353, INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE) \
     M(354, ZLIB_INFLATE_FAILED) \
     M(355, ZLIB_DEFLATE_FAILED) \
-    M(356, BAD_LAMBDA) \
-    M(357, RESERVED_IDENTIFIER_NAME) \
     M(358, INTO_OUTFILE_NOT_ALLOWED) \
     M(359, TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT) \
     M(360, CANNOT_CREATE_CHARSET_CONVERTER) \
@@ -341,7 +285,6 @@
     M(363, CANNOT_CREATE_IO_BUFFER) \
     M(364, RECEIVED_ERROR_TOO_MANY_REQUESTS) \
     M(366, SIZES_OF_NESTED_COLUMNS_ARE_INCONSISTENT) \
-    M(367, TOO_MANY_FETCHES) \
     M(369, ALL_REPLICAS_ARE_STALE) \
     M(370, DATA_TYPE_CANNOT_BE_USED_IN_TABLES) \
     M(371, INCONSISTENT_CLUSTER_DEFINITION) \
@@ -352,7 +295,6 @@
     M(376, CANNOT_PARSE_UUID) \
     M(377, ILLEGAL_SYNTAX_FOR_DATA_TYPE) \
     M(378, DATA_TYPE_CANNOT_HAVE_ARGUMENTS) \
-    M(379, UNKNOWN_STATUS_OF_DISTRIBUTED_DDL_TASK) \
     M(380, CANNOT_KILL) \
     M(381, HTTP_LENGTH_REQUIRED) \
     M(382, CANNOT_LOAD_CATBOOST_MODEL) \
@@ -378,11 +320,9 @@
     M(402, CANNOT_IOSETUP) \
     M(403, INVALID_JOIN_ON_EXPRESSION) \
     M(404, BAD_ODBC_CONNECTION_STRING) \
-    M(405, PARTITION_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT) \
     M(406, TOP_AND_LIMIT_TOGETHER) \
     M(407, DECIMAL_OVERFLOW) \
     M(408, BAD_REQUEST_PARAMETER) \
-    M(409, EXTERNAL_EXECUTABLE_NOT_FOUND) \
     M(410, EXTERNAL_SERVER_IS_NOT_RESPONDING) \
     M(411, PTHREAD_ERROR) \
     M(412, NETLINK_ERROR) \
@@ -399,7 +339,6 @@
     M(424, CANNOT_LINK) \
     M(425, SYSTEM_ERROR) \
     M(427, CANNOT_COMPILE_REGEXP) \
-    M(428, UNKNOWN_LOG_LEVEL) \
     M(429, FAILED_TO_GETPWUID) \
     M(430, MISMATCHING_USERS_FOR_PROCESS_AND_DATA) \
     M(431, ILLEGAL_SYNTAX_FOR_CODEC_TYPE) \
@@ -433,7 +372,6 @@
     M(459, CANNOT_SET_THREAD_PRIORITY) \
     M(460, CANNOT_CREATE_TIMER) \
     M(461, CANNOT_SET_TIMER_PERIOD) \
-    M(462, CANNOT_DELETE_TIMER) \
     M(463, CANNOT_FCNTL) \
     M(464, CANNOT_PARSE_ELF) \
     M(465, CANNOT_PARSE_DWARF) \
@@ -456,15 +394,12 @@
     M(482, DICTIONARY_ACCESS_DENIED) \
     M(483, TOO_MANY_REDIRECTS) \
     M(484, INTERNAL_REDIS_ERROR) \
-    M(485, SCALAR_ALREADY_EXISTS) \
     M(487, CANNOT_GET_CREATE_DICTIONARY_QUERY) \
-    M(488, UNKNOWN_DICTIONARY) \
     M(489, INCORRECT_DICTIONARY_DEFINITION) \
     M(490, CANNOT_FORMAT_DATETIME) \
     M(491, UNACCEPTABLE_URL) \
     M(492, ACCESS_ENTITY_NOT_FOUND) \
     M(493, ACCESS_ENTITY_ALREADY_EXISTS) \
-    M(494, ACCESS_ENTITY_FOUND_DUPLICATES) \
     M(495, ACCESS_STORAGE_READONLY) \
     M(496, QUOTA_REQUIRES_CLIENT_KEY) \
     M(497, ACCESS_DENIED) \
@@ -475,8 +410,6 @@
     M(502, CANNOT_SIGQUEUE) \
     M(503, AGGREGATE_FUNCTION_THROW) \
     M(504, FILE_ALREADY_EXISTS) \
-    M(505, CANNOT_DELETE_DIRECTORY) \
-    M(506, UNEXPECTED_ERROR_CODE) \
     M(507, UNABLE_TO_SKIP_UNUSED_SHARDS) \
     M(508, UNKNOWN_ACCESS_TYPE) \
     M(509, INVALID_GRANT) \
@@ -501,8 +434,6 @@
     M(530, CANNOT_CONNECT_RABBITMQ) \
     M(531, CANNOT_FSTAT) \
     M(532, LDAP_ERROR) \
-    M(533, INCONSISTENT_RESERVATIONS) \
-    M(534, NO_RESERVATIONS_PROVIDED) \
     M(535, UNKNOWN_RAID_TYPE) \
     M(536, CANNOT_RESTORE_FROM_FIELD_DUMP) \
     M(537, ILLEGAL_MYSQL_VARIABLE) \
@@ -518,8 +449,6 @@
     M(547, INVALID_RAID_TYPE) \
     M(548, UNKNOWN_VOLUME) \
     M(549, DATA_TYPE_CANNOT_BE_USED_IN_KEY) \
-    M(550, CONDITIONAL_TREE_PARENT_NOT_FOUND) \
-    M(551, ILLEGAL_PROJECTION_MANIPULATOR) \
     M(552, UNRECOGNIZED_ARGUMENTS) \
     M(553, LZMA_STREAM_ENCODER_FAILED) \
     M(554, LZMA_STREAM_DECODER_FAILED) \
@@ -580,8 +509,6 @@
     M(609, FUNCTION_ALREADY_EXISTS) \
     M(610, CANNOT_DROP_FUNCTION) \
     M(611, CANNOT_CREATE_RECURSIVE_FUNCTION) \
-    M(612, OBJECT_ALREADY_STORED_ON_DISK) \
-    M(613, OBJECT_WAS_NOT_STORED_ON_DISK) \
     M(614, POSTGRESQL_CONNECTION_FAILURE) \
     M(615, CANNOT_ADVISE) \
     M(616, UNKNOWN_READ_METHOD) \
@@ -612,9 +539,7 @@
     M(641, CANNOT_APPEND_TO_FILE) \
     M(642, CANNOT_PACK_ARCHIVE) \
     M(643, CANNOT_UNPACK_ARCHIVE) \
-    M(644, REMOTE_FS_OBJECT_CACHE_ERROR) \
     M(645, NUMBER_OF_DIMENSIONS_MISMATCHED) \
-    M(646, CANNOT_BACKUP_DATABASE) \
     M(647, CANNOT_BACKUP_TABLE) \
     M(648, WRONG_DDL_RENAMING_SETTINGS) \
     M(649, INVALID_TRANSACTION) \

From b7d8c99fbeb1da4e3d1a3c780e69576471b912e4 Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Tue, 9 May 2023 13:12:11 +0000
Subject: [PATCH 338/689] Fix member call on null pointer in fuzzer

---
 programs/client/Client.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index df0abceb8c6..fb30f60a0b8 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -862,7 +862,8 @@ bool Client::processWithFuzzing(const String & full_query)
                 const auto * tmp_pos = text_2.c_str();
                 const auto ast_3 = parseQuery(tmp_pos, tmp_pos + text_2.size(),
                     false /* allow_multi_statements */);
-                const auto text_3 = ast_3->formatForErrorMessage();
+                const auto text_3 = ast_3 ? ast_3->formatForErrorMessage() : "";
+
                 if (text_3 != text_2)
                 {
                     fmt::print(stderr, "Found error: The query formatting is broken.\n");
@@ -877,7 +878,7 @@ bool Client::processWithFuzzing(const String & full_query)
                     fmt::print(stderr, "Text-1 (AST-1 formatted):\n'{}'\n", query_to_execute);
                     fmt::print(stderr, "AST-2 (Text-1 parsed):\n'{}'\n", ast_2->dumpTree());
                     fmt::print(stderr, "Text-2 (AST-2 formatted):\n'{}'\n", text_2);
-                    fmt::print(stderr, "AST-3 (Text-2 parsed):\n'{}'\n", ast_3->dumpTree());
+                    fmt::print(stderr, "AST-3 (Text-2 parsed):\n'{}'\n", ast_3 ? ast_3->dumpTree() : "");
                     fmt::print(stderr, "Text-3 (AST-3 formatted):\n'{}'\n", text_3);
                     fmt::print(stderr, "Text-3 must be equal to Text-2, but it is not.\n");
 

From 75aa775d2b62fc29ed697e7a0e2c915362220b17 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Tue, 9 May 2023 11:43:35 +0200
Subject: [PATCH 339/689] Add logging

---
 src/Interpreters/Cache/FileCache.cpp | 19 +++++++++++++++++++
 src/Interpreters/Cache/Metadata.cpp  |  7 +++++--
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index 1e42cc05aa3..2d5744b630e 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -533,6 +533,11 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size)
     assertInitialized();
     auto cache_lock = cache_guard.lock();
 
+    LOG_TEST(
+        log, "Trying to reserve space ({} bytes) for {}:{}, current usage {}/{}",
+        size, file_segment.key(), file_segment.offset(),
+        main_priority->getSize(cache_lock), main_priority->getSizeLimit());
+
     /// In case of per query cache limit (by default disabled), we add/remove entries from both
     /// (main_priority and query_priority) priority queues, but iterate entries in order of query_priority,
     /// while checking the limits in both.
@@ -545,7 +550,17 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size)
 
         const bool query_limit_exceeded = query_priority->getSize(cache_lock) + size > query_priority->getSizeLimit();
         if (query_limit_exceeded && !query_context->recacheOnFileCacheQueryLimitExceeded())
+        {
+            LOG_TEST(log, "Query limit exceeded, space reservation failed, "
+                     "recache_on_query_limit_exceeded is disabled (while reserving for {}:{})",
+                     file_segment.key(), file_segment.offset());
             return false;
+        }
+
+        LOG_TEST(
+            log, "Using query limit, current usage: {}/{} (while reserving for {}:{})",
+            query_priority->getSize(cache_lock), query_priority->getSizeLimit(),
+            file_segment.key(), file_segment.offset());
     }
 
     size_t queue_size = main_priority->getElementsCount(cache_lock);
@@ -630,6 +645,10 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size)
 
         if (is_query_priority_overflow())
             return false;
+
+        LOG_TEST(
+            log, "Query limits satisfied (while reserving for {}:{})",
+            file_segment.key(), file_segment.offset());
     }
 
     auto is_main_priority_overflow = [&]
diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp
index 1aa13d4e98c..de07bb29fec 100644
--- a/src/Interpreters/Cache/Metadata.cpp
+++ b/src/Interpreters/Cache/Metadata.cpp
@@ -336,13 +336,16 @@ void LockedKey::removeAllReleasable()
 
 KeyMetadata::iterator LockedKey::removeFileSegment(size_t offset, const FileSegmentGuard::Lock & segment_lock)
 {
-    LOG_DEBUG(log, "Remove from cache. Key: {}, offset: {}", getKey(), offset);
-
     auto it = key_metadata->find(offset);
     if (it == key_metadata->end())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no offset {}", offset);
 
     auto file_segment = it->second->file_segment;
+
+    LOG_DEBUG(
+        log, "Remove from cache. Key: {}, offset: {}, size: {}",
+        getKey(), offset, file_segment->getReservedSize());
+
     if (file_segment->queue_iterator)
         file_segment->queue_iterator->annul();
 

From a899b5a3993dd1914d2c83638138e27e2755f498 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Tue, 9 May 2023 18:00:05 +0200
Subject: [PATCH 340/689] Ping CI


From 7cd6db3fb37e6396221778bda7d60c852df8431b Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 9 May 2023 18:04:47 +0200
Subject: [PATCH 341/689] Enable `do_not_evict_index_and_mark_files` by default

---
 src/Interpreters/Cache/FileCacheSettings.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/Cache/FileCacheSettings.cpp b/src/Interpreters/Cache/FileCacheSettings.cpp
index 1737defd316..b6bf77cb306 100644
--- a/src/Interpreters/Cache/FileCacheSettings.cpp
+++ b/src/Interpreters/Cache/FileCacheSettings.cpp
@@ -47,7 +47,7 @@ void FileCacheSettings::loadFromConfig(const Poco::Util::AbstractConfiguration &
     else
          bypass_cache_threashold = FILECACHE_BYPASS_THRESHOLD;
 
-    do_not_evict_index_and_mark_files = config.getUInt64(config_prefix + ".do_not_evict_index_and_mark_files", false);
+    do_not_evict_index_and_mark_files = config.getUInt64(config_prefix + ".do_not_evict_index_and_mark_files", true);
 
     delayed_cleanup_interval_ms = config.getUInt64(config_prefix + ".delayed_cleanup_interval_ms", FILECACHE_DELAYED_CLEANUP_INTERVAL_MS);
 }

From d88224dc43694a87466da6ff7c3e4a81e51d8c09 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Tue, 9 May 2023 13:59:06 +0000
Subject: [PATCH 342/689] fix build

---
 src/Interpreters/tests/gtest_context_race.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Interpreters/tests/gtest_context_race.cpp b/src/Interpreters/tests/gtest_context_race.cpp
index 32922305435..60531494592 100644
--- a/src/Interpreters/tests/gtest_context_race.cpp
+++ b/src/Interpreters/tests/gtest_context_race.cpp
@@ -5,7 +5,7 @@
 using namespace DB;
 
 template <typename Ptr>
-void run(Ptr && context)
+void run(Ptr context)
 {
     for (size_t i = 0; i < 100; ++i)
     {
@@ -31,19 +31,19 @@ TEST(Context, MutableRace)
 {
     auto context = Context::createCopy(getContext().context);
     context->makeQueryContext();
-    run<ContextMutablePtr>(std::move(context));
+    run<ContextMutablePtr>(context);
 }
 
 TEST(Context, ConstRace)
 {
     auto context = Context::createCopy(getContext().context);
     context->makeQueryContext();
-    run<ContextPtr>(std::move(context));
+    run<ContextPtr>(context);
 }
 
 TEST(Context, WeakRace)
 {
     auto context = Context::createCopy(getContext().context);
     context->makeQueryContext();
-    run<ContextWeakPtr>(std::move(context));
+    run<ContextWeakPtr>(context);
 }

From cef98626281b928f2eddc620af9879baa49247e5 Mon Sep 17 00:00:00 2001
From: Constantine Peresypkin <constantine@caspiandb.com>
Date: Tue, 9 May 2023 12:40:33 -0400
Subject: [PATCH 343/689] docs: fix the is_deleted option docs

- `is_deleted` type __must__ be `UInt8` and not `Int8`
- strange wording on how `clean_deleted_rows` works is fixed
---
 .../table-engines/mergetree-family/replacingmergetree.md    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
index 81d8cc2d3ca..742ea408117 100644
--- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
@@ -90,12 +90,12 @@ SELECT * FROM mySecondReplacingMT FINAL;
 
 ### is_deleted
 
-`is_deleted` —  Name of the column with the type of row: `1` is a “deleted“ row, `0` is a “state“ row.
+`is_deleted` —  Name of a `UInt8` column with the type of row: `1` is a “deleted“ row, `0` is a “state“ row.
 
-    Column data type — `Int8`.
+    Column data type — `UInt8`.
 
     Can only be enabled when `ver` is used.
-    The row is deleted when use the `OPTIMIZE ... FINAL CLEANUP`, or `OPTIMIZE ... FINAL` if the engine settings `clean_deleted_rows` has been set to `Always`.
+    The row is deleted when the `OPTIMIZE ... FINAL CLEANUP`, or `OPTIMIZE ... FINAL` is used or if the engine settings `clean_deleted_rows` has been set to `Always`.
     No matter the operation on the data, the version must be increased. If two inserted rows have the same version number, the last inserted one is the one kept.
 
 

From 3f6ca2f9e8cff7d30347787b1f6a7095e2102c1b Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 9 May 2023 19:05:40 +0200
Subject: [PATCH 344/689] Backport fix for UBSan error in musl/logf.c

---
 base/glibc-compatibility/musl/logf.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base/glibc-compatibility/musl/logf.c b/base/glibc-compatibility/musl/logf.c
index 7ee5d7fe623..e4c2237caa2 100644
--- a/base/glibc-compatibility/musl/logf.c
+++ b/base/glibc-compatibility/musl/logf.c
@@ -53,7 +53,7 @@ float logf(float x)
 	tmp = ix - OFF;
 	i = (tmp >> (23 - LOGF_TABLE_BITS)) % N;
 	k = (int32_t)tmp >> 23; /* arithmetic shift */
-	iz = ix - (tmp & 0x1ff << 23);
+	iz = ix - (tmp & 0xff800000);
 	invc = T[i].invc;
 	logc = T[i].logc;
 	z = (double_t)asfloat(iz);

From 13b77f1daccf332ea0a57fb3e1094b459f954b13 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Tue, 9 May 2023 19:13:39 +0200
Subject: [PATCH 345/689] Fix

---
 src/Interpreters/Cache/LRUFileCachePriority.cpp | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp
index 20a358f9110..1f637d776f2 100644
--- a/src/Interpreters/Cache/LRUFileCachePriority.cpp
+++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp
@@ -145,21 +145,17 @@ LRUFileCachePriority::Iterator LRUFileCachePriority::LRUFileCacheIterator::remov
 
 void LRUFileCachePriority::LRUFileCacheIterator::annul()
 {
-    cache_priority->current_size -= queue_iter->size;
-    queue_iter->size = 0;
+    updateSize(-queue_iter->size);
+    chassert(queue_iter->size == 0);
 }
 
 void LRUFileCachePriority::LRUFileCacheIterator::updateSize(int64_t size)
 {
     cache_priority->current_size += size;
-
-    if (size > 0)
-        CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size);
-    else
-        CurrentMetrics::sub(CurrentMetrics::FilesystemCacheSize, size);
-
     queue_iter->size += size;
 
+    CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size);
+
     chassert(cache_priority->current_size >= 0);
     chassert(queue_iter->size >= 0);
 }

From 2e20603b01b000e0ab10754358cb0e7eb91762de Mon Sep 17 00:00:00 2001
From: Dan Roscigno <dan@roscigno.com>
Date: Tue, 9 May 2023 13:18:09 -0400
Subject: [PATCH 346/689] Update
 docs/en/engines/table-engines/mergetree-family/replacingmergetree.md

---
 .../table-engines/mergetree-family/replacingmergetree.md        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
index 742ea408117..8115b7a14ee 100644
--- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
@@ -96,7 +96,7 @@ SELECT * FROM mySecondReplacingMT FINAL;
 
     Can only be enabled when `ver` is used.
     The row is deleted when the `OPTIMIZE ... FINAL CLEANUP`, or `OPTIMIZE ... FINAL` is used or if the engine settings `clean_deleted_rows` has been set to `Always`.
-    No matter the operation on the data, the version must be increased. If two inserted rows have the same version number, the last inserted one is the one kept.
+  No matter the operation on the data, the version must be increased. If two inserted rows have the same version number, the last inserted row is the one kept.
 
 
 

From 582767e9ab3fd215942504b75286c5c785d84831 Mon Sep 17 00:00:00 2001
From: Dan Roscigno <dan@roscigno.com>
Date: Tue, 9 May 2023 13:18:18 -0400
Subject: [PATCH 347/689] Update
 docs/en/engines/table-engines/mergetree-family/replacingmergetree.md

---
 .../table-engines/mergetree-family/replacingmergetree.md        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
index 8115b7a14ee..601711c9c14 100644
--- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
@@ -92,7 +92,7 @@ SELECT * FROM mySecondReplacingMT FINAL;
 
 `is_deleted` —  Name of a `UInt8` column with the type of row: `1` is a “deleted“ row, `0` is a “state“ row.
 
-    Column data type — `UInt8`.
+  Column data type — `UInt8`.
 
     Can only be enabled when `ver` is used.
     The row is deleted when the `OPTIMIZE ... FINAL CLEANUP`, or `OPTIMIZE ... FINAL` is used or if the engine settings `clean_deleted_rows` has been set to `Always`.

From cd62df48878c4a8a037836c864b4969aa632bb46 Mon Sep 17 00:00:00 2001
From: Dan Roscigno <dan@roscigno.com>
Date: Tue, 9 May 2023 13:18:26 -0400
Subject: [PATCH 348/689] Update
 docs/en/engines/table-engines/mergetree-family/replacingmergetree.md

---
 .../table-engines/mergetree-family/replacingmergetree.md        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
index 601711c9c14..9c18e54a565 100644
--- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
@@ -90,7 +90,7 @@ SELECT * FROM mySecondReplacingMT FINAL;
 
 ### is_deleted
 
-`is_deleted` —  Name of a `UInt8` column with the type of row: `1` is a “deleted“ row, `0` is a “state“ row.
+`is_deleted` —  Name of a column used during a merge to determine whether the data in this row represents the state or is to be deleted; `1` is a “deleted“ row, `0` is a “state“ row.
 
   Column data type — `UInt8`.
 

From 199838e7fb57ba5e09c94bff35fdc0ba76d86a26 Mon Sep 17 00:00:00 2001
From: Dan Roscigno <dan@roscigno.com>
Date: Tue, 9 May 2023 13:18:35 -0400
Subject: [PATCH 349/689] Update
 docs/en/engines/table-engines/mergetree-family/replacingmergetree.md

---
 .../table-engines/mergetree-family/replacingmergetree.md      | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
index 9c18e54a565..0966168a37e 100644
--- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
@@ -94,7 +94,9 @@ SELECT * FROM mySecondReplacingMT FINAL;
 
   Column data type — `UInt8`.
 
-    Can only be enabled when `ver` is used.
+  
+  :::note
+  `is_deleted` can only be enabled when `ver` is used.
     The row is deleted when the `OPTIMIZE ... FINAL CLEANUP`, or `OPTIMIZE ... FINAL` is used or if the engine settings `clean_deleted_rows` has been set to `Always`.
   No matter the operation on the data, the version must be increased. If two inserted rows have the same version number, the last inserted row is the one kept.
 

From c75e056d98f75c5de58900916b04399fa719c30e Mon Sep 17 00:00:00 2001
From: Dan Roscigno <dan@roscigno.com>
Date: Tue, 9 May 2023 13:18:42 -0400
Subject: [PATCH 350/689] Update
 docs/en/engines/table-engines/mergetree-family/replacingmergetree.md

---
 .../table-engines/mergetree-family/replacingmergetree.md        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
index 0966168a37e..b8f4709c246 100644
--- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
@@ -97,7 +97,7 @@ SELECT * FROM mySecondReplacingMT FINAL;
   
   :::note
   `is_deleted` can only be enabled when `ver` is used.
-    The row is deleted when the `OPTIMIZE ... FINAL CLEANUP`, or `OPTIMIZE ... FINAL` is used or if the engine settings `clean_deleted_rows` has been set to `Always`.
+    The row is deleted when `OPTIMIZE ... FINAL CLEANUP` or `OPTIMIZE ... FINAL` is used, or if the engine setting `clean_deleted_rows` has been set to `Always`.
   No matter the operation on the data, the version must be increased. If two inserted rows have the same version number, the last inserted row is the one kept.
 
 

From c20aa0786d5d8a87a2bdfa9991f1c79c078188d8 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Tue, 9 May 2023 19:29:57 +0200
Subject: [PATCH 351/689] More logging

---
 .../Cache/LRUFileCachePriority.cpp            | 31 +++++++++++++------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp
index 1f637d776f2..6deadb3a6ee 100644
--- a/src/Interpreters/Cache/LRUFileCachePriority.cpp
+++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp
@@ -32,7 +32,8 @@ IFileCachePriority::Iterator LRUFileCachePriority::add(
         if (entry.size != 0 && entry.key == key && entry.offset == offset)
             throw Exception(
                 ErrorCodes::LOGICAL_ERROR,
-                "Attempt to add duplicate queue entry to queue. (Key: {}, offset: {}, size: {})",
+                "Attempt to add duplicate queue entry to queue. "
+                "(Key: {}, offset: {}, size: {})",
                 entry.key, entry.offset, entry.size);
     }
 #endif
@@ -46,14 +47,15 @@ IFileCachePriority::Iterator LRUFileCachePriority::add(
             key, offset, size, current_size, getSizeLimit());
     }
 
-    current_size += size;
-
     auto iter = queue.insert(queue.end(), Entry(key, offset, size, key_metadata));
+    current_size += size;
 
     CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size);
     CurrentMetrics::add(CurrentMetrics::FilesystemCacheElements);
 
-    LOG_TEST(log, "Added entry into LRU queue, key: {}, offset: {}, size: {}", key, offset, size);
+    LOG_TEST(
+        log, "Added entry into LRU queue, key: {}, offset: {}, size: {}",
+        key, offset, size);
 
     return std::make_shared<LRUFileCacheIterator>(this, iter);
 }
@@ -81,13 +83,19 @@ LRUFileCachePriority::LRUQueueIterator LRUFileCachePriority::remove(LRUQueueIter
     CurrentMetrics::sub(CurrentMetrics::FilesystemCacheSize, it->size);
     CurrentMetrics::sub(CurrentMetrics::FilesystemCacheElements);
 
-    LOG_TEST(log, "Removed entry from LRU queue, key: {}, offset: {}", it->key, it->offset);
+    LOG_TEST(
+        log, "Removed entry from LRU queue, key: {}, offset: {}, size: {}",
+        it->key, it->offset, it->size);
+
+    it->size = 0;
     return queue.erase(it);
 }
 
 LRUFileCachePriority::LRUFileCacheIterator::LRUFileCacheIterator(
-    LRUFileCachePriority * cache_priority_, LRUFileCachePriority::LRUQueueIterator queue_iter_)
-    : cache_priority(cache_priority_), queue_iter(queue_iter_)
+    LRUFileCachePriority * cache_priority_,
+    LRUFileCachePriority::LRUQueueIterator queue_iter_)
+    : cache_priority(cache_priority_)
+    , queue_iter(queue_iter_)
 {
 }
 
@@ -113,7 +121,8 @@ void LRUFileCachePriority::iterate(IterateFunc && func, const CacheGuard::Lock &
         {
             throw Exception(
                 ErrorCodes::LOGICAL_ERROR,
-                "Mismatch of file segment size in file segment metadata and priority queue: {} != {} ({})",
+                "Mismatch of file segment size in file segment metadata "
+                "and priority queue: {} != {} ({})",
                 it->size, metadata->size(), metadata->file_segment->getInfoForLog());
         }
 
@@ -138,9 +147,11 @@ void LRUFileCachePriority::iterate(IterateFunc && func, const CacheGuard::Lock &
     }
 }
 
-LRUFileCachePriority::Iterator LRUFileCachePriority::LRUFileCacheIterator::remove(const CacheGuard::Lock &)
+LRUFileCachePriority::Iterator
+LRUFileCachePriority::LRUFileCacheIterator::remove(const CacheGuard::Lock &)
 {
-    return std::make_shared<LRUFileCacheIterator>(cache_priority, cache_priority->remove(queue_iter));
+    return std::make_shared<LRUFileCacheIterator>(
+        cache_priority, cache_priority->remove(queue_iter));
 }
 
 void LRUFileCachePriority::LRUFileCacheIterator::annul()

From 0a83f2aee81011c068c0267ec08a5c841a4bcdc3 Mon Sep 17 00:00:00 2001
From: DanRoscigno <dan@roscigno.com>
Date: Tue, 9 May 2023 13:29:59 -0400
Subject: [PATCH 352/689] fix broken note rendering

---
 .../mergetree-family/replacingmergetree.md             | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
index b8f4709c246..7db2f3b465a 100644
--- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
@@ -94,13 +94,13 @@ SELECT * FROM mySecondReplacingMT FINAL;
 
   Column data type — `UInt8`.
 
-  
-  :::note
-  `is_deleted` can only be enabled when `ver` is used.
-    The row is deleted when `OPTIMIZE ... FINAL CLEANUP` or `OPTIMIZE ... FINAL` is used, or if the engine setting `clean_deleted_rows` has been set to `Always`.
-  No matter the operation on the data, the version must be increased. If two inserted rows have the same version number, the last inserted row is the one kept.
+:::note
+`is_deleted` can only be enabled when `ver` is used.
 
+The row is deleted when `OPTIMIZE ... FINAL CLEANUP` or `OPTIMIZE ... FINAL` is used, or if the engine setting `clean_deleted_rows` has been set to `Always`.
 
+No matter the operation on the data, the version must be increased. If two inserted rows have the same version number, the last inserted row is the one kept.
+:::
 
 ## Query clauses
 

From ddce47f79ed74bf07aee4bfc91598a4617167dee Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 9 May 2023 20:17:54 +0200
Subject: [PATCH 353/689] refine table source for regexp tree dictionary

---
 .../ClickHouseDictionarySource.cpp            | 14 ++++-----
 src/Dictionaries/ClickHouseDictionarySource.h |  4 +--
 src/Dictionaries/ExternalQueryBuilder.h       | 10 +++++--
 src/Dictionaries/RegExpTreeDictionary.cpp     | 30 ++++++++++++++++++-
 .../02504_regexp_dictionary_table_source.sql  |  2 +-
 5 files changed, 47 insertions(+), 13 deletions(-)

diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp
index 05cc03b6ced..65147ee664e 100644
--- a/src/Dictionaries/ClickHouseDictionarySource.cpp
+++ b/src/Dictionaries/ClickHouseDictionarySource.cpp
@@ -71,11 +71,11 @@ ClickHouseDictionarySource::ClickHouseDictionarySource(
     : update_time{std::chrono::system_clock::from_time_t(0)}
     , dict_struct{dict_struct_}
     , configuration{configuration_}
-    , query_builder{dict_struct, configuration.db, "", configuration.table, configuration.query, configuration.where, IdentifierQuotingStyle::Backticks}
+    , query_builder(std::make_shared<ExternalQueryBuilder>(dict_struct, configuration.db, "", configuration.table, configuration.query, configuration.where, IdentifierQuotingStyle::Backticks))
     , sample_block{sample_block_}
     , context(context_)
     , pool{createPool(configuration)}
-    , load_all_query{query_builder.composeLoadAllQuery()}
+    , load_all_query{query_builder->composeLoadAllQuery()}
 {
 }
 
@@ -84,7 +84,7 @@ ClickHouseDictionarySource::ClickHouseDictionarySource(const ClickHouseDictionar
     , dict_struct{other.dict_struct}
     , configuration{other.configuration}
     , invalidate_query_response{other.invalidate_query_response}
-    , query_builder{dict_struct, configuration.db, "", configuration.table, configuration.query, configuration.where, IdentifierQuotingStyle::Backticks}
+    , query_builder(std::make_shared<ExternalQueryBuilder>(dict_struct, configuration.db, "", configuration.table, configuration.query, configuration.where, IdentifierQuotingStyle::Backticks))
     , sample_block{other.sample_block}
     , context(Context::createCopy(other.context))
     , pool{createPool(configuration)}
@@ -99,12 +99,12 @@ std::string ClickHouseDictionarySource::getUpdateFieldAndDate()
         time_t hr_time = std::chrono::system_clock::to_time_t(update_time) - configuration.update_lag;
         std::string str_time = DateLUT::instance().timeToString(hr_time);
         update_time = std::chrono::system_clock::now();
-        return query_builder.composeUpdateQuery(configuration.update_field, str_time);
+        return query_builder->composeUpdateQuery(configuration.update_field, str_time);
     }
     else
     {
         update_time = std::chrono::system_clock::now();
-        return query_builder.composeLoadAllQuery();
+        return query_builder->composeLoadAllQuery();
     }
 }
 
@@ -121,13 +121,13 @@ QueryPipeline ClickHouseDictionarySource::loadUpdatedAll()
 
 QueryPipeline ClickHouseDictionarySource::loadIds(const std::vector<UInt64> & ids)
 {
-    return createStreamForQuery(query_builder.composeLoadIdsQuery(ids));
+    return createStreamForQuery(query_builder->composeLoadIdsQuery(ids));
 }
 
 
 QueryPipeline ClickHouseDictionarySource::loadKeys(const Columns & key_columns, const std::vector<size_t> & requested_rows)
 {
-    String query = query_builder.composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::IN_WITH_TUPLES);
+    String query = query_builder->composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::IN_WITH_TUPLES);
     return createStreamForQuery(query);
 }
 
diff --git a/src/Dictionaries/ClickHouseDictionarySource.h b/src/Dictionaries/ClickHouseDictionarySource.h
index f37fae389ac..124d4c8db3f 100644
--- a/src/Dictionaries/ClickHouseDictionarySource.h
+++ b/src/Dictionaries/ClickHouseDictionarySource.h
@@ -78,11 +78,11 @@ private:
     const DictionaryStructure dict_struct;
     const Configuration configuration;
     mutable std::string invalidate_query_response;
-    ExternalQueryBuilder query_builder;
+    ExternalQueryBuilderPtr query_builder;
     Block sample_block;
     ContextMutablePtr context;
     ConnectionPoolWithFailoverPtr pool;
-    const std::string load_all_query;
+    std::string load_all_query;
     Poco::Logger * log = &Poco::Logger::get("ClickHouseDictionarySource");
 
     /// RegExpTreeDictionary is the only dictionary whose structure of attributions differ from the input block.
diff --git a/src/Dictionaries/ExternalQueryBuilder.h b/src/Dictionaries/ExternalQueryBuilder.h
index 353367b12b6..05979d1fb86 100644
--- a/src/Dictionaries/ExternalQueryBuilder.h
+++ b/src/Dictionaries/ExternalQueryBuilder.h
@@ -36,6 +36,10 @@ struct ExternalQueryBuilder
         const std::string & where_,
         IdentifierQuotingStyle quoting_style_);
 
+    ExternalQueryBuilder(const ExternalQueryBuilder &) = default;
+
+    virtual ~ExternalQueryBuilder() = default;
+
     /** Generate a query to load all data. */
     std::string composeLoadAllQuery() const;
 
@@ -61,10 +65,10 @@ struct ExternalQueryBuilder
     std::string composeLoadKeysQuery(const Columns & key_columns, const std::vector<size_t> & requested_rows, LoadKeysMethod method, size_t partition_key_prefix = 0) const;
 
 
-private:
+protected:
     const FormatSettings format_settings = {};
 
-    void composeLoadAllQuery(WriteBuffer & out) const;
+    virtual void composeLoadAllQuery(WriteBuffer & out) const;
 
     /// In the following methods `beg` and `end` specifies which columns to write in expression
 
@@ -93,4 +97,6 @@ private:
     void writeQuoted(const std::string & s, WriteBuffer & out) const;
 };
 
+using ExternalQueryBuilderPtr = std::shared_ptr<ExternalQueryBuilder>;
+
 }
diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp
index 68f21087f0c..a0c66f42e8b 100644
--- a/src/Dictionaries/RegExpTreeDictionary.cpp
+++ b/src/Dictionaries/RegExpTreeDictionary.cpp
@@ -20,6 +20,7 @@
 #include <Functions/Regexps.h>
 #include <Functions/checkHyperscanRegexp.h>
 #include <QueryPipeline/QueryPipeline.h>
+#include <Processors/Sources/BlocksListSource.h>
 
 #include <Dictionaries/ClickHouseDictionarySource.h>
 #include <Dictionaries/DictionaryFactory.h>
@@ -31,7 +32,6 @@
 
 #include <re2_st/stringpiece.h>
 
-#include "Processors/Sources/BlocksListSource.h"
 #include "config.h"
 
 #if USE_VECTORSCAN
@@ -87,6 +87,32 @@ namespace
     }
 }
 
+struct ExternalRegexpQueryBuilder final : public ExternalQueryBuilder
+{
+    explicit ExternalRegexpQueryBuilder(const ExternalQueryBuilder & builder) : ExternalQueryBuilder(builder) {}
+
+    void composeLoadAllQuery(WriteBuffer & out) const override
+    {
+        writeString("SELECT id, parent_id, regexp, keys, values FROM ", out);
+        if (!db.empty())
+        {
+            writeQuoted(db, out);
+            writeChar('.', out);
+        }
+        if (!schema.empty())
+        {
+            writeQuoted(schema, out);
+            writeChar('.', out);
+        }
+        writeQuoted(table, out);
+        if (!where.empty())
+        {
+            writeString(" WHERE ", out);
+            writeString(where, out);
+        }
+    }
+};
+
 struct RegExpTreeDictionary::RegexTreeNode
 {
     std::vector<UInt64> children;
@@ -385,6 +411,8 @@ RegExpTreeDictionary::RegExpTreeDictionary(
         sample_block.insert(ColumnWithTypeAndName(std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), kKeys));
         sample_block.insert(ColumnWithTypeAndName(std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), kValues));
         ch_source->sample_block = std::move(sample_block);
+        ch_source->query_builder = std::make_shared<ExternalRegexpQueryBuilder>(*ch_source->query_builder);
+        ch_source->load_all_query = ch_source->query_builder->composeLoadAllQuery();
     }
 
     loadData();
diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql b/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql
index e7937b347a0..15e8adce403 100644
--- a/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql
+++ b/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql
@@ -29,7 +29,7 @@ create dictionary regexp_dict1
     comment String default 'nothing'
 )
 PRIMARY KEY(regexp)
-SOURCE(CLICKHOUSE(QUERY concat('select * from ', currentDatabase() , '.regexp_dictionary_source_table')))
+SOURCE(CLICKHOUSE(TABLE 'regexp_dictionary_source_table'))
 LIFETIME(0)
 LAYOUT(regexp_tree);
 

From 432250a85db01c8f5dd56b8f6cc30593ba4a6473 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 20:28:38 +0200
Subject: [PATCH 354/689] Update instructions for building under OSX to
 clang-16

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docs/en/development/build-cross-osx.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/en/development/build-cross-osx.md b/docs/en/development/build-cross-osx.md
index e6e5bd6ca4d..ce8d1b77526 100644
--- a/docs/en/development/build-cross-osx.md
+++ b/docs/en/development/build-cross-osx.md
@@ -11,14 +11,14 @@ This is intended for continuous integration checks that run on Linux servers. If
 
 The cross-build for macOS is based on the [Build instructions](../development/build.md), follow them first.
 
-## Install Clang-15
+## Install Clang-16
 
 Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup.
 For example the commands for Bionic are like:
 
 ``` bash
-sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-15 main" >> /etc/apt/sources.list
-sudo apt-get install clang-15
+sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-16 main" >> /etc/apt/sources.list
+sudo apt-get install clang-16
 ```
 
 ## Install Cross-Compilation Toolset {#install-cross-compilation-toolset}
@@ -55,7 +55,7 @@ curl -L 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX1
 cd ClickHouse
 mkdir build-darwin
 cd build-darwin
-CC=clang-15 CXX=clang++-15 cmake -DCMAKE_AR:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ar -DCMAKE_INSTALL_NAME_TOOL=${CCTOOLS}/bin/x86_64-apple-darwin-install_name_tool -DCMAKE_RANLIB:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ranlib -DLINKER_NAME=${CCTOOLS}/bin/x86_64-apple-darwin-ld -DCMAKE_TOOLCHAIN_FILE=cmake/darwin/toolchain-x86_64.cmake ..
+CC=clang-16 CXX=clang++-16 cmake -DCMAKE_AR:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ar -DCMAKE_INSTALL_NAME_TOOL=${CCTOOLS}/bin/x86_64-apple-darwin-install_name_tool -DCMAKE_RANLIB:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ranlib -DLINKER_NAME=${CCTOOLS}/bin/x86_64-apple-darwin-ld -DCMAKE_TOOLCHAIN_FILE=cmake/darwin/toolchain-x86_64.cmake ..
 ninja
 ```
 

From 184efcb77aadb7f407bccb46dde87083ce2e7721 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 20:32:03 +0200
Subject: [PATCH 355/689] Get back -fno-sanitize-address-use-after-scope quirk
 for llvm/clang-15

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 cmake/sanitize.cmake | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake
index 0dd7fda4b45..b2fbdb256fd 100644
--- a/cmake/sanitize.cmake
+++ b/cmake/sanitize.cmake
@@ -11,6 +11,15 @@ set (SAN_FLAGS "${SAN_FLAGS} -g -fno-omit-frame-pointer -DSANITIZER")
 if (SANITIZE)
     if (SANITIZE STREQUAL "address")
         set (ASAN_FLAGS "-fsanitize=address -fsanitize-address-use-after-scope")
+        if (COMPILER_CLANG)
+            if (${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER_EQUAL 15 AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 16)
+                # LLVM-15 has a bug in Address Sanitizer, preventing the usage
+                # of 'sanitize-address-use-after-scope', see [1].
+                #
+                #   [1]: https://github.com/llvm/llvm-project/issues/58633
+                set (ASAN_FLAGS "${ASAN_FLAGS} -fno-sanitize-address-use-after-scope")
+            endif()
+        endif()
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${ASAN_FLAGS}")
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${ASAN_FLAGS}")
 

From 833652b7c9c1e9f72511e2dc6d707656de4a61ff Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 20:36:39 +0200
Subject: [PATCH 356/689] Revert "Suppress clang-analyzer-cplusplus.NewDelete
 in MsgPackRowInputFormat"

Let's try to revert this quirk during upgrading to clang 16.

This reverts commit c1e70169d2eec449012ac1bb555347a840c9c181.
---
 src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp
index eeca14176cc..3600fcd4086 100644
--- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp
+++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp
@@ -2,15 +2,6 @@
 
 #if USE_MSGPACK
 
-/// FIXME: there is some issue with clang-15, that incorrectly detect a
-/// "Attempt to free released memory" in msgpack::unpack(), because of delete
-/// operator for zone (from msgpack/v1/detail/cpp11_zone.hpp), hence NOLINT
-///
-/// NOTE: that I was not able to suppress it locally, only with
-/// NOLINTBEGIN/NOLINTEND
-//
-// NOLINTBEGIN(clang-analyzer-cplusplus.NewDelete)
-
 #include <cstdlib>
 #include <Common/assert_cast.h>
 #include <IO/ReadHelpers.h>
@@ -678,8 +669,6 @@ void registerMsgPackSchemaReader(FormatFactory & factory)
 
 }
 
-// NOLINTEND(clang-analyzer-cplusplus.NewDelete)
-
 #else
 
 namespace DB

From e1d7f466cb7454ca3ddaaf560cbdaf884e4d3cb6 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 20:43:52 +0200
Subject: [PATCH 357/689] Disable misc-use-anonymous-namespace check in
 clang-tidy

It is too noisy during upgrade:

$ grep error: build_log.log | awk '{print $NF}' | sort | uniq -c
      1 [bugprone-standalone-empty,-warnings-as-errors]
      1 [misc-misplaced-const,-warnings-as-errors]
   2147 [misc-use-anonymous-namespace,-warnings-as-errors]
      6 [modernize-loop-convert,-warnings-as-errors]
      9 [readability-redundant-string-cstr,-warnings-as-errors]

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .clang-tidy | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.clang-tidy b/.clang-tidy
index 7f78143ec3d..2c5f8c918ba 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -111,6 +111,7 @@ Checks: '*,
     -misc-no-recursion,
     -misc-non-private-member-variables-in-classes,
     -misc-confusable-identifiers, # useful but slooow
+    -misc-use-anonymous-namespace,
 
     -modernize-avoid-c-arrays,
     -modernize-concat-nested-namespaces,

From 051d0e57ff0dfd0493a85266a69962101f3f863e Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 20:47:52 +0200
Subject: [PATCH 358/689] Suppress bugprone-standalone-empty in DatabaseMySQL

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Databases/MySQL/DatabaseMySQL.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp
index 9bd1dd65ab3..70bd32efed9 100644
--- a/src/Databases/MySQL/DatabaseMySQL.cpp
+++ b/src/Databases/MySQL/DatabaseMySQL.cpp
@@ -67,7 +67,7 @@ DatabaseMySQL::DatabaseMySQL(
     try
     {
         /// Test that the database is working fine; it will also fetch tables.
-        empty();
+        empty(); // NOLINT(bugprone-standalone-empty)
     }
     catch (...)
     {

From 930c8c3043fbb2ff462f6ee3d14b2568271c774d Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Tue, 9 May 2023 18:56:09 +0000
Subject: [PATCH 359/689] Use own fiber local implementation

---
 src/Common/AsyncTaskExecutor.cpp         | 10 +++++
 src/Common/AsyncTaskExecutor.h           | 25 +++++++++++
 src/Common/OpenTelemetryTraceContext.cpp | 56 ++++++++++--------------
 3 files changed, 58 insertions(+), 33 deletions(-)

diff --git a/src/Common/AsyncTaskExecutor.cpp b/src/Common/AsyncTaskExecutor.cpp
index 4e2fb18fb78..d0c6454a849 100644
--- a/src/Common/AsyncTaskExecutor.cpp
+++ b/src/Common/AsyncTaskExecutor.cpp
@@ -3,11 +3,18 @@
 namespace DB
 {
 
+thread_local const Fiber * current_fiber = nullptr;
+
 AsyncTaskExecutor::AsyncTaskExecutor(std::unique_ptr<AsyncTask> task_) : task(std::move(task_))
 {
     createFiber();
 }
 
+const Fiber * AsyncTaskExecutor::getCurrentFiber()
+{
+    return current_fiber;
+}
+
 void AsyncTaskExecutor::resume()
 {
     if (routine_is_finished)
@@ -31,7 +38,10 @@ void AsyncTaskExecutor::resume()
 
 void AsyncTaskExecutor::resumeUnlocked()
 {
+    const auto * parent_fiber = current_fiber;
+    current_fiber = &fiber;
     fiber = std::move(fiber).resume();
+    current_fiber = parent_fiber;
 }
 
 void AsyncTaskExecutor::cancel()
diff --git a/src/Common/AsyncTaskExecutor.h b/src/Common/AsyncTaskExecutor.h
index f749c3066fc..cf7cdc5ad82 100644
--- a/src/Common/AsyncTaskExecutor.h
+++ b/src/Common/AsyncTaskExecutor.h
@@ -74,6 +74,7 @@ public:
         ERROR = 4,
     };
 #endif
+    static const Fiber * getCurrentFiber();
 
 protected:
     /// Method that is called in resume() before actual fiber resuming.
@@ -118,6 +119,30 @@ private:
     std::unique_ptr<AsyncTask> task;
 };
 
+/// Simple class for storing fiber local variables.
+template <typename T>
+class FiberLocalVariable
+{
+public:
+    T & operator*()
+    {
+        return get();
+    }
+
+    T * operator->()
+    {
+        return &get();
+    }
+
+private:
+    T & get()
+    {
+        return data[AsyncTaskExecutor::getCurrentFiber()];
+    }
+
+    std::unordered_map<const Fiber *, T> data;
+};
+
 String getSocketTimeoutExceededMessageByTimeoutType(AsyncEventTimeoutType type, Poco::Timespan timeout, const String & socket_description);
 
 }
diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp
index 86ce30941a3..f25acc571d8 100644
--- a/src/Common/OpenTelemetryTraceContext.cpp
+++ b/src/Common/OpenTelemetryTraceContext.cpp
@@ -6,26 +6,16 @@
 #include <base/hex.h>
 #include <Core/Settings.h>
 #include <IO/Operators.h>
-#include <boost/fiber/fss.hpp>
+
+#include <Common/AsyncTaskExecutor.h>
 
 namespace DB
 {
 namespace OpenTelemetry
 {
 
-static TracingContextOnThread & getCurrentThreadTraceContext()
-{
-    static boost::fibers::fiber_specific_ptr<TracingContextOnThread> current_thread_trace_context;
-
-    auto * ptr = current_thread_trace_context.get();
-    if (unlikely(!ptr))
-    {
-        ptr = new TracingContextOnThread();
-        current_thread_trace_context.reset(ptr);
-    }
-    return *ptr;
-}
-
+/// This code can be executed inside fiber, we should use fiber local context.
+thread_local FiberLocalVariable<TracingContextOnThread> current_fiber_trace_context;
 
 bool Span::addAttribute(std::string_view name, UInt64 value) noexcept
 {
@@ -117,7 +107,7 @@ bool Span::addAttributeImpl(std::string_view name, std::string_view value) noexc
 
 SpanHolder::SpanHolder(std::string_view _operation_name, SpanKind _kind)
 {
-    if (!getCurrentThreadTraceContext().isTraceEnabled())
+    if (!current_fiber_trace_context->isTraceEnabled())
     {
         return;
     }
@@ -125,8 +115,8 @@ SpanHolder::SpanHolder(std::string_view _operation_name, SpanKind _kind)
     /// Use try-catch to make sure the ctor is exception safe.
     try
     {
-        this->trace_id = getCurrentThreadTraceContext().trace_id;
-        this->parent_span_id = getCurrentThreadTraceContext().span_id;
+        this->trace_id =current_fiber_trace_context->trace_id;
+        this->parent_span_id =current_fiber_trace_context->span_id;
         this->span_id = thread_local_rng(); // create a new id for this span
         this->operation_name = _operation_name;
         this->kind = _kind;
@@ -145,7 +135,7 @@ SpanHolder::SpanHolder(std::string_view _operation_name, SpanKind _kind)
     }
 
     /// Set current span as parent of other spans created later on this thread.
-    getCurrentThreadTraceContext().span_id = this->span_id;
+   current_fiber_trace_context->span_id = this->span_id;
 }
 
 void SpanHolder::finish() noexcept
@@ -154,12 +144,12 @@ void SpanHolder::finish() noexcept
         return;
 
     // First of all, restore old value of current span.
-    assert(getCurrentThreadTraceContext().span_id == span_id);
-    getCurrentThreadTraceContext().span_id = parent_span_id;
+    assert(current_fiber_trace_context->span_id == span_id);
+    current_fiber_trace_context->span_id = parent_span_id;
 
     try
     {
-        auto log = getCurrentThreadTraceContext().span_log.lock();
+        auto log =current_fiber_trace_context->span_log.lock();
 
         /// The log might be disabled, check it before use
         if (log)
@@ -282,7 +272,7 @@ void TracingContext::serialize(WriteBuffer & buf) const
 
 const TracingContextOnThread & CurrentContext()
 {
-    return getCurrentThreadTraceContext();
+    return*current_fiber_trace_context;
 }
 
 void TracingContextOnThread::reset() noexcept
@@ -304,7 +294,7 @@ TracingContextHolder::TracingContextHolder(
     /// If any exception is raised during the construction, the tracing is not enabled on current thread.
     try
     {
-        if (getCurrentThreadTraceContext().isTraceEnabled())
+        if (current_fiber_trace_context->isTraceEnabled())
         {
             ///
             /// This is not the normal case,
@@ -317,15 +307,15 @@ TracingContextHolder::TracingContextHolder(
             /// So this branch ensures this class can be instantiated multiple times on one same thread safely.
             ///
             this->is_context_owner = false;
-            this->root_span.trace_id = getCurrentThreadTraceContext().trace_id;
-            this->root_span.parent_span_id = getCurrentThreadTraceContext().span_id;
+            this->root_span.trace_id =current_fiber_trace_context->trace_id;
+            this->root_span.parent_span_id =current_fiber_trace_context->span_id;
             this->root_span.span_id = thread_local_rng();
             this->root_span.operation_name = _operation_name;
             this->root_span.start_time_us
                 = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
 
             /// Set the root span as parent of other spans created on current thread
-            getCurrentThreadTraceContext().span_id = this->root_span.span_id;
+           current_fiber_trace_context->span_id = this->root_span.span_id;
             return;
         }
 
@@ -369,10 +359,10 @@ TracingContextHolder::TracingContextHolder(
     }
 
     /// Set up trace context on current thread only when the root span is successfully initialized.
-    getCurrentThreadTraceContext() = _parent_trace_context;
-    getCurrentThreadTraceContext().span_id = this->root_span.span_id;
-    getCurrentThreadTraceContext().trace_flags = TRACE_FLAG_SAMPLED;
-    getCurrentThreadTraceContext().span_log = _span_log;
+   *current_fiber_trace_context = _parent_trace_context;
+   current_fiber_trace_context->span_id = this->root_span.span_id;
+   current_fiber_trace_context->trace_flags = TRACE_FLAG_SAMPLED;
+   current_fiber_trace_context->span_log = _span_log;
 }
 
 TracingContextHolder::~TracingContextHolder()
@@ -384,7 +374,7 @@ TracingContextHolder::~TracingContextHolder()
 
     try
     {
-        auto shared_span_log = getCurrentThreadTraceContext().span_log.lock();
+        auto shared_span_log =current_fiber_trace_context->span_log.lock();
         if (shared_span_log)
         {
             try
@@ -415,11 +405,11 @@ TracingContextHolder::~TracingContextHolder()
     if (this->is_context_owner)
     {
         /// Clear the context on current thread
-        getCurrentThreadTraceContext().reset();
+       current_fiber_trace_context->reset();
     }
     else
     {
-        getCurrentThreadTraceContext().span_id = this->root_span.parent_span_id;
+       current_fiber_trace_context->span_id = this->root_span.parent_span_id;
     }
 }
 

From 66971662de3a5cb7c5a3edc29b4a103c7c862329 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Tue, 9 May 2023 18:56:59 +0000
Subject: [PATCH 360/689] Update cmake

---
 contrib/boost-cmake/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt
index c8be40be1d4..cb0db5622a8 100644
--- a/contrib/boost-cmake/CMakeLists.txt
+++ b/contrib/boost-cmake/CMakeLists.txt
@@ -174,6 +174,7 @@ set (SRCS_FIBER
 add_library (_boost_fiber ${SRCS_FIBER})
 add_library (boost::fiber ALIAS _boost_fiber)
 target_include_directories (_boost_fiber PRIVATE ${LIBRARY_DIR})
+target_link_libraries(_boost_fiber PRIVATE _boost_context)
 
 # coroutine
 

From df97f5890a115b8a4ee495f677c58c2818ad0c1e Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Tue, 9 May 2023 20:58:59 +0200
Subject: [PATCH 361/689] Done

---
 .../0_stateless/02706_kolmogorov_smirnov_test_scipy.python    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02706_kolmogorov_smirnov_test_scipy.python b/tests/queries/0_stateless/02706_kolmogorov_smirnov_test_scipy.python
index a2a34b57c90..01f245e0cf0 100644
--- a/tests/queries/0_stateless/02706_kolmogorov_smirnov_test_scipy.python
+++ b/tests/queries/0_stateless/02706_kolmogorov_smirnov_test_scipy.python
@@ -62,8 +62,8 @@ def test_ks_all_alternatives(rvs1, rvs2):
 
 
 def test_kolmogorov_smirnov():
-    rvs1 = np.round(stats.norm.rvs(loc=1, scale=5, size=10), 2)
-    rvs2 = np.round(stats.norm.rvs(loc=1.5, scale=5, size=20), 2)
+    rvs1 = np.round(stats.norm.rvs(loc=1, scale=5, size=100), 2)
+    rvs2 = np.round(stats.norm.rvs(loc=1.5, scale=5, size=200), 2)
     test_ks_all_alternatives(rvs1, rvs2)
 
     rvs1 = np.round(stats.norm.rvs(loc=13, scale=1, size=100), 2)

From fa4099bef8eca70feb42bc7c8eacaf8904a951ab Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 20:59:24 +0200
Subject: [PATCH 362/689] Update
 02417_opentelemetry_insert_on_distributed_table for clang 16

CI: https://s3.amazonaws.com/clickhouse-test-reports/49678/6e9985a94d88d49169a48551ba8910c2b373d0de/stateless_tests__debug__[1_5].html
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 ...entelemetry_insert_on_distributed_table.reference | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference
index 312904a3cf9..02df9aa06c2 100644
--- a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference
+++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference
@@ -1,6 +1,6 @@
 ===1===
-{"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const DB::Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
-{"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const DB::Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
+{"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const Block const DB::Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
+{"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const Block const DB::Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
 1
 ===2===
 {"operation_name":"void DB::DistributedAsyncInsertDirectoryQueue::processFile(const std::string &)","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
@@ -8,11 +8,11 @@
 3
 2
 ===3===
-{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
-{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
+{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
+{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
 1
 ===4===
-{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
-{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
+{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
+{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
 3
 2

From d03ae2abfafe4268ffa8c9bbc9872d989646837c Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 21:19:37 +0200
Subject: [PATCH 363/689] Fix modernize-loop-convert clang-tidy warning

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/DataTypes/Serializations/SerializationArray.cpp        | 4 ++--
 src/Interpreters/InterpreterSelectQuery.cpp                | 6 +++---
 src/Interpreters/convertFieldToType.cpp                    | 4 ++--
 src/Planner/PlannerJoinTree.cpp                            | 4 ++--
 src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp | 4 ++--
 src/Storages/MergeTree/MutateTask.cpp                      | 4 +---
 6 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/src/DataTypes/Serializations/SerializationArray.cpp b/src/DataTypes/Serializations/SerializationArray.cpp
index 4a477958daa..cedcca870dd 100644
--- a/src/DataTypes/Serializations/SerializationArray.cpp
+++ b/src/DataTypes/Serializations/SerializationArray.cpp
@@ -32,9 +32,9 @@ void SerializationArray::serializeBinary(const Field & field, WriteBuffer & ostr
 {
     const Array & a = field.get<const Array &>();
     writeVarUInt(a.size(), ostr);
-    for (size_t i = 0; i < a.size(); ++i)
+    for (const auto & i : a)
     {
-        nested->serializeBinary(a[i], ostr, settings);
+        nested->serializeBinary(i, ostr, settings);
     }
 }
 
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index afd34cb044e..f7ddf0e1ede 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -301,13 +301,13 @@ void checkAccessRightsForSelect(
 }
 
 ASTPtr parseAdditionalFilterConditionForTable(
-    const Map & setting,
+    const Map & additional_table_filters,
     const DatabaseAndTableWithAlias & target,
     const Context & context)
 {
-    for (size_t i = 0; i < setting.size(); ++i)
+    for (const auto & additional_filter : additional_table_filters)
     {
-        const auto & tuple = setting[i].safeGet<const Tuple &>();
+        const auto & tuple = additional_filter.safeGet<const Tuple &>();
         auto & table = tuple.at(0).safeGet<String>();
         auto & filter = tuple.at(1).safeGet<String>();
 
diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp
index 08ac2fce556..dc61e748db6 100644
--- a/src/Interpreters/convertFieldToType.cpp
+++ b/src/Interpreters/convertFieldToType.cpp
@@ -448,9 +448,9 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
         {
             Object object;
             const auto & map = src.get<const Map &>();
-            for (size_t i = 0; i < map.size(); ++i)
+            for (const auto & element : map)
             {
-                const auto & map_entry = map[i].get<Tuple>();
+                const auto & map_entry = element.get<Tuple>();
                 const auto & key = map_entry[0];
                 const auto & value = map_entry[1];
 
diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index 93a9945b1ca..4f091f73187 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -447,9 +447,9 @@ FilterDAGInfo buildAdditionalFiltersIfNeeded(const StoragePtr & storage,
     auto const & storage_id = storage->getStorageID();
 
     ASTPtr additional_filter_ast;
-    for (size_t i = 0; i < additional_filters.size(); ++i)
+    for (const auto & additional_filter : additional_filters)
     {
-        const auto & tuple = additional_filters[i].safeGet<const Tuple &>();
+        const auto & tuple = additional_filter.safeGet<const Tuple &>();
         auto const & table = tuple.at(0).safeGet<String>();
         auto const & filter = tuple.at(1).safeGet<String>();
 
diff --git a/src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp b/src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp
index d290280bdbf..c2ee5923c01 100644
--- a/src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp
+++ b/src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp
@@ -281,9 +281,9 @@ static void columnMapToContainer(const ColumnMap * col_map, size_t row_num, Cont
     Field field;
     col_map->get(row_num, field);
     const auto & map_field = field.get<Map>();
-    for (size_t i = 0; i < map_field.size(); ++i)
+    for (const auto & map_element : map_field)
     {
-        const auto & map_entry = map_field[i].get<Tuple>();
+        const auto & map_entry = map_element.get<Tuple>();
 
         String entry_key;
         String entry_value;
diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp
index a3fa210ac42..1d9423fc9b4 100644
--- a/src/Storages/MergeTree/MutateTask.cpp
+++ b/src/Storages/MergeTree/MutateTask.cpp
@@ -464,10 +464,8 @@ static std::set<MergeTreeIndexPtr> getIndicesToRecalculate(
     ASTPtr indices_recalc_expr_list = std::make_shared<ASTExpressionList>();
     const auto & indices = metadata_snapshot->getSecondaryIndices();
 
-    for (size_t i = 0; i < indices.size(); ++i)
+    for (const auto & index : indices)
     {
-        const auto & index = indices[i];
-
         bool has_index =
             source_part->checksums.has(INDEX_FILE_PREFIX + index.name + ".idx") ||
             source_part->checksums.has(INDEX_FILE_PREFIX + index.name + ".idx2");

From 0610865811edd223cd34925921511e50d588f81e Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 21:22:05 +0200
Subject: [PATCH 364/689] Fix readability-redundant-string-cstr clang-tidy
 warning

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Parsers/Access/ParserCreateQuotaQuery.cpp            | 2 +-
 src/Parsers/Access/ParserDropAccessEntityQuery.cpp       | 4 ++--
 src/Parsers/Access/ParserShowAccessEntitiesQuery.cpp     | 4 ++--
 src/Parsers/Access/ParserShowCreateAccessEntityQuery.cpp | 8 ++++----
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/Parsers/Access/ParserCreateQuotaQuery.cpp b/src/Parsers/Access/ParserCreateQuotaQuery.cpp
index a67051be398..e40aa8db913 100644
--- a/src/Parsers/Access/ParserCreateQuotaQuery.cpp
+++ b/src/Parsers/Access/ParserCreateQuotaQuery.cpp
@@ -82,7 +82,7 @@ namespace
         {
             for (auto qt : collections::range(QuotaType::MAX))
             {
-                if (ParserKeyword{QuotaTypeInfo::get(qt).keyword.c_str()}.ignore(pos, expected))
+                if (ParserKeyword{QuotaTypeInfo::get(qt).keyword}.ignore(pos, expected))
                 {
                     quota_type = qt;
                     return true;
diff --git a/src/Parsers/Access/ParserDropAccessEntityQuery.cpp b/src/Parsers/Access/ParserDropAccessEntityQuery.cpp
index 1a6784d2d3c..d21164bc1a2 100644
--- a/src/Parsers/Access/ParserDropAccessEntityQuery.cpp
+++ b/src/Parsers/Access/ParserDropAccessEntityQuery.cpp
@@ -17,8 +17,8 @@ namespace
         for (auto i : collections::range(AccessEntityType::MAX))
         {
             const auto & type_info = AccessEntityTypeInfo::get(i);
-            if (ParserKeyword{type_info.name.c_str()}.ignore(pos, expected)
-                || (!type_info.alias.empty() && ParserKeyword{type_info.alias.c_str()}.ignore(pos, expected)))
+            if (ParserKeyword{type_info.name}.ignore(pos, expected)
+                || (!type_info.alias.empty() && ParserKeyword{type_info.alias}.ignore(pos, expected)))
             {
                 type = i;
                 return true;
diff --git a/src/Parsers/Access/ParserShowAccessEntitiesQuery.cpp b/src/Parsers/Access/ParserShowAccessEntitiesQuery.cpp
index 3953c28c356..15cb815f457 100644
--- a/src/Parsers/Access/ParserShowAccessEntitiesQuery.cpp
+++ b/src/Parsers/Access/ParserShowAccessEntitiesQuery.cpp
@@ -15,8 +15,8 @@ namespace
         for (auto i : collections::range(AccessEntityType::MAX))
         {
             const auto & type_info = AccessEntityTypeInfo::get(i);
-            if (ParserKeyword{type_info.plural_name.c_str()}.ignore(pos, expected)
-                || (!type_info.plural_alias.empty() && ParserKeyword{type_info.plural_alias.c_str()}.ignore(pos, expected)))
+            if (ParserKeyword{type_info.plural_name}.ignore(pos, expected)
+                || (!type_info.plural_alias.empty() && ParserKeyword{type_info.plural_alias}.ignore(pos, expected)))
             {
                 type = i;
                 return true;
diff --git a/src/Parsers/Access/ParserShowCreateAccessEntityQuery.cpp b/src/Parsers/Access/ParserShowCreateAccessEntityQuery.cpp
index 15622062961..17caa6366f1 100644
--- a/src/Parsers/Access/ParserShowCreateAccessEntityQuery.cpp
+++ b/src/Parsers/Access/ParserShowCreateAccessEntityQuery.cpp
@@ -25,8 +25,8 @@ namespace
         for (auto i : collections::range(AccessEntityType::MAX))
         {
             const auto & type_info = AccessEntityTypeInfo::get(i);
-            if (ParserKeyword{type_info.name.c_str()}.ignore(pos, expected)
-                || (!type_info.alias.empty() && ParserKeyword{type_info.alias.c_str()}.ignore(pos, expected)))
+            if (ParserKeyword{type_info.name}.ignore(pos, expected)
+                || (!type_info.alias.empty() && ParserKeyword{type_info.alias}.ignore(pos, expected)))
             {
                 type = i;
                 plural = false;
@@ -37,8 +37,8 @@ namespace
         for (auto i : collections::range(AccessEntityType::MAX))
         {
             const auto & type_info = AccessEntityTypeInfo::get(i);
-            if (ParserKeyword{type_info.plural_name.c_str()}.ignore(pos, expected)
-                || (!type_info.plural_alias.empty() && ParserKeyword{type_info.plural_alias.c_str()}.ignore(pos, expected)))
+            if (ParserKeyword{type_info.plural_name}.ignore(pos, expected)
+                || (!type_info.plural_alias.empty() && ParserKeyword{type_info.plural_alias}.ignore(pos, expected)))
             {
                 type = i;
                 plural = true;

From d8dd50a9c6ca29abc18bfbcd6fcc94d0a3f5f491 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 21:27:21 +0200
Subject: [PATCH 365/689] Fix misc-misplaced-const clang-tidy warning

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Processors/Formats/Impl/TemplateRowInputFormat.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp b/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp
index 1a519fa977f..8a09e800fa7 100644
--- a/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp
+++ b/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp
@@ -169,7 +169,7 @@ bool TemplateRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & col
 {
     out << "Suffix does not match: ";
     size_t last_successfully_parsed_idx = format_reader->getFormatDataIdx() + 1;
-    const ReadBuffer::Position row_begin_pos = buf->position();
+    auto * const row_begin_pos = buf->position();
     bool caught = false;
     try
     {

From 9b3bcd0870f2d62e8636a662dc7e9eaa20b7a958 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 9 May 2023 21:32:13 +0200
Subject: [PATCH 366/689] Fix stack-use-after-scope in
 gtest_resource_manager_static

Reordering of the static variables should be enough to fix this:

    [ RUN      ] IOResourceStaticResourceManager.Prioritization
    =================================================================
    ==13==ERROR: AddressSanitizer: stack-use-after-scope on address 0x7f7d8d621970 at pc 0x5636b80dcbb2 bp 0x7f7c48e47dd0 sp 0x7f7c48e47dc8
    READ of size 8 at 0x7f7d8d621970 thread T3975 (ThreadPool)
        #0 0x5636b80dcbb1 in IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_1::operator()(long) const build_docker/./src/IO/Resource/tests/gtest_resource_manager_static.cpp:78:13
        #1 0x5636b80dcbb1 in IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0::operator()() const build_docker/./src/IO/Resource/tests/gtest_resource_manager_static.cpp:95:21
        #2 0x5636b80dcbb1 in decltype(std::declval<IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&>()()) std::__1::__invoke[abi:v15000]<IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&>(IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23
        #3 0x5636b80dcbb1 in decltype(auto) std::__1::__apply_tuple_impl[abi:v15000]<IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&, std::__1::tuple<>&>(IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&, std::__1::tuple<>&, std::__1::__tuple_indices<>) build_docker/./contrib/llvm-project/libcxx/include/tuple:1789:1
        #4 0x5636b80dcbb1 in decltype(auto) std::__1::apply[abi:v15000]<IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&, std::__1::tuple<>&>(IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&, std::__1::tuple<>&) build_docker/./contrib/llvm-project/libcxx/include/tuple:1798:1
        #5 0x5636b80dcbb1 in ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0>(IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&&)::'lambda'()::operator()() build_docker/./src/Common/ThreadPool.h:228:13
        #6 0x5636b80dcbb1 in decltype(std::declval<IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0>()()) std::__1::__invoke[abi:v15000]<ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0>(IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&&)::'lambda'()&>(IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23
        #7 0x5636b80dcbb1 in void std::__1::__invoke_void_return_wrapper<void, true>::__call<ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0>(IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&&)::'lambda'()&>(ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0>(IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&&)::'lambda'()&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:479:9
        #8 0x5636b80dcbb1 in std::__1::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0>(IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&&)::'lambda'(), void ()>::operator()[abi:v15000]() build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:235:12
        #9 0x5636b80dcbb1 in void std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0>(IOResourceStaticResourceManager_Prioritization_Test::TestBody()::$_0&&)::'lambda'(), void ()>>(std::__1::__function::__policy_storage const*) build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:716:16
        #10 0x5636ea219310 in std::__1::__function::__policy_func<void ()>::operator()[abi:v15000]() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:848:16
        #11 0x5636ea219310 in std::__1::function<void ()>::operator()() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:1187:12
        #12 0x5636ea219310 in ThreadPoolImpl<std::__1::thread>::worker(std::__1::__list_iterator<std::__1::thread, void*>) build_docker/./src/Common/ThreadPool.cpp:416:13
        #13 0x5636ea2258ac in void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()::operator()() const build_docker/./src/Common/ThreadPool.cpp:180:73
        #14 0x5636ea2258ac in decltype(std::declval<void>()()) std::__1::__invoke[abi:v15000]<void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23
        #15 0x5636ea2258ac in void std::__1::__thread_execute[abi:v15000]<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(std::__1::tuple<void, void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>&, std::__1::__tuple_indices<>) build_docker/./contrib/llvm-project/libcxx/include/thread:284:5
        #16 0x5636ea2258ac in void* std::__1::__thread_proxy[abi:v15000]<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>>(void*) build_docker/./contrib/llvm-project/libcxx/include/thread:295:5
        #17 0x7f7d8fcf1608 in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x8608) (BuildId: 7b4536f41cdaa5888408e82d0836e33dcf436466)
        #18 0x7f7d8fc16132 in __clone (/lib/x86_64-linux-gnu/libc.so.6+0x11f132) (BuildId: 1878e6b475720c7c51969e69ab2d276fae6d1dee)

    Address 0x7f7d8d621970 is located in stack of thread T0 at offset 368 in frame
        #0 0x5636b80d9bef in IOResourceStaticResourceManager_Prioritization_Test::TestBody() build_docker/./src/IO/Resource/tests/gtest_resource_manager_static.cpp:46

      This frame has 10 object(s):
        [32, 36) 'requests_per_thead' (line 48)
        [48, 208) 't' (line 49)
        [272, 296) 'ref.tmp' (line 51)
        [336, 352) 'last_priority' (line 74)
        [368, 376) 'check' (line 75) <== Memory access at offset 368 is inside this variable
        [400, 424) 'name' (line 83)
        [464, 512) 'ref.tmp13' (line 87)
        [544, 560) 'c' (line 101)
        [576, 600) 'ref.tmp32' (line 101)
        [640, 664) 'ref.tmp41' (line 102)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../tests/gtest_resource_manager_static.cpp    | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/IO/Resource/tests/gtest_resource_manager_static.cpp b/src/IO/Resource/tests/gtest_resource_manager_static.cpp
index cd426b1a6bc..35bb78bbbe3 100644
--- a/src/IO/Resource/tests/gtest_resource_manager_static.cpp
+++ b/src/IO/Resource/tests/gtest_resource_manager_static.cpp
@@ -44,6 +44,15 @@ TEST(IOResourceStaticResourceManager, Smoke)
 
 TEST(IOResourceStaticResourceManager, Prioritization)
 {
+    std::optional<Int64> last_priority;
+    auto check = [&] (Int64 priority)
+    {
+        // Lock is not required here because this is called during request execution and we have max_requests = 1
+        if (last_priority)
+            EXPECT_TRUE(priority <= *last_priority); // Should be true if every queue arrived at the same time at busy period start
+        last_priority = priority;
+    };
+
     constexpr size_t threads_per_queue = 2;
     int requests_per_thead = 100;
     ResourceTest t(4 * threads_per_queue + 1);
@@ -71,15 +80,6 @@ TEST(IOResourceStaticResourceManager, Prioritization)
         </clickhouse>
     )CONFIG");
 
-    std::optional<Int64> last_priority;
-    auto check = [&] (Int64 priority)
-    {
-        // Lock is not required here because this is called during request execution and we have max_requests = 1
-        if (last_priority)
-            EXPECT_TRUE(priority <= *last_priority); // Should be true if every queue arrived at the same time at busy period start
-        last_priority = priority;
-    };
-
     for (String name : {"A", "B", "C", "D"})
     {
         for (int thr = 0; thr < threads_per_queue; thr++)

From 815cc8de9bb843d6b0d54b5507f08533f52b8190 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Tue, 9 May 2023 19:43:39 +0000
Subject: [PATCH 367/689] removed unnecessary check for thread's joinability

---
 src/Interpreters/TraceCollector.cpp | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/src/Interpreters/TraceCollector.cpp b/src/Interpreters/TraceCollector.cpp
index b8ef7044781..cb00e37df69 100644
--- a/src/Interpreters/TraceCollector.cpp
+++ b/src/Interpreters/TraceCollector.cpp
@@ -45,20 +45,15 @@ TraceCollector::~TraceCollector()
 {
     try
     {
-        if (thread.joinable())
-        {
-            /** Sends TraceCollector stop message
-            *
-            * Each sequence of data for TraceCollector thread starts with a boolean flag.
-            * If this flag is true, TraceCollector must stop reading trace_pipe and exit.
-            * This function sends flag with a true value to stop TraceCollector gracefully.
-            */
-            WriteBufferFromFileDescriptor out(TraceSender::pipe.fds_rw[1]);
-            writeChar(true, out);
-            out.next();
-        }
-        else
-            LOG_ERROR(&Poco::Logger::get("TraceCollector"), "TraceCollector thread is malformed and cannot be joined");
+        /** Sends TraceCollector stop message
+        *
+        * Each sequence of data for TraceCollector thread starts with a boolean flag.
+        * If this flag is true, TraceCollector must stop reading trace_pipe and exit.
+        * This function sends flag with a true value to stop TraceCollector gracefully.
+        */
+        WriteBufferFromFileDescriptor out(TraceSender::pipe.fds_rw[1]);
+        writeChar(true, out);
+        out.next();
     }
     catch (...)
     {
@@ -69,6 +64,8 @@ TraceCollector::~TraceCollector()
 
     if (thread.joinable())
         thread.join();
+    else
+        LOG_ERROR(&Poco::Logger::get("TraceCollector"), "TraceCollector thread is malformed and cannot be joined");
 }
 
 

From ce471a2e8bc9572c19b5fc2ab1bf92867c4ffe82 Mon Sep 17 00:00:00 2001
From: Suzy Wang <suzy.wang@ibm.com>
Date: Tue, 9 May 2023 13:31:54 -0700
Subject: [PATCH 368/689] Updated code as suggested

---
 src/Functions/reinterpretAs.cpp | 59 ++++++++++++++-------------------
 1 file changed, 24 insertions(+), 35 deletions(-)

diff --git a/src/Functions/reinterpretAs.cpp b/src/Functions/reinterpretAs.cpp
index e1d0dcd22da..93949bf50c3 100644
--- a/src/Functions/reinterpretAs.cpp
+++ b/src/Functions/reinterpretAs.cpp
@@ -176,23 +176,23 @@ public:
                     size_t size = offsets_from.size();
                     auto & vec_res = col_res->getData();
                     vec_res.resize_fill(size);
-                    size_t copy_size = 0;
-                    size_t offset_to = 0;
 
                     size_t offset = 0;
                     for (size_t i = 0; i < size; ++i)
                     {
-                        copy_size = std::min(static_cast<UInt64>(sizeof(ToFieldType)), offsets_from[i] - offset - 1);
-                        offset_to = sizeof(ToFieldType) > copy_size ? sizeof(ToFieldType) - copy_size : 0;
+                        size_t copy_size = std::min(static_cast<UInt64>(sizeof(ToFieldType)), offsets_from[i] - offset - 1);
                         if constexpr (std::endian::native == std::endian::little)
                             memcpy(&vec_res[i],
                                 &data_from[offset],
                                 copy_size);
                         else
+                        {
+                            size_t offset_to = sizeof(ToFieldType) > copy_size ? sizeof(ToFieldType) - copy_size : 0;
                             reverseMemcpy(
                                 reinterpret_cast<char*>(&vec_res[i]) + offset_to,
                                 &data_from[offset],
                                 copy_size);
+                        }
                         offset = offsets_from[i];
                     }
 
@@ -221,9 +221,14 @@ public:
 
                     for (size_t i = 0; i < size; ++i)
                     {
-                        memcpy(&vec_res[i], &data_from[offset], copy_size);
+                        if constexpr (std::endian::native == std::endian::little)
+                            memcpy(&vec_res[i], &data_from[offset], copy_size);
+                        else
+                        {
+                            size_t offset_to = sizeof(ToFieldType) > copy_size ? sizeof(ToFieldType) - copy_size : 0;
+                            memcpy(reinterpret_cast<char*>(&vec_res[i]) + offset_to, &data_from[offset], copy_size);
+                        }
                         offset += step;
-                    }
 
                     result = std::move(col_res);
 
@@ -247,18 +252,20 @@ public:
                     to.resize_fill(size);
 
                     static constexpr size_t copy_size = std::min(sizeof(From), sizeof(To));
-                    size_t offset_to = sizeof(To) > sizeof(From) ? sizeof(To) - sizeof(From):0;
 
                     for (size_t i = 0; i < size; ++i)
                     {
                         if constexpr (std::endian::native == std::endian::little)
                             memcpy(static_cast<void*>(&to[i]), static_cast<const void*>(&from[i]), copy_size);
                         else
+                        {
+                            size_t offset_to = sizeof(To) > sizeof(From) ? sizeof(To) - sizeof(From) : 0;
                             memcpy(reinterpret_cast<char*>(&to[i]) + offset_to, static_cast<const void*>(&from[i]), copy_size);
+                        }
+
                     }
 
                     result = std::move(column_to);
-
                     return true;
                 }
             }
@@ -333,20 +340,20 @@ private:
             StringRef data = src.getDataAt(i);
 
             /// Cut trailing zero bytes.
-#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-            size_t index = 0;
-            while (index < data.size && data.data[index] == 0)
-                    index++;
-            data.size -= index;
-#else
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
             while (data.size && data.data[data.size - 1] == 0)
                 --data.size;
+#else
+            size_t index = 0;
+            while (index < data.size && data.data[index] == 0)
+                index++;
+            data.size -= index;
 #endif
             data_to.resize(offset + data.size + 1);
-#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-            reverseMemcpy(&data_to[offset], data.data+index, data.size);
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+            memcpy(&data_to[offset], data.data, data.size);
 #else
-            memcpy(&data_to[offset],data.data, data.size);
+            reverseMemcpy(&data_to[offset], data.data + index, data.size);
 #endif
             offset += data.size;
             data_to[offset] = 0;
@@ -367,24 +374,6 @@ private:
         else
             return ColumnType::create(column_size);
     }
-
-    template <typename FromContainer, typename ToContainer>
-    static void reinterpretImpl(const FromContainer & from, ToContainer & to)
-    {
-        using From = typename FromContainer::value_type;
-        using To = typename ToContainer::value_type;
-
-        size_t size = from.size();
-        static constexpr size_t copy_size = std::min(sizeof(From), sizeof(To));
-
-        if (sizeof(To) <= sizeof(From))
-            to.resize(size);
-        else
-            to.resize_fill(size);
-
-        for (size_t i = 0; i < size; ++i)
-            memcpy(static_cast<void*>(&to[i]), static_cast<const void*>(&from[i]), copy_size);
-    }
 };
 
 template <typename ToDataType, typename Name>

From e35c6698924c0494f48cd9df2b35360414da7a2e Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Tue, 9 May 2023 22:51:47 +0200
Subject: [PATCH 369/689] Fix

---
 src/Interpreters/Cache/FileSegment.cpp          | 2 +-
 src/Interpreters/Cache/LRUFileCachePriority.cpp | 1 -
 src/Interpreters/Cache/Metadata.cpp             | 4 +++-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp
index 8ff6c782fe5..60228573666 100644
--- a/src/Interpreters/Cache/FileSegment.cpp
+++ b/src/Interpreters/Cache/FileSegment.cpp
@@ -753,7 +753,7 @@ bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock &) cons
     }
     else
     {
-        if (download_state == State::DOWNLOADED)
+        if (download_state == State::DOWNLOADING)
         {
             chassert(!downloader_id.empty());
         }
diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp
index 6deadb3a6ee..5e05acd9680 100644
--- a/src/Interpreters/Cache/LRUFileCachePriority.cpp
+++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp
@@ -87,7 +87,6 @@ LRUFileCachePriority::LRUQueueIterator LRUFileCachePriority::remove(LRUQueueIter
         log, "Removed entry from LRU queue, key: {}, offset: {}, size: {}",
         it->key, it->offset, it->size);
 
-    it->size = 0;
     return queue.erase(it);
 }
 
diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp
index de07bb29fec..a97b10ffbfa 100644
--- a/src/Interpreters/Cache/Metadata.cpp
+++ b/src/Interpreters/Cache/Metadata.cpp
@@ -344,7 +344,9 @@ KeyMetadata::iterator LockedKey::removeFileSegment(size_t offset, const FileSegm
 
     LOG_DEBUG(
         log, "Remove from cache. Key: {}, offset: {}, size: {}",
-        getKey(), offset, file_segment->getReservedSize());
+        getKey(), offset, file_segment->reserved_size);
+
+    chassert(file_segment->assertCorrectnessUnlocked(segment_lock));
 
     if (file_segment->queue_iterator)
         file_segment->queue_iterator->annul();

From 1b21f1360566da9ccf2d2bfad48f0b970e8f27ab Mon Sep 17 00:00:00 2001
From: Suzy Wang <suzy.wang@ibm.com>
Date: Tue, 9 May 2023 13:57:22 -0700
Subject: [PATCH 370/689] ip encoding fix

---
 src/Functions/FunctionsCodingIP.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Functions/FunctionsCodingIP.cpp b/src/Functions/FunctionsCodingIP.cpp
index 8279aa490db..2671418fc7b 100644
--- a/src/Functions/FunctionsCodingIP.cpp
+++ b/src/Functions/FunctionsCodingIP.cpp
@@ -580,7 +580,7 @@ private:
 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
             unalignedStoreLittleEndian<UInt64>(buf + 8, 0x00000000FFFF0000ull | (static_cast<UInt64>(ntohl(in)) << 32));
 #else
-            unalignedStoreLittleEndian<UInt64>(buf + 8, 0x00000000FFFF0000ull | (static_cast<UInt64>(in)) << 32));
+            unalignedStoreLittleEndian<UInt64>(buf + 8, 0x00000000FFFF0000ull | (static_cast<UInt64>(__builtin_bswap32(in))) << 32));
 #endif
     }
 };

From 99677f3d1a58f0d143c0bf079bd35bcb4b94bc69 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Wed, 10 May 2023 00:22:55 +0300
Subject: [PATCH 371/689] Update clickhouse-test (#49712)

---
 tests/clickhouse-test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index e61749e5377..e279b899a93 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -2244,7 +2244,7 @@ def main(args):
                     "\nFound hung queries in processlist:", args, "red", attrs=["bold"]
                 )
             )
-            print(processlist)
+            print(processlist.decode())
             print(get_transactions_list(args))
 
             print_stacktraces()

From 996fcfe120177ce3f05ceaef5c6fe243453542d8 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Tue, 9 May 2023 21:37:43 +0000
Subject: [PATCH 372/689] fix column ttl with sparse columns

---
 src/Storages/MergeTree/MergeTask.cpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp
index eee550f8dd6..2b7dab91934 100644
--- a/src/Storages/MergeTree/MergeTask.cpp
+++ b/src/Storages/MergeTree/MergeTask.cpp
@@ -326,6 +326,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare()
     if (!ctx->need_remove_expired_values)
     {
         size_t expired_columns = 0;
+        auto part_serialization_infos = global_ctx->new_data_part->getSerializationInfos();
 
         for (auto & [column_name, ttl] : global_ctx->new_data_part->ttl_infos.columns_ttl)
         {
@@ -335,6 +336,8 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare()
                 LOG_TRACE(ctx->log, "Adding expired column {} for part {}", column_name, global_ctx->new_data_part->name);
                 std::erase(global_ctx->gathering_column_names, column_name);
                 std::erase(global_ctx->merging_column_names, column_name);
+                std::erase(global_ctx->all_column_names, column_name);
+                part_serialization_infos.erase(column_name);
                 ++expired_columns;
             }
         }
@@ -343,6 +346,12 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare()
         {
             global_ctx->gathering_columns = global_ctx->gathering_columns.filter(global_ctx->gathering_column_names);
             global_ctx->merging_columns = global_ctx->merging_columns.filter(global_ctx->merging_column_names);
+            global_ctx->storage_columns = global_ctx->storage_columns.filter(global_ctx->all_column_names);
+
+            global_ctx->new_data_part->setColumns(
+                global_ctx->storage_columns,
+                part_serialization_infos,
+                global_ctx->metadata_snapshot->getMetadataVersion());
         }
     }
 

From 8ff5fda62816f7973466fe3a6a8923fb88ef4a1f Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Tue, 9 May 2023 23:54:13 +0000
Subject: [PATCH 373/689] fix short curcuit function with sparse arguments

---
 src/Columns/ColumnFunction.cpp                |  4 ++-
 .../02734_sparse_columns_mutation.reference   |  5 ++++
 .../02734_sparse_columns_mutation.sql         | 30 +++++++++++++++++++
 ...734_sparse_columns_short_circuit.reference |  1 +
 .../02734_sparse_columns_short_circuit.sql    | 13 ++++++++
 5 files changed, 52 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/02734_sparse_columns_mutation.reference
 create mode 100644 tests/queries/0_stateless/02734_sparse_columns_mutation.sql
 create mode 100644 tests/queries/0_stateless/02734_sparse_columns_short_circuit.reference
 create mode 100644 tests/queries/0_stateless/02734_sparse_columns_short_circuit.sql

diff --git a/src/Columns/ColumnFunction.cpp b/src/Columns/ColumnFunction.cpp
index 3ea7073d8c8..7c7b87bf4a6 100644
--- a/src/Columns/ColumnFunction.cpp
+++ b/src/Columns/ColumnFunction.cpp
@@ -264,7 +264,9 @@ void ColumnFunction::appendArgument(const ColumnWithTypeAndName & column)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot capture column {} because it has incompatible type: "
                         "got {}, but {} is expected.", argument_types.size(), column.type->getName(), argument_types[index]->getName());
 
-    captured_columns.push_back(column);
+    auto captured_column = column;
+    captured_column.column = captured_column.column->convertToFullColumnIfSparse();
+    captured_columns.push_back(std::move(captured_column));
 }
 
 DataTypePtr ColumnFunction::getResultType() const
diff --git a/tests/queries/0_stateless/02734_sparse_columns_mutation.reference b/tests/queries/0_stateless/02734_sparse_columns_mutation.reference
new file mode 100644
index 00000000000..bdce3124792
--- /dev/null
+++ b/tests/queries/0_stateless/02734_sparse_columns_mutation.reference
@@ -0,0 +1,5 @@
+5000	1189524
+1
+3333	0
+0
+3333	0
diff --git a/tests/queries/0_stateless/02734_sparse_columns_mutation.sql b/tests/queries/0_stateless/02734_sparse_columns_mutation.sql
new file mode 100644
index 00000000000..6fdb5b5f4a7
--- /dev/null
+++ b/tests/queries/0_stateless/02734_sparse_columns_mutation.sql
@@ -0,0 +1,30 @@
+DROP TABLE IF EXISTS t_sparse_mutation;
+
+CREATE TABLE t_sparse_mutation (id UInt64, v UInt64)
+ENGINE = MergeTree ORDER BY id
+SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9;
+
+INSERT INTO t_sparse_mutation select number, if (number % 21 = 0, number, 0) FROM numbers(10000);
+
+SET mutations_sync = 2;
+
+DELETE FROM t_sparse_mutation WHERE id % 2 = 0;
+
+SELECT count(), sum(v) FROM t_sparse_mutation;
+
+SELECT sum(has_lightweight_delete) FROM system.parts
+WHERE database = currentDatabase() AND table = 't_sparse_mutation' AND active;
+
+ALTER TABLE t_sparse_mutation UPDATE v = v * 2 WHERE id % 5 = 0;
+ALTER TABLE t_sparse_mutation DELETE WHERE id % 3 = 0;
+
+SELECT count(), sum(v) FROM t_sparse_mutation;
+
+OPTIMIZE TABLE t_sparse_mutation FINAL;
+
+SELECT sum(has_lightweight_delete) FROM system.parts
+WHERE database = currentDatabase() AND table = 't_sparse_mutation' AND active;
+
+SELECT count(), sum(v) FROM t_sparse_mutation;
+
+DROP TABLE t_sparse_mutation;
diff --git a/tests/queries/0_stateless/02734_sparse_columns_short_circuit.reference b/tests/queries/0_stateless/02734_sparse_columns_short_circuit.reference
new file mode 100644
index 00000000000..bf2c10d23ec
--- /dev/null
+++ b/tests/queries/0_stateless/02734_sparse_columns_short_circuit.reference
@@ -0,0 +1 @@
+477
diff --git a/tests/queries/0_stateless/02734_sparse_columns_short_circuit.sql b/tests/queries/0_stateless/02734_sparse_columns_short_circuit.sql
new file mode 100644
index 00000000000..da8de22a80e
--- /dev/null
+++ b/tests/queries/0_stateless/02734_sparse_columns_short_circuit.sql
@@ -0,0 +1,13 @@
+DROP TABLE IF EXISTS t_sparse_short_circuit;
+
+SET short_circuit_function_evaluation = 'force_enable';
+
+CREATE TABLE t_sparse_short_circuit (a UInt64, b UInt64)
+ENGINE = MergeTree ORDER BY tuple()
+SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9;
+
+INSERT INTO t_sparse_short_circuit select number, if (number % 21 = 0, number % 10 + 1, 0) FROM numbers(100000);
+
+SELECT sum(if(a % 10 = 0, CAST(b, 'UInt8'), 0)) FROM t_sparse_short_circuit;
+
+DROP TABLE t_sparse_short_circuit;

From a2c4b8e23d7a6f7706a1707a42cd789a07efc552 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 10 May 2023 03:16:52 +0200
Subject: [PATCH 374/689] Disable mmap for server

---
 src/Core/Settings.h          |  2 +-
 src/Storages/StorageFile.cpp | 14 +++++++++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 1bea2c26392..642862ec2c3 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -641,7 +641,7 @@ class IColumn;
     M(UInt64, function_range_max_elements_in_block, 500000000, "Maximum number of values generated by function 'range' per block of data (sum of array sizes for every row in a block, see also 'max_block_size' and 'min_insert_block_size_rows'). It is a safety threshold.", 0) \
     M(ShortCircuitFunctionEvaluation, short_circuit_function_evaluation, ShortCircuitFunctionEvaluation::ENABLE, "Setting for short-circuit function evaluation configuration. Possible values: 'enable' - use short-circuit function evaluation for functions that are suitable for it, 'disable' - disable short-circuit function evaluation, 'force_enable' - use short-circuit function evaluation for all functions.", 0) \
     \
-    M(LocalFSReadMethod, storage_file_read_method, LocalFSReadMethod::mmap, "Method of reading data from storage file, one of: read, pread, mmap.", 0) \
+    M(LocalFSReadMethod, storage_file_read_method, LocalFSReadMethod::mmap, "Method of reading data from storage file, one of: read, pread, mmap. The mmap method only applies to clickhouse-local.", 0) \
     M(String, local_filesystem_read_method, "pread_threadpool", "Method of reading data from local filesystem, one of: read, pread, mmap, io_uring, pread_threadpool. The 'io_uring' method is experimental and does not work for Log, TinyLog, StripeLog, File, Set and Join, and other tables with append-able files in presence of concurrent reads and writes.", 0) \
     M(String, remote_filesystem_read_method, "threadpool", "Method of reading data from remote filesystem, one of: read, threadpool.", 0) \
     M(Bool, local_filesystem_read_prefetch, false, "Should use prefetching when reading data from local filesystem.", 0) \
diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index 26d4aee8cfe..425fe6bee31 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -201,7 +201,19 @@ std::unique_ptr<ReadBuffer> selectReadBuffer(
 {
     auto read_method = context->getSettingsRef().storage_file_read_method;
 
-    if (S_ISREG(file_stat.st_mode) && read_method == LocalFSReadMethod::mmap)
+    /** But using mmap on server-side is unsafe for the following reasons:
+      * - concurrent modifications of a file will result in SIGBUS;
+      * - IO error from the device will result in SIGBUS;
+      * - recovery from this signal is not feasible even with the usage of siglongjmp,
+      *   as it might require stack unwinding from arbitrary place;
+      * - arbitrary slowdown due to page fault in arbitrary place in the code is difficult to debug.
+      *
+      * But we keep this mode for clickhouse-local as it is not so bad for a command line tool.
+      */
+
+    if (S_ISREG(file_stat.st_mode)
+        && context->getApplicationType() != Context::ApplicationType::SERVER
+        && read_method == LocalFSReadMethod::mmap)
     {
         try
         {

From 8a23d8b38925a6b29bedcb60f142871cb3e09a1f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 10 May 2023 03:17:57 +0200
Subject: [PATCH 375/689] Disable mmap for server

---
 src/Core/Settings.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 642862ec2c3..99909434194 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -641,7 +641,7 @@ class IColumn;
     M(UInt64, function_range_max_elements_in_block, 500000000, "Maximum number of values generated by function 'range' per block of data (sum of array sizes for every row in a block, see also 'max_block_size' and 'min_insert_block_size_rows'). It is a safety threshold.", 0) \
     M(ShortCircuitFunctionEvaluation, short_circuit_function_evaluation, ShortCircuitFunctionEvaluation::ENABLE, "Setting for short-circuit function evaluation configuration. Possible values: 'enable' - use short-circuit function evaluation for functions that are suitable for it, 'disable' - disable short-circuit function evaluation, 'force_enable' - use short-circuit function evaluation for all functions.", 0) \
     \
-    M(LocalFSReadMethod, storage_file_read_method, LocalFSReadMethod::mmap, "Method of reading data from storage file, one of: read, pread, mmap. The mmap method only applies to clickhouse-local.", 0) \
+    M(LocalFSReadMethod, storage_file_read_method, LocalFSReadMethod::mmap, "Method of reading data from storage file, one of: read, pread, mmap. The mmap method does not apply to clickhouse-server (it's intended for clickhouse-local).", 0) \
     M(String, local_filesystem_read_method, "pread_threadpool", "Method of reading data from local filesystem, one of: read, pread, mmap, io_uring, pread_threadpool. The 'io_uring' method is experimental and does not work for Log, TinyLog, StripeLog, File, Set and Join, and other tables with append-able files in presence of concurrent reads and writes.", 0) \
     M(String, remote_filesystem_read_method, "threadpool", "Method of reading data from remote filesystem, one of: read, threadpool.", 0) \
     M(Bool, local_filesystem_read_prefetch, false, "Should use prefetching when reading data from local filesystem.", 0) \

From 56ea17c922c008fc3804770a19688e153daa8c7f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 10 May 2023 03:21:36 +0200
Subject: [PATCH 376/689] Fix typo

---
 src/IO/S3/copyS3File.cpp     |  6 +++---
 src/IO/WriteBufferFromS3.cpp | 14 +++++++-------
 src/IO/WriteBufferFromS3.h   |  6 +++---
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp
index b8b3cbbc54e..3d99a584933 100644
--- a/src/IO/S3/copyS3File.cpp
+++ b/src/IO/S3/copyS3File.cpp
@@ -239,11 +239,11 @@ namespace
                 // Multipart upload failed because it wasn't possible to schedule all the tasks.
                 // To avoid execution of already scheduled tasks we abort MultipartUpload.
                 abortMultipartUpload();
-                waitForAllBackGroundTasks();
+                waitForAllBackgroundTasks();
                 throw;
             }
 
-            waitForAllBackGroundTasks();
+            waitForAllBackgroundTasks();
             completeMultipartUpload();
         }
 
@@ -381,7 +381,7 @@ namespace
         virtual std::unique_ptr<Aws::AmazonWebServiceRequest> fillUploadPartRequest(size_t part_number, size_t part_offset, size_t part_size) = 0;
         virtual String processUploadPartRequest(Aws::AmazonWebServiceRequest & request) = 0;
 
-        void waitForAllBackGroundTasks()
+        void waitForAllBackgroundTasks()
         {
             if (!schedule)
                 return;
diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp
index dde19c37e29..5a25cb89107 100644
--- a/src/IO/WriteBufferFromS3.cpp
+++ b/src/IO/WriteBufferFromS3.cpp
@@ -109,7 +109,7 @@ void WriteBufferFromS3::nextImpl()
     else
         processWithDynamicParts();
 
-    waitForReadyBackGroundTasks();
+    waitForReadyBackgroundTasks();
 }
 
 void WriteBufferFromS3::processWithStrictParts()
@@ -225,7 +225,7 @@ void WriteBufferFromS3::finalizeImpl()
     if (!is_prefinalized)
         preFinalize();
 
-    waitForAllBackGroundTasks();
+    waitForAllBackgroundTasks();
 
     if (!multipart_upload_id.empty())
         completeMultipartUpload();
@@ -632,7 +632,7 @@ void WriteBufferFromS3::processPutRequest(const PutObjectTask & task)
         max_retry, key, bucket);
 }
 
-void WriteBufferFromS3::waitForReadyBackGroundTasks()
+void WriteBufferFromS3::waitForReadyBackgroundTasks()
 {
     if (schedule)
     {
@@ -650,7 +650,7 @@ void WriteBufferFromS3::waitForReadyBackGroundTasks()
 
             if (exception)
             {
-                waitForAllBackGroundTasksUnlocked(lock);
+                waitForAllBackgroundTasksUnlocked(lock);
                 std::rethrow_exception(exception);
             }
 
@@ -659,16 +659,16 @@ void WriteBufferFromS3::waitForReadyBackGroundTasks()
     }
 }
 
-void WriteBufferFromS3::waitForAllBackGroundTasks()
+void WriteBufferFromS3::waitForAllBackgroundTasks()
 {
     if (schedule)
     {
         std::unique_lock lock(bg_tasks_mutex);
-        waitForAllBackGroundTasksUnlocked(lock);
+        waitForAllBackgroundTasksUnlocked(lock);
     }
 }
 
-void WriteBufferFromS3::waitForAllBackGroundTasksUnlocked(std::unique_lock<std::mutex> & bg_tasks_lock)
+void WriteBufferFromS3::waitForAllBackgroundTasksUnlocked(std::unique_lock<std::mutex> & bg_tasks_lock)
 {
     if (schedule)
     {
diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h
index d3c7cdf2407..2374f1502f5 100644
--- a/src/IO/WriteBufferFromS3.h
+++ b/src/IO/WriteBufferFromS3.h
@@ -79,9 +79,9 @@ private:
     void fillPutRequest(S3::PutObjectRequest & req);
     void processPutRequest(const PutObjectTask & task);
 
-    void waitForReadyBackGroundTasks();
-    void waitForAllBackGroundTasks();
-    void waitForAllBackGroundTasksUnlocked(std::unique_lock<std::mutex> & bg_tasks_lock);
+    void waitForReadyBackgroundTasks();
+    void waitForAllBackgroundTasks();
+    void waitForAllBackgroundTasksUnlocked(std::unique_lock<std::mutex> & bg_tasks_lock);
 
     const String bucket;
     const String key;

From 8a6e07f0eafd9ef86a931abf0a283b08b6c02f14 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 10 May 2023 03:35:13 +0200
Subject: [PATCH 377/689] Make projections production-ready

---
 docs/en/operations/settings/settings.md       |  4 +-
 .../mergetree-family/mergetree.md             |  2 +-
 docs/ru/operations/settings/settings.md       |  4 +-
 .../sparse-primary-indexes.md                 |  2 +-
 src/Core/Settings.h                           | 33 +++++-----
 .../QueryPlanOptimizationSettings.cpp         |  2 +-
 .../QueryPlan/Optimizations/optimizeTree.cpp  |  2 +-
 src/Storages/MergeTree/MergeTreeData.cpp      |  4 +-
 .../MergeTree/MergeTreeDataSelectExecutor.cpp |  4 +-
 tests/performance/group_by_sundy_li.xml       |  2 +-
 tests/performance/groupby_onekey_nullable.xml |  4 +-
 .../01576_alias_column_rewrite.sql            |  2 +-
 ...1710_aggregate_projection_with_hashing.sql |  2 +-
 .../01710_aggregate_projections.sh            | 24 +++----
 .../01710_force_use_projection.sql            |  2 +-
 .../01710_minmax_count_projection.sql         |  2 +-
 .../01710_normal_projection_fix1.sql          |  2 +-
 .../01710_normal_projections.reference        | 32 +++++-----
 .../0_stateless/01710_normal_projections.sh   | 64 +++++++++----------
 .../01710_projection_aggregation_in_order.sql |  4 +-
 .../01710_projection_array_join.sql           |  2 +-
 .../01710_projection_detach_part.sql          |  2 +-
 .../0_stateless/01710_projection_in_index.sql |  2 +-
 .../0_stateless/01710_projection_in_set.sql   |  2 +-
 .../0_stateless/01710_projection_mutation.sql |  2 +-
 .../01710_projection_vertical_merges.sql      |  2 +-
 .../01710_projection_with_joins.sql           | 12 ++--
 .../01710_projection_with_mixed_pipeline.sql  |  2 +-
 .../queries/0_stateless/01710_projections.sql |  4 +-
 .../01710_projections_group_by_no_key.sql     |  4 +-
 ...01710_projections_in_distributed_query.sql |  2 +-
 ...ojections_optimize_aggregation_in_order.sh |  2 +-
 ...s_partial_optimize_aggregation_in_order.sh |  2 +-
 ..._projection_and_mutation_work_together.sql |  4 +-
 .../02521_aggregation_by_partitions.sql       |  2 +-
 .../0_stateless/02524_fuzz_and_fuss_2.sql     |  2 +-
 36 files changed, 123 insertions(+), 122 deletions(-)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 0a4b9a6bd9d..ad7ef52412a 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -3562,7 +3562,7 @@ Default value: `1`.
 
 If the setting is set to `0`, the table function does not make Nullable columns and inserts default values instead of NULL. This is also applicable for NULL values inside arrays.
 
-## allow_experimental_projection_optimization {#allow-experimental-projection-optimization}
+## optimize_use_projections {#optimize_use_projections}
 
 Enables or disables [projection](../../engines/table-engines/mergetree-family/mergetree.md/#projections) optimization when processing `SELECT` queries.
 
@@ -3575,7 +3575,7 @@ Default value: `1`.
 
 ## force_optimize_projection {#force-optimize-projection}
 
-Enables or disables the obligatory use of [projections](../../engines/table-engines/mergetree-family/mergetree.md/#projections) in `SELECT` queries, when projection optimization is enabled (see [allow_experimental_projection_optimization](#allow-experimental-projection-optimization) setting).
+Enables or disables the obligatory use of [projections](../../engines/table-engines/mergetree-family/mergetree.md/#projections) in `SELECT` queries, when projection optimization is enabled (see [optimize_use_projections](#optimize_use_projections) setting).
 
 Possible values:
 
diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md
index 6182ab20203..812b0c0a2d4 100644
--- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md
+++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md
@@ -391,7 +391,7 @@ INDEX b (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100) GRANULARIT
 ## Проекции {#projections}
 Проекции похожи на [материализованные представления](../../../sql-reference/statements/create/view.md#materialized), но определяются на уровне кусков данных. Это обеспечивает гарантии согласованности данных наряду с автоматическим использованием в запросах.
 
-Проекции — это экспериментальная возможность. Чтобы включить поддержку проекций, установите настройку [allow_experimental_projection_optimization](../../../operations/settings/settings.md#allow-experimental-projection-optimization) в значение `1`. См. также настройку [force_optimize_projection ](../../../operations/settings/settings.md#force-optimize-projection).
+Проекции — это экспериментальная возможность. Чтобы включить поддержку проекций, установите настройку [optimize_use_projections](../../../operations/settings/settings.md#allow-experimental-projection-optimization) в значение `1`. См. также настройку [force_optimize_projection ](../../../operations/settings/settings.md#optimize_use_projections).
 
 Проекции не поддерживаются для запросов `SELECT` с модификатором [FINAL](../../../sql-reference/statements/select/from.md#select-from-final).
 
diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md
index 7ecc8b7cb42..066e71c25a5 100644
--- a/docs/ru/operations/settings/settings.md
+++ b/docs/ru/operations/settings/settings.md
@@ -3588,7 +3588,7 @@ SETTINGS index_granularity = 8192 │
 
 Строка с идентификатором снэпшота, из которого будет выполняться [исходный дамп таблиц PostgreSQL](../../engines/database-engines/materialized-postgresql.md). Эта настройка должна использоваться совместно с [materialized_postgresql_replication_slot](#materialized-postgresql-replication-slot).
 
-## allow_experimental_projection_optimization {#allow-experimental-projection-optimization}
+## optimize_use_projections {#optimize_use_projections}
 
 Включает или отключает поддержку [проекций](../../engines/table-engines/mergetree-family/mergetree.md#projections) при обработке запросов `SELECT`.
 
@@ -3601,7 +3601,7 @@ SETTINGS index_granularity = 8192 │
 
 ## force_optimize_projection {#force-optimize-projection}
 
-Включает или отключает обязательное использование [проекций](../../engines/table-engines/mergetree-family/mergetree.md#projections) в запросах `SELECT`, если поддержка проекций включена (см. настройку [allow_experimental_projection_optimization](#allow-experimental-projection-optimization)).
+Включает или отключает обязательное использование [проекций](../../engines/table-engines/mergetree-family/mergetree.md#projections) в запросах `SELECT`, если поддержка проекций включена (см. настройку [optimize_use_projections](#optimize_use_projections)).
 
 Возможные значения:
 
diff --git a/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md b/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md
index 51167521018..eedc913cf82 100644
--- a/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md
+++ b/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md
@@ -1074,7 +1074,7 @@ ClickHouse服务器日志文件中相应的跟踪日志确认了ClickHouse正在
 <a href="https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/#projections" target="_blank">Projections</a>目前是一个实验性的功能，因此我们需要告诉ClickHouse：
 
 ```sql
-SET allow_experimental_projection_optimization = 1;
+SET optimize_use_projections = 1;
 ```
 
 
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 1bea2c26392..f60632fae91 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -560,6 +560,7 @@ class IColumn;
     M(Bool, asterisk_include_alias_columns, false, "Include ALIAS columns for wildcard query", 0) \
     M(Bool, optimize_skip_merged_partitions, false, "Skip partitions with one part with level > 0 in optimize final", 0) \
     M(Bool, optimize_on_insert, true, "Do the same transformation for inserted block of data as if merge was done on this block.", 0) \
+    M(Bool, optimize_use_projections, true, "Automatically choose projections to perform SELECT query", 0) \
     M(Bool, force_optimize_projection, false, "If projection optimization is enabled, SELECT queries need to use projection", 0) \
     M(Bool, async_socket_for_remote, true, "Asynchronously read from socket executing remote query", 0) \
     M(Bool, async_query_sending_for_remote, true, "Asynchronously create connections and send query to shards in remote query", 0) \
@@ -715,26 +716,12 @@ class IColumn;
     M(Milliseconds, storage_system_stack_trace_pipe_read_timeout_ms, 100, "Maximum time to read from a pipe for receiving information from the threads when querying the `system.stack_trace` table. This setting is used for testing purposes and not meant to be changed by users.", 0) \
     \
     M(Bool, parallelize_output_from_storages, true, "Parallelize output for reading step from storage. It allows parallelizing query processing right after reading from storage if possible", 0) \
-    \
-    /** Experimental functions */ \
-    M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \
-    M(Bool, allow_experimental_nlp_functions, false, "Enable experimental functions for natural language processing.", 0) \
-    M(Bool, allow_experimental_hash_functions, false, "Enable experimental hash functions (hashid, etc)", 0) \
-    M(Bool, allow_experimental_object_type, false, "Allow Object and JSON data types", 0) \
     M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \
     M(String, ann_index_select_query_params, "", "Parameters passed to ANN indexes in SELECT queries, the format is 'param1=x, param2=y, ...'", 0) \
-    M(UInt64, max_limit_for_ann_queries, 1000000, "Maximum limit value for using ANN indexes is used to prevent memory overflow in search queries for indexes", 0) \
-    M(Bool, allow_experimental_annoy_index, false, "Allows to use Annoy index. Disabled by default because this feature is experimental", 0) \
     M(Bool, count_distinct_optimization, false, "Rewrite count distinct to subquery of group by", 0) \
-    M(Bool, throw_on_unsupported_query_inside_transaction, true, "Throw exception if unsupported query is used inside transaction", 0) \
-    M(TransactionsWaitCSNMode, wait_changes_become_visible_after_commit_mode, TransactionsWaitCSNMode::WAIT_UNKNOWN, "Wait for committed changes to become actually visible in the latest snapshot", 0) \
-    M(Bool, implicit_transaction, false, "If enabled and not already inside a transaction, wraps the query inside a full transaction (begin + commit or rollback)", 0) \
     M(Bool, throw_if_no_data_to_insert, true, "Enables or disables empty INSERTs, enabled by default", 0) \
     M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \
     M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \
-    M(UInt64, grace_hash_join_initial_buckets, 1, "Initial number of grace hash join buckets", 0) \
-    M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \
-    M(Bool, optimize_distinct_in_order, false, "This optimization has a bug and it is disabled. Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
     M(Bool, optimize_sorting_by_input_stream_properties, true, "Optimize sorting by sorting properties of input stream", 0) \
     M(UInt64, insert_keeper_max_retries, 20, "Max retries for keeper operations during insert", 0) \
     M(UInt64, insert_keeper_retry_initial_backoff_ms, 100, "Initial backoff timeout for keeper operations during insert", 0) \
@@ -743,10 +730,24 @@ class IColumn;
     M(UInt64, insert_keeper_fault_injection_seed, 0, "0 - random seed, otherwise the setting value", 0) \
     M(Bool, force_aggregation_in_order, false, "Force use of aggregation in order on remote nodes during distributed aggregation. PLEASE, NEVER CHANGE THIS SETTING VALUE MANUALLY!", IMPORTANT) \
     M(UInt64, http_max_request_param_data_size, 10_MiB, "Limit on size of request data used as a query parameter in predefined HTTP requests.", 0) \
+    M(Bool, function_json_value_return_type_allow_nullable, false, "Allow function JSON_VALUE to return nullable type.", 0) \
+    M(Bool, function_json_value_return_type_allow_complex, false, "Allow function JSON_VALUE to return complex type, such as: struct, array, map.", 0) \
+    \
+    /** Experimental functions */ \
+    M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \
+    M(Bool, allow_experimental_nlp_functions, false, "Enable experimental functions for natural language processing.", 0) \
+    M(Bool, allow_experimental_hash_functions, false, "Enable experimental hash functions (hashid, etc)", 0) \
+    M(Bool, allow_experimental_object_type, false, "Allow Object and JSON data types", 0) \
+    M(Bool, allow_experimental_annoy_index, false, "Allows to use Annoy index. Disabled by default because this feature is experimental", 0) \
+    M(UInt64, max_limit_for_ann_queries, 1000000, "Maximum limit value for using ANN indexes is used to prevent memory overflow in search queries for indexes", 0) \
+    M(Bool, throw_on_unsupported_query_inside_transaction, true, "Throw exception if unsupported query is used inside transaction", 0) \
+    M(TransactionsWaitCSNMode, wait_changes_become_visible_after_commit_mode, TransactionsWaitCSNMode::WAIT_UNKNOWN, "Wait for committed changes to become actually visible in the latest snapshot", 0) \
+    M(Bool, implicit_transaction, false, "If enabled and not already inside a transaction, wraps the query inside a full transaction (begin + commit or rollback)", 0) \
+    M(UInt64, grace_hash_join_initial_buckets, 1, "Initial number of grace hash join buckets", 0) \
+    M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \
+    M(Bool, optimize_distinct_in_order, false, "This optimization has a bug and it is disabled. Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
     M(Bool, allow_experimental_undrop_table_query, false, "Allow to use undrop query to restore dropped table in a limited time", 0) \
     M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \
-    M(Bool, function_json_value_return_type_allow_nullable, false, "Allow function to return nullable type.", 0) \
-    M(Bool, function_json_value_return_type_allow_complex, false, "Allow function to return complex type, such as: struct, array, map.", 0) \
     // End of COMMON_SETTINGS
     // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS.
 
diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp
index 7db1ba1db71..21c7d362b17 100644
--- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp
+++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp
@@ -17,7 +17,7 @@ QueryPlanOptimizationSettings QueryPlanOptimizationSettings::fromSettings(const
     settings.remove_redundant_sorting = from.query_plan_remove_redundant_sorting;
     settings.aggregate_partitions_independently = from.allow_aggregate_partitions_independently;
     settings.remove_redundant_distinct = from.query_plan_remove_redundant_distinct;
-    settings.optimize_projection = from.allow_experimental_projection_optimization && from.query_plan_optimize_projection;
+    settings.optimize_projection = from.optimize_use_projections && from.query_plan_optimize_projection;
     settings.force_use_projection = settings.optimize_projection && from.force_optimize_projection;
     return settings;
 }
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp
index 37e3b2f67d8..dd17c1b5a30 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp
@@ -170,7 +170,7 @@ void optimizeTreeSecondPass(const QueryPlanOptimizationSettings & optimization_s
     if (optimization_settings.force_use_projection && has_reading_from_mt && num_applied_projection == 0)
         throw Exception(
             ErrorCodes::PROJECTION_NOT_USED,
-            "No projection is used when allow_experimental_projection_optimization = 1 and force_optimize_projection = 1");
+            "No projection is used when optimize_use_projections = 1 and force_optimize_projection = 1");
 }
 
 }
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index a2a3348eb42..827f453c32c 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -6242,7 +6242,7 @@ bool MergeTreeData::mayBenefitFromIndexForIn(
                     return true;
         }
 
-        if (query_settings.allow_experimental_projection_optimization)
+        if (query_settings.optimize_use_projections)
         {
             for (const auto & projection : metadata_snapshot->getProjections())
                 if (projection.isPrimaryKeyColumnPossiblyWrappedInFunctions(ast))
@@ -6613,7 +6613,7 @@ std::optional<ProjectionCandidate> MergeTreeData::getQueryProcessingStageWithAgg
     if (!query_info.syntax_analyzer_result)
         return std::nullopt;
 
-    if (!settings.allow_experimental_projection_optimization || query_info.ignore_projections || query_info.is_projection_query
+    if (!settings.optimize_use_projections || query_info.ignore_projections || query_info.is_projection_query
         || settings.aggregate_functions_null_for_empty /* projections don't work correctly with this setting */)
         return std::nullopt;
 
diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
index 22df8f298c4..dea22813bd1 100644
--- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
@@ -175,10 +175,10 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read(
             query_info.merge_tree_select_result_ptr,
             enable_parallel_reading);
 
-        if (!step && settings.allow_experimental_projection_optimization && settings.force_optimize_projection
+        if (!step && settings.optimize_use_projections && settings.force_optimize_projection
             && !metadata_for_reading->projections.empty() && !settings.query_plan_optimize_projection)
             throw Exception(ErrorCodes::PROJECTION_NOT_USED,
-                            "No projection is used when allow_experimental_projection_optimization = 1 and force_optimize_projection = 1");
+                            "No projection is used when optimize_use_projections = 1 and force_optimize_projection = 1");
 
         auto plan = std::make_unique<QueryPlan>();
         if (step)
diff --git a/tests/performance/group_by_sundy_li.xml b/tests/performance/group_by_sundy_li.xml
index 07c5aa5cd7f..694fafcbbcd 100644
--- a/tests/performance/group_by_sundy_li.xml
+++ b/tests/performance/group_by_sundy_li.xml
@@ -1,7 +1,7 @@
 <test>
     <settings>
         <max_insert_threads>8</max_insert_threads>
-        <allow_experimental_projection_optimization>0</allow_experimental_projection_optimization>
+        <optimize_use_projections>0</optimize_use_projections>
     </settings>
 
     <create_query>
diff --git a/tests/performance/groupby_onekey_nullable.xml b/tests/performance/groupby_onekey_nullable.xml
index 2c03cb0861f..9072538039b 100644
--- a/tests/performance/groupby_onekey_nullable.xml
+++ b/tests/performance/groupby_onekey_nullable.xml
@@ -1,7 +1,7 @@
 <test>
     <settings>
         <max_insert_threads>8</max_insert_threads>
-        <allow_experimental_projection_optimization>0</allow_experimental_projection_optimization>
+        <optimize_use_projections>0</optimize_use_projections>
     </settings>
 
     <substitutions>
@@ -46,4 +46,4 @@
     <query>select min(i1), min(i2) from t_nullable group by {key_type} format Null</query>
 
     <drop_query>drop table if exists t_nullable</drop_query>
-</test>
\ No newline at end of file
+</test>
diff --git a/tests/queries/0_stateless/01576_alias_column_rewrite.sql b/tests/queries/0_stateless/01576_alias_column_rewrite.sql
index 1f28225bef8..d59889c1af8 100644
--- a/tests/queries/0_stateless/01576_alias_column_rewrite.sql
+++ b/tests/queries/0_stateless/01576_alias_column_rewrite.sql
@@ -122,7 +122,7 @@ create table pl (dt DateTime, i int, projection p (select sum(i) group by toStar
 insert into pl values ('2020-10-24', 1);
 
 set max_rows_to_read = 2;
-select sum(i) from pd group by dt_m settings allow_experimental_projection_optimization = 1, force_optimize_projection = 1;
+select sum(i) from pd group by dt_m settings optimize_use_projections = 1, force_optimize_projection = 1;
 
 drop table pd;
 drop table pl;
diff --git a/tests/queries/0_stateless/01710_aggregate_projection_with_hashing.sql b/tests/queries/0_stateless/01710_aggregate_projection_with_hashing.sql
index d5eaa2617a6..0429865e7b1 100644
--- a/tests/queries/0_stateless/01710_aggregate_projection_with_hashing.sql
+++ b/tests/queries/0_stateless/01710_aggregate_projection_with_hashing.sql
@@ -1,4 +1,4 @@
-set allow_experimental_projection_optimization = 1, force_optimize_projection = 1;
+set optimize_use_projections = 1, force_optimize_projection = 1;
 
 drop table if exists tp;
 
diff --git a/tests/queries/0_stateless/01710_aggregate_projections.sh b/tests/queries/0_stateless/01710_aggregate_projections.sh
index 561b8927579..326a564a208 100755
--- a/tests/queries/0_stateless/01710_aggregate_projections.sh
+++ b/tests/queries/0_stateless/01710_aggregate_projections.sh
@@ -8,22 +8,22 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 $CLICKHOUSE_CLIENT -q "CREATE TABLE test_agg_proj (x Int32, y Int32, PROJECTION x_plus_y (SELECT sum(x - y), argMax(x, y) group by x + y)) ENGINE = MergeTree ORDER BY tuple() settings index_granularity = 1"
 $CLICKHOUSE_CLIENT -q "insert into test_agg_proj select intDiv(number, 2), -intDiv(number,3) - 1 from numbers(100)"
 
-$CLICKHOUSE_CLIENT -q "select x + y, sum(x - y) as s from test_agg_proj group by x + y order by s desc limit 5 settings allow_experimental_projection_optimization=1"
-$CLICKHOUSE_CLIENT -q "select x + y, sum(x - y) as s from test_agg_proj group by x + y order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT -q "select x + y, sum(x - y) as s from test_agg_proj group by x + y order by s desc limit 5 settings optimize_use_projections=1"
+$CLICKHOUSE_CLIENT -q "select x + y, sum(x - y) as s from test_agg_proj group by x + y order by s desc limit 5 settings optimize_use_projections=1 format JSON" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT -q "select (x + y) * 2, sum(x - y) * 2 as s from test_agg_proj group by x + y order by s desc limit 5 settings allow_experimental_projection_optimization=1"
-$CLICKHOUSE_CLIENT -q "select (x + y) * 2, sum(x - y) * 2 as s from test_agg_proj group by x + y order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT -q "select (x + y) * 2, sum(x - y) * 2 as s from test_agg_proj group by x + y order by s desc limit 5 settings optimize_use_projections=1"
+$CLICKHOUSE_CLIENT -q "select (x + y) * 2, sum(x - y) * 2 as s from test_agg_proj group by x + y order by s desc limit 5 settings optimize_use_projections=1 format JSON" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2) as v, intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc, v limit 5 settings allow_experimental_projection_optimization=1"
-$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2) as v, intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc, v limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2) as v, intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc, v limit 5 settings optimize_use_projections=1"
+$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2) as v, intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc, v limit 5 settings optimize_use_projections=1 format JSON" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(x, y) * sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings allow_experimental_projection_optimization=1"
-$CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(x, y) * sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(x, y) * sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings optimize_use_projections=1"
+$CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(x, y) * sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings optimize_use_projections=1 format JSON" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(y, x), sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings allow_experimental_projection_optimization=1"
-$CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(y, x), sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(y, x), sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings optimize_use_projections=1"
+$CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(y, x), sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings optimize_use_projections=1 format JSON" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT -q "select x + y, sum(x - y) as s from test_agg_proj prewhere (x + y) % 2 = 1 group by x + y order by s desc limit 5 settings allow_experimental_projection_optimization=1"
-$CLICKHOUSE_CLIENT -q "select x + y, sum(x - y) as s from test_agg_proj prewhere (x + y) % 2 = 1 group by x + y order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT -q "select x + y, sum(x - y) as s from test_agg_proj prewhere (x + y) % 2 = 1 group by x + y order by s desc limit 5 settings optimize_use_projections=1"
+$CLICKHOUSE_CLIENT -q "select x + y, sum(x - y) as s from test_agg_proj prewhere (x + y) % 2 = 1 group by x + y order by s desc limit 5 settings optimize_use_projections=1 format JSON" | grep "rows_read"
 
 $CLICKHOUSE_CLIENT -q "drop table test_agg_proj"
diff --git a/tests/queries/0_stateless/01710_force_use_projection.sql b/tests/queries/0_stateless/01710_force_use_projection.sql
index 8931c65e34e..af6ca69c540 100644
--- a/tests/queries/0_stateless/01710_force_use_projection.sql
+++ b/tests/queries/0_stateless/01710_force_use_projection.sql
@@ -2,7 +2,7 @@ drop table if exists tp;
 
 create table tp (d1 Int32, d2 Int32, eventcnt Int64, projection p (select sum(eventcnt) group by d1)) engine = MergeTree order by (d1, d2);
 
-set allow_experimental_projection_optimization = 1, force_optimize_projection = 1;
+set optimize_use_projections = 1, force_optimize_projection = 1;
 
 select sum(eventcnt) eventcnt, d1 from tp group by d1;
 
diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.sql b/tests/queries/0_stateless/01710_minmax_count_projection.sql
index f7645414187..c17f0e1e1fb 100644
--- a/tests/queries/0_stateless/01710_minmax_count_projection.sql
+++ b/tests/queries/0_stateless/01710_minmax_count_projection.sql
@@ -4,7 +4,7 @@ create table d (i int, j int) engine MergeTree partition by i % 2 order by tuple
 
 insert into d select number, number from numbers(10000);
 
-set max_rows_to_read = 2, allow_experimental_projection_optimization = 1;
+set max_rows_to_read = 2, optimize_use_projections = 1;
 
 select min(i), max(i), count() from d;
 select min(i), max(i), count() from d group by _partition_id order by _partition_id;
diff --git a/tests/queries/0_stateless/01710_normal_projection_fix1.sql b/tests/queries/0_stateless/01710_normal_projection_fix1.sql
index e8ba830e5a5..cce8309046a 100644
--- a/tests/queries/0_stateless/01710_normal_projection_fix1.sql
+++ b/tests/queries/0_stateless/01710_normal_projection_fix1.sql
@@ -9,7 +9,7 @@ alter table t add projection x (select * order by j);
 insert into t values (1, 4);
 insert into t values (1, 5);
 
-set allow_experimental_projection_optimization = 1, force_optimize_projection = 1;
+set optimize_use_projections = 1, force_optimize_projection = 1;
 
 select i from t prewhere j = 4;
 
diff --git a/tests/queries/0_stateless/01710_normal_projections.reference b/tests/queries/0_stateless/01710_normal_projections.reference
index 22ee679ce53..ccb4359b0e9 100644
--- a/tests/queries/0_stateless/01710_normal_projections.reference
+++ b/tests/queries/0_stateless/01710_normal_projections.reference
@@ -1,5 +1,5 @@
 select where x < 10
-optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0
+optimize_move_to_prewhere = 0, optimize_use_projections = 0
 0	4294967295
 1	4294967294
 2	4294967293
@@ -10,7 +10,7 @@ optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0
 7	4294967288
 8	4294967287
 9	4294967286
-optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1
+optimize_move_to_prewhere = 0, optimize_use_projections = 1
 0	4294967295
 1	4294967294
 2	4294967293
@@ -21,7 +21,7 @@ optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1
 7	4294967288
 8	4294967287
 9	4294967286
-optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0
+optimize_move_to_prewhere = 1, optimize_use_projections = 0
 0	4294967295
 1	4294967294
 2	4294967293
@@ -32,7 +32,7 @@ optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0
 7	4294967288
 8	4294967287
 9	4294967286
-optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1
+optimize_move_to_prewhere = 1, optimize_use_projections = 1
 0	4294967295
 1	4294967294
 2	4294967293
@@ -43,16 +43,16 @@ optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1
 7	4294967288
 8	4294967287
 9	4294967286
-optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0
+optimize_move_to_prewhere = 0, optimize_use_projections = 0
 		"rows_read": 100,
-optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1
+optimize_move_to_prewhere = 0, optimize_use_projections = 1
 		"rows_read": 100,
-optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0
+optimize_move_to_prewhere = 1, optimize_use_projections = 0
 		"rows_read": 100,
-optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1
+optimize_move_to_prewhere = 1, optimize_use_projections = 1
 		"rows_read": 100,
 select where y > 4294967286
-optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0
+optimize_move_to_prewhere = 0, optimize_use_projections = 0
 0	4294967295
 1	4294967294
 2	4294967293
@@ -62,7 +62,7 @@ optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0
 6	4294967289
 7	4294967288
 8	4294967287
-optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1
+optimize_move_to_prewhere = 0, optimize_use_projections = 1
 0	4294967295
 1	4294967294
 2	4294967293
@@ -72,7 +72,7 @@ optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1
 6	4294967289
 7	4294967288
 8	4294967287
-optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0
+optimize_move_to_prewhere = 1, optimize_use_projections = 0
 0	4294967295
 1	4294967294
 2	4294967293
@@ -82,7 +82,7 @@ optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0
 6	4294967289
 7	4294967288
 8	4294967287
-optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1
+optimize_move_to_prewhere = 1, optimize_use_projections = 1
 0	4294967295
 1	4294967294
 2	4294967293
@@ -92,12 +92,12 @@ optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1
 6	4294967289
 7	4294967288
 8	4294967287
-optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0
+optimize_move_to_prewhere = 0, optimize_use_projections = 0
 		"rows_read": 100,
-optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1
+optimize_move_to_prewhere = 0, optimize_use_projections = 1
 		"rows_read": 100,
-optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0
+optimize_move_to_prewhere = 1, optimize_use_projections = 0
 		"rows_read": 100,
-optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1
+optimize_move_to_prewhere = 1, optimize_use_projections = 1
 		"rows_read": 100,
 50
diff --git a/tests/queries/0_stateless/01710_normal_projections.sh b/tests/queries/0_stateless/01710_normal_projections.sh
index 3f2114b9a2b..5eed9b23573 100755
--- a/tests/queries/0_stateless/01710_normal_projections.sh
+++ b/tests/queries/0_stateless/01710_normal_projections.sh
@@ -9,73 +9,73 @@ $CLICKHOUSE_CLIENT -q "insert into test_sort_proj select number, toUInt32(-numbe
 
 echo "select where x < 10"
 
-echo "optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0"
+echo "optimize_move_to_prewhere = 0, optimize_use_projections = 0"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE x < 10 order by x
-                       SETTINGS optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0"
+                       SETTINGS optimize_move_to_prewhere = 0, optimize_use_projections = 0"
 
-echo "optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1"
+echo "optimize_move_to_prewhere = 0, optimize_use_projections = 1"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE x < 10 order by x
-                       SETTINGS optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1"
+                       SETTINGS optimize_move_to_prewhere = 0, optimize_use_projections = 1"
 
-echo "optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0"
+echo "optimize_move_to_prewhere = 1, optimize_use_projections = 0"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE x < 10 order by x
-                       SETTINGS optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0"
+                       SETTINGS optimize_move_to_prewhere = 1, optimize_use_projections = 0"
 
-echo "optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1"
+echo "optimize_move_to_prewhere = 1, optimize_use_projections = 1"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE x < 10 order by x
-                       SETTINGS optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1"
+                       SETTINGS optimize_move_to_prewhere = 1, optimize_use_projections = 1"
 
-echo "optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0"
+echo "optimize_move_to_prewhere = 0, optimize_use_projections = 0"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE x < 10 order by x FORMAT JSON
-                       SETTINGS optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0" | grep rows_read
+                       SETTINGS optimize_move_to_prewhere = 0, optimize_use_projections = 0" | grep rows_read
 
-echo "optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1"
+echo "optimize_move_to_prewhere = 0, optimize_use_projections = 1"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE x < 10 order by x FORMAT JSON
-                       SETTINGS optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1" | grep rows_read
+                       SETTINGS optimize_move_to_prewhere = 0, optimize_use_projections = 1" | grep rows_read
 
-echo "optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0"
+echo "optimize_move_to_prewhere = 1, optimize_use_projections = 0"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE x < 10 order by x FORMAT JSON
-                       SETTINGS optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0" | grep rows_read
+                       SETTINGS optimize_move_to_prewhere = 1, optimize_use_projections = 0" | grep rows_read
 
-echo "optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1"
+echo "optimize_move_to_prewhere = 1, optimize_use_projections = 1"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE x < 10 order by x FORMAT JSON
-                       SETTINGS optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1" | grep rows_read
+                       SETTINGS optimize_move_to_prewhere = 1, optimize_use_projections = 1" | grep rows_read
 
 
 echo "select where y > 4294967286"
 
-echo "optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0"
+echo "optimize_move_to_prewhere = 0, optimize_use_projections = 0"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE y > 4294967286 order by x
-                       SETTINGS optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0"
+                       SETTINGS optimize_move_to_prewhere = 0, optimize_use_projections = 0"
 
-echo "optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1"
+echo "optimize_move_to_prewhere = 0, optimize_use_projections = 1"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE y > 4294967286 order by x
-                       SETTINGS optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1"
+                       SETTINGS optimize_move_to_prewhere = 0, optimize_use_projections = 1"
 
-echo "optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0"
+echo "optimize_move_to_prewhere = 1, optimize_use_projections = 0"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE y > 4294967286 order by x
-                       SETTINGS optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0"
+                       SETTINGS optimize_move_to_prewhere = 1, optimize_use_projections = 0"
 
-echo "optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1"
+echo "optimize_move_to_prewhere = 1, optimize_use_projections = 1"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE y > 4294967286 order by x
-                       SETTINGS optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1"
+                       SETTINGS optimize_move_to_prewhere = 1, optimize_use_projections = 1"
 
 
-echo "optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0"
+echo "optimize_move_to_prewhere = 0, optimize_use_projections = 0"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE y > 4294967286 order by x FORMAT JSON
-                       SETTINGS optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 0" | grep rows_read
+                       SETTINGS optimize_move_to_prewhere = 0, optimize_use_projections = 0" | grep rows_read
 
-echo "optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1"
+echo "optimize_move_to_prewhere = 0, optimize_use_projections = 1"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE y > 4294967286 order by x FORMAT JSON
-                       SETTINGS optimize_move_to_prewhere = 0, allow_experimental_projection_optimization = 1" | grep rows_read
+                       SETTINGS optimize_move_to_prewhere = 0, optimize_use_projections = 1" | grep rows_read
 
-echo "optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0"
+echo "optimize_move_to_prewhere = 1, optimize_use_projections = 0"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE y > 4294967286 order by x FORMAT JSON
-                       SETTINGS optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 0" | grep rows_read
+                       SETTINGS optimize_move_to_prewhere = 1, optimize_use_projections = 0" | grep rows_read
 
-echo "optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1"
+echo "optimize_move_to_prewhere = 1, optimize_use_projections = 1"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_sort_proj WHERE y > 4294967286 order by x FORMAT JSON
-                       SETTINGS optimize_move_to_prewhere = 1, allow_experimental_projection_optimization = 1" | grep rows_read
+                       SETTINGS optimize_move_to_prewhere = 1, optimize_use_projections = 1" | grep rows_read
 
 $CLICKHOUSE_CLIENT -q "ALTER TABLE test_sort_proj DELETE WHERE x % 2 = 0 SETTINGS mutations_sync=2;"
 $CLICKHOUSE_CLIENT -q "SELECT count() from test_sort_proj;"
diff --git a/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql b/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql
index f9de1275dbd..e4fb1816c89 100644
--- a/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql
+++ b/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql
@@ -28,7 +28,7 @@ INSERT INTO normal SELECT
     number
 FROM numbers(100000);
 
-SET allow_experimental_projection_optimization=1, optimize_aggregation_in_order=1, force_optimize_projection=1;
+SET optimize_use_projections=1, optimize_aggregation_in_order=1, force_optimize_projection=1;
 
 WITH toStartOfHour(ts) AS a SELECT sum(value) v FROM normal WHERE ts > '2021-12-06 22:00:00' GROUP BY a ORDER BY v LIMIT 5;
 WITH toStartOfHour(ts) AS a SELECT sum(value) v FROM normal WHERE ts > '2021-12-06 22:00:00' GROUP BY toStartOfHour(ts), a ORDER BY v LIMIT 5;
@@ -60,7 +60,7 @@ INSERT INTO agg SELECT
     number
 FROM numbers(100000);
 
-SET allow_experimental_projection_optimization=1, optimize_aggregation_in_order=1, force_optimize_projection = 1;
+SET optimize_use_projections=1, optimize_aggregation_in_order=1, force_optimize_projection = 1;
 
 WITH toStartOfHour(ts) AS a SELECT sum(value) v FROM agg WHERE ts > '2021-12-06 22:00:00' GROUP BY a ORDER BY v LIMIT 5;
 WITH toStartOfHour(ts) AS a SELECT sum(value) v FROM agg WHERE ts > '2021-12-06 22:00:00' GROUP BY toStartOfHour(ts), a ORDER BY v LIMIT 5;
diff --git a/tests/queries/0_stateless/01710_projection_array_join.sql b/tests/queries/0_stateless/01710_projection_array_join.sql
index cd18d9282b9..509e053e5bf 100644
--- a/tests/queries/0_stateless/01710_projection_array_join.sql
+++ b/tests/queries/0_stateless/01710_projection_array_join.sql
@@ -1,4 +1,4 @@
-set allow_experimental_projection_optimization = 1;
+set optimize_use_projections = 1;
 
 drop table if exists x;
 
diff --git a/tests/queries/0_stateless/01710_projection_detach_part.sql b/tests/queries/0_stateless/01710_projection_detach_part.sql
index d28c0848d42..c77a2abfbf9 100644
--- a/tests/queries/0_stateless/01710_projection_detach_part.sql
+++ b/tests/queries/0_stateless/01710_projection_detach_part.sql
@@ -1,4 +1,4 @@
-set allow_experimental_projection_optimization = 1;
+set optimize_use_projections = 1;
 
 drop table if exists t;
 
diff --git a/tests/queries/0_stateless/01710_projection_in_index.sql b/tests/queries/0_stateless/01710_projection_in_index.sql
index 87f5e79e37e..b858418584c 100644
--- a/tests/queries/0_stateless/01710_projection_in_index.sql
+++ b/tests/queries/0_stateless/01710_projection_in_index.sql
@@ -4,7 +4,7 @@ create table t (i int, j int, k int, projection p (select * order by j)) engine
 
 insert into t select number, number, number from numbers(10);
 
-set allow_experimental_projection_optimization = 1, max_rows_to_read = 3;
+set optimize_use_projections = 1, max_rows_to_read = 3;
 
 select * from t where i < 5 and j in (1, 2);
 
diff --git a/tests/queries/0_stateless/01710_projection_in_set.sql b/tests/queries/0_stateless/01710_projection_in_set.sql
index 5bc84645cec..3d1abd0c09f 100644
--- a/tests/queries/0_stateless/01710_projection_in_set.sql
+++ b/tests/queries/0_stateless/01710_projection_in_set.sql
@@ -3,7 +3,7 @@ create table x (i UInt64, j UInt64, k UInt64, projection agg (select sum(j), avg
 
 insert into x values (1, 2, 3);
 
-set allow_experimental_projection_optimization = 1, use_index_for_in_with_subqueries = 0;
+set optimize_use_projections = 1, use_index_for_in_with_subqueries = 0;
 
 select sum(j), avg(k) from x where i in (select number from numbers(4));
 
diff --git a/tests/queries/0_stateless/01710_projection_mutation.sql b/tests/queries/0_stateless/01710_projection_mutation.sql
index 30e84f23a53..d963cde7409 100644
--- a/tests/queries/0_stateless/01710_projection_mutation.sql
+++ b/tests/queries/0_stateless/01710_projection_mutation.sql
@@ -4,6 +4,6 @@ CREATE TABLE t (`key` UInt32, `created_at` Date, `value` UInt32, PROJECTION xxx
 
 INSERT INTO t SELECT 1 AS key, today() + (number % 30), number FROM numbers(1000);
 
-ALTER TABLE t UPDATE value = 0 WHERE (value > 0) AND (created_at >= '2021-12-21') SETTINGS allow_experimental_projection_optimization = 1;
+ALTER TABLE t UPDATE value = 0 WHERE (value > 0) AND (created_at >= '2021-12-21') SETTINGS optimize_use_projections = 1;
 
 DROP TABLE IF EXISTS t;
diff --git a/tests/queries/0_stateless/01710_projection_vertical_merges.sql b/tests/queries/0_stateless/01710_projection_vertical_merges.sql
index e8a4a384017..2c4378bb7a4 100644
--- a/tests/queries/0_stateless/01710_projection_vertical_merges.sql
+++ b/tests/queries/0_stateless/01710_projection_vertical_merges.sql
@@ -12,7 +12,7 @@ optimize table t final;
 
 alter table t materialize projection p_norm settings mutations_sync = 1;
 
-set allow_experimental_projection_optimization = 1, max_rows_to_read = 3;
+set optimize_use_projections = 1, max_rows_to_read = 3;
 
 select c18 from t where c1 < 0;
 
diff --git a/tests/queries/0_stateless/01710_projection_with_joins.sql b/tests/queries/0_stateless/01710_projection_with_joins.sql
index f6ae9255b60..94c15d00e2a 100644
--- a/tests/queries/0_stateless/01710_projection_with_joins.sql
+++ b/tests/queries/0_stateless/01710_projection_with_joins.sql
@@ -2,20 +2,20 @@ drop table if exists t;
 
 create table t (s UInt16, l UInt16, projection p (select s, l order by l)) engine MergeTree order by s;
 
-select s from t join (select toUInt16(1) as s) x using (s) order by s settings allow_experimental_projection_optimization = 1;
-select s from t join (select toUInt16(1) as s) x using (s) order by s settings allow_experimental_projection_optimization = 0;
+select s from t join (select toUInt16(1) as s) x using (s) order by s settings optimize_use_projections = 1;
+select s from t join (select toUInt16(1) as s) x using (s) order by s settings optimize_use_projections = 0;
 
 drop table t;
 
 drop table if exists mt;
 create table mt (id1 Int8, id2 Int8) Engine=MergeTree order by tuple();
-select alias1 from (select id1, id1 as alias1 from mt) as l all inner join (select id2 as alias1 from mt) as t using (alias1) order by l.id1 settings allow_experimental_projection_optimization = 1;
-select id1 from mt all inner join (select id2 as id1 from mt) as t using (id1) order by id1 settings allow_experimental_projection_optimization = 1;
-select id2 as id1 from mt all inner join (select id1 from mt) as t using (id1) order by id1 settings allow_experimental_projection_optimization = 1;
+select alias1 from (select id1, id1 as alias1 from mt) as l all inner join (select id2 as alias1 from mt) as t using (alias1) order by l.id1 settings optimize_use_projections = 1;
+select id1 from mt all inner join (select id2 as id1 from mt) as t using (id1) order by id1 settings optimize_use_projections = 1;
+select id2 as id1 from mt all inner join (select id1 from mt) as t using (id1) order by id1 settings optimize_use_projections = 1;
 drop table mt;
 
 drop table if exists j;
 create table j (id1 Int8, id2 Int8, projection p (select id1, id2 order by id2)) Engine=MergeTree order by id1 settings index_granularity = 1;
 insert into j select number, number from numbers(10);
-select alias1 from (select id1, id1 as alias1 from j) as l all inner join (select id2, id2 as alias1 from j where id2 in (1, 2, 3)) as t using (alias1) where id2 in (2, 3, 4) order by id1 settings allow_experimental_projection_optimization = 1;
+select alias1 from (select id1, id1 as alias1 from j) as l all inner join (select id2, id2 as alias1 from j where id2 in (1, 2, 3)) as t using (alias1) where id2 in (2, 3, 4) order by id1 settings optimize_use_projections = 1;
 drop table j;
diff --git a/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql b/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql
index 5169c667b81..877fca4590d 100644
--- a/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql
+++ b/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql
@@ -4,6 +4,6 @@ create table t (x UInt32) engine = MergeTree order by tuple() settings index_gra
 insert into t select number from numbers(100);
 alter table t add projection p (select uniqHLL12(x));
 insert into t select number + 100 from numbers(100);
-select uniqHLL12(x) from t settings allow_experimental_projection_optimization = 1, max_bytes_to_read=400, max_block_size=8; -- { serverError 307 }
+select uniqHLL12(x) from t settings optimize_use_projections = 1, max_bytes_to_read=400, max_block_size=8; -- { serverError 307 }
 
 drop table if exists t;
diff --git a/tests/queries/0_stateless/01710_projections.sql b/tests/queries/0_stateless/01710_projections.sql
index 111dd6eb16e..a96339e30fa 100644
--- a/tests/queries/0_stateless/01710_projections.sql
+++ b/tests/queries/0_stateless/01710_projections.sql
@@ -4,7 +4,7 @@ create table projection_test (`sum(block_count)` UInt64, domain_alias UInt64 ali
 
 insert into projection_test with rowNumberInAllBlocks() as id select 1, toDateTime('2020-10-24 00:00:00') + (id / 20), toString(id % 100), * from generateRandom('x_id String, y_id String, block_count Int64, retry_count Int64, duration Int64, kbytes Int64, buffer_time Int64, first_time Int64, total_bytes Nullable(UInt64), valid_bytes Nullable(UInt64), completed_bytes Nullable(UInt64), fixed_bytes Nullable(UInt64), force_bytes Nullable(UInt64)', 10, 10, 1) limit 1000 settings max_threads = 1;
 
-set allow_experimental_projection_optimization = 1, force_optimize_projection = 1;
+set optimize_use_projections = 1, force_optimize_projection = 1;
 
 select * from projection_test; -- { serverError 584 }
 select toStartOfMinute(datetime) dt_m, countIf(first_time = 0) from projection_test join (select 1) x on 1 where domain = '1' group by dt_m order by dt_m; -- { serverError 584 }
@@ -47,6 +47,6 @@ drop table if exists projection_test;
 drop table if exists projection_without_key;
 create table projection_without_key (key UInt32, PROJECTION x (SELECT max(key))) engine MergeTree order by key;
 insert into projection_without_key select number from numbers(1000);
-set force_optimize_projection = 1, allow_experimental_projection_optimization = 1;
+set force_optimize_projection = 1, optimize_use_projections = 1;
 select max(key) from projection_without_key;
 drop table projection_without_key;
diff --git a/tests/queries/0_stateless/01710_projections_group_by_no_key.sql b/tests/queries/0_stateless/01710_projections_group_by_no_key.sql
index eefc03afb7a..98545bdd05a 100644
--- a/tests/queries/0_stateless/01710_projections_group_by_no_key.sql
+++ b/tests/queries/0_stateless/01710_projections_group_by_no_key.sql
@@ -2,7 +2,7 @@ drop table if exists projection_without_key;
 
 create table projection_without_key (key UInt32, PROJECTION x (SELECT sum(key) group by key % 3)) engine MergeTree order by key;
 insert into projection_without_key select number from numbers(1000);
-select sum(key) from projection_without_key settings allow_experimental_projection_optimization = 1;
-select sum(key) from projection_without_key settings allow_experimental_projection_optimization = 0;
+select sum(key) from projection_without_key settings optimize_use_projections = 1;
+select sum(key) from projection_without_key settings optimize_use_projections = 0;
 
 drop table projection_without_key;
diff --git a/tests/queries/0_stateless/01710_projections_in_distributed_query.sql b/tests/queries/0_stateless/01710_projections_in_distributed_query.sql
index 7600340579d..616bf3b5505 100644
--- a/tests/queries/0_stateless/01710_projections_in_distributed_query.sql
+++ b/tests/queries/0_stateless/01710_projections_in_distributed_query.sql
@@ -8,7 +8,7 @@ create table projection_test (dt DateTime, cost Int64, projection p (select toSt
 
 insert into projection_test with rowNumberInAllBlocks() as id select toDateTime('2020-10-24 00:00:00') + (id / 20), * from generateRandom('cost Int64', 10, 10, 1) limit 1000 settings max_threads = 1;
 
-set allow_experimental_projection_optimization = 1, force_optimize_projection = 1;
+set optimize_use_projections = 1, force_optimize_projection = 1;
 
 select toStartOfMinute(dt) dt_m, sum(cost) from projection_test group by dt_m;
 select sum(cost) from projection_test;
diff --git a/tests/queries/0_stateless/01710_projections_optimize_aggregation_in_order.sh b/tests/queries/0_stateless/01710_projections_optimize_aggregation_in_order.sh
index 2c6a6ef35eb..a166837e01a 100755
--- a/tests/queries/0_stateless/01710_projections_optimize_aggregation_in_order.sh
+++ b/tests/queries/0_stateless/01710_projections_optimize_aggregation_in_order.sh
@@ -38,7 +38,7 @@ function run_query()
 
     echo "$query"
     local opts=(
-        --allow_experimental_projection_optimization 1
+        --optimize_use_projections 1
         --force_optimize_projection 1
         --log_processors_profiles 1
         --query_id "$query_id"
diff --git a/tests/queries/0_stateless/01710_projections_partial_optimize_aggregation_in_order.sh b/tests/queries/0_stateless/01710_projections_partial_optimize_aggregation_in_order.sh
index 5a9c480c78c..ee73974e8a4 100755
--- a/tests/queries/0_stateless/01710_projections_partial_optimize_aggregation_in_order.sh
+++ b/tests/queries/0_stateless/01710_projections_partial_optimize_aggregation_in_order.sh
@@ -44,7 +44,7 @@ function run_query()
 
     echo "$query"
     local opts=(
-        --allow_experimental_projection_optimization 1
+        --optimize_use_projections 1
         --force_optimize_projection 1
         --log_processors_profiles 1
         --query_id "$query_id"
diff --git a/tests/queries/0_stateless/02494_zero_copy_and_projection_and_mutation_work_together.sql b/tests/queries/0_stateless/02494_zero_copy_and_projection_and_mutation_work_together.sql
index 7a51d86dd30..98427874160 100644
--- a/tests/queries/0_stateless/02494_zero_copy_and_projection_and_mutation_work_together.sql
+++ b/tests/queries/0_stateless/02494_zero_copy_and_projection_and_mutation_work_together.sql
@@ -72,8 +72,8 @@ SYSTEM SYNC REPLICA wikistat2;
 -- Such condition will lead to successful queries.
 SELECT 0 FROM numbers(5) WHERE sleepEachRow(1) = 1;
 
-select sum(hits), count() from wikistat1 GROUP BY project, subproject, path settings allow_experimental_projection_optimization = 1, force_optimize_projection = 1;
-select sum(hits), count() from wikistat2 GROUP BY project, subproject, path settings allow_experimental_projection_optimization = 1, force_optimize_projection = 1;
+select sum(hits), count() from wikistat1 GROUP BY project, subproject, path settings optimize_use_projections = 1, force_optimize_projection = 1;
+select sum(hits), count() from wikistat2 GROUP BY project, subproject, path settings optimize_use_projections = 1, force_optimize_projection = 1;
 
 DROP TABLE wikistat1;
 DROP TABLE wikistat2;
diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql
index aaa1b8b9b2d..b7d4a6ee93a 100644
--- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql
+++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql
@@ -3,7 +3,7 @@
 set max_threads = 16;
 set allow_aggregate_partitions_independently = 1;
 set force_aggregate_partitions_independently = 1;
-set allow_experimental_projection_optimization = 0;
+set optimize_use_projections = 0;
 
 create table t1(a UInt32) engine=MergeTree order by tuple() partition by a % 4 settings index_granularity = 8192, index_granularity_bytes = 10485760;
 
diff --git a/tests/queries/0_stateless/02524_fuzz_and_fuss_2.sql b/tests/queries/0_stateless/02524_fuzz_and_fuss_2.sql
index e32238001bf..f0d8d5202db 100644
--- a/tests/queries/0_stateless/02524_fuzz_and_fuss_2.sql
+++ b/tests/queries/0_stateless/02524_fuzz_and_fuss_2.sql
@@ -9,6 +9,6 @@ ENGINE = Memory;
 INSERT INTO data_a_02187
 SELECT *
 FROM system.one
-SETTINGS max_block_size = '1', min_insert_block_size_rows = '65536', min_insert_block_size_bytes = '0', max_insert_threads = '0', max_threads = '3', receive_timeout = '10', receive_data_timeout_ms = '10000', connections_with_failover_max_tries = '0', extremes = '1', use_uncompressed_cache = '0', optimize_move_to_prewhere = '1', optimize_move_to_prewhere_if_final = '0', replication_alter_partitions_sync = '2', totals_mode = 'before_having', allow_suspicious_low_cardinality_types = '1', compile_expressions = '1', min_count_to_compile_expression = '0', group_by_two_level_threshold = '100', distributed_aggregation_memory_efficient = '0', distributed_group_by_no_merge = '1', optimize_distributed_group_by_sharding_key = '1', optimize_skip_unused_shards = '1', optimize_skip_unused_shards_rewrite_in = '1', force_optimize_skip_unused_shards = '2', optimize_skip_unused_shards_nesting = '1', force_optimize_skip_unused_shards_nesting = '2', merge_tree_min_rows_for_concurrent_read = '10000', force_primary_key = '1', network_compression_method = 'ZSTD', network_zstd_compression_level = '7', log_queries = '0', log_queries_min_type = 'QUERY_FINISH', distributed_product_mode = 'local', insert_quorum = '2', insert_quorum_timeout = '0', insert_quorum_parallel = '0', select_sequential_consistency = '1', join_use_nulls = '1', any_join_distinct_right_table_keys = '1', preferred_max_column_in_block_size_bytes = '32', insert_distributed_sync = '1', insert_allow_materialized_columns = '1', use_index_for_in_with_subqueries = '1', joined_subquery_requires_alias = '0', empty_result_for_aggregation_by_empty_set = '1', allow_suspicious_codecs = '1', query_profiler_real_time_period_ns = '0', query_profiler_cpu_time_period_ns = '0', opentelemetry_start_trace_probability = '1', max_rows_to_read = '1000000', read_overflow_mode = 'break', max_rows_to_group_by = '10', group_by_overflow_mode = 'any', max_rows_to_sort = '100', sort_overflow_mode = 'break', max_result_rows = '10', max_execution_time = '3', max_execution_speed = '1', max_bytes_in_join = '100', join_algorithm = 'partial_merge', max_memory_usage = '1099511627776', log_query_threads = '1', send_logs_level = 'fatal', enable_optimize_predicate_expression = '1', prefer_localhost_replica = '1', optimize_read_in_order = '1', optimize_aggregation_in_order = '1', read_in_order_two_level_merge_threshold = '1', allow_introspection_functions = '1', check_query_single_value_result = '1', allow_experimental_live_view = '1', default_table_engine = 'Memory', mutations_sync = '2', convert_query_to_cnf = '0', optimize_arithmetic_operations_in_aggregate_functions = '1', optimize_duplicate_order_by_and_distinct = '0', optimize_multiif_to_if = '0', optimize_monotonous_functions_in_order_by = '1', optimize_functions_to_subcolumns = '1', optimize_using_constraints = '1', optimize_substitute_columns = '1', optimize_append_index = '1', transform_null_in = '1', allow_experimental_geo_types = '1', data_type_default_nullable = '1', cast_keep_nullable = '1', cast_ipv4_ipv6_default_on_conversion_error = '0', system_events_show_zero_values = '1', enable_global_with_statement = '1', optimize_on_insert = '0', optimize_rewrite_sum_if_to_count_if = '1', distributed_ddl_output_mode = 'throw', union_default_mode = 'ALL', optimize_aggregators_of_group_by_keys = '1', optimize_group_by_function_keys = '1', short_circuit_function_evaluation = 'enable', async_insert = '1', enable_filesystem_cache = '0', allow_deprecated_database_ordinary = '1', allow_deprecated_syntax_for_merge_tree = '1', allow_experimental_nlp_functions = '1', allow_experimental_object_type = '1', allow_experimental_map_type = '1', allow_experimental_projection_optimization = '1', input_format_null_as_default = '1', input_format_ipv4_default_on_conversion_error = '0', input_format_ipv6_default_on_conversion_error = '0', output_format_json_named_tuples_as_objects = '1', output_format_write_statistics = '0', output_format_pretty_row_numbers = '1';
+SETTINGS max_block_size = '1', min_insert_block_size_rows = '65536', min_insert_block_size_bytes = '0', max_insert_threads = '0', max_threads = '3', receive_timeout = '10', receive_data_timeout_ms = '10000', connections_with_failover_max_tries = '0', extremes = '1', use_uncompressed_cache = '0', optimize_move_to_prewhere = '1', optimize_move_to_prewhere_if_final = '0', replication_alter_partitions_sync = '2', totals_mode = 'before_having', allow_suspicious_low_cardinality_types = '1', compile_expressions = '1', min_count_to_compile_expression = '0', group_by_two_level_threshold = '100', distributed_aggregation_memory_efficient = '0', distributed_group_by_no_merge = '1', optimize_distributed_group_by_sharding_key = '1', optimize_skip_unused_shards = '1', optimize_skip_unused_shards_rewrite_in = '1', force_optimize_skip_unused_shards = '2', optimize_skip_unused_shards_nesting = '1', force_optimize_skip_unused_shards_nesting = '2', merge_tree_min_rows_for_concurrent_read = '10000', force_primary_key = '1', network_compression_method = 'ZSTD', network_zstd_compression_level = '7', log_queries = '0', log_queries_min_type = 'QUERY_FINISH', distributed_product_mode = 'local', insert_quorum = '2', insert_quorum_timeout = '0', insert_quorum_parallel = '0', select_sequential_consistency = '1', join_use_nulls = '1', any_join_distinct_right_table_keys = '1', preferred_max_column_in_block_size_bytes = '32', insert_distributed_sync = '1', insert_allow_materialized_columns = '1', use_index_for_in_with_subqueries = '1', joined_subquery_requires_alias = '0', empty_result_for_aggregation_by_empty_set = '1', allow_suspicious_codecs = '1', query_profiler_real_time_period_ns = '0', query_profiler_cpu_time_period_ns = '0', opentelemetry_start_trace_probability = '1', max_rows_to_read = '1000000', read_overflow_mode = 'break', max_rows_to_group_by = '10', group_by_overflow_mode = 'any', max_rows_to_sort = '100', sort_overflow_mode = 'break', max_result_rows = '10', max_execution_time = '3', max_execution_speed = '1', max_bytes_in_join = '100', join_algorithm = 'partial_merge', max_memory_usage = '1099511627776', log_query_threads = '1', send_logs_level = 'fatal', enable_optimize_predicate_expression = '1', prefer_localhost_replica = '1', optimize_read_in_order = '1', optimize_aggregation_in_order = '1', read_in_order_two_level_merge_threshold = '1', allow_introspection_functions = '1', check_query_single_value_result = '1', allow_experimental_live_view = '1', default_table_engine = 'Memory', mutations_sync = '2', convert_query_to_cnf = '0', optimize_arithmetic_operations_in_aggregate_functions = '1', optimize_duplicate_order_by_and_distinct = '0', optimize_multiif_to_if = '0', optimize_monotonous_functions_in_order_by = '1', optimize_functions_to_subcolumns = '1', optimize_using_constraints = '1', optimize_substitute_columns = '1', optimize_append_index = '1', transform_null_in = '1', allow_experimental_geo_types = '1', data_type_default_nullable = '1', cast_keep_nullable = '1', cast_ipv4_ipv6_default_on_conversion_error = '0', system_events_show_zero_values = '1', enable_global_with_statement = '1', optimize_on_insert = '0', optimize_rewrite_sum_if_to_count_if = '1', distributed_ddl_output_mode = 'throw', union_default_mode = 'ALL', optimize_aggregators_of_group_by_keys = '1', optimize_group_by_function_keys = '1', short_circuit_function_evaluation = 'enable', async_insert = '1', enable_filesystem_cache = '0', allow_deprecated_database_ordinary = '1', allow_deprecated_syntax_for_merge_tree = '1', allow_experimental_nlp_functions = '1', allow_experimental_object_type = '1', allow_experimental_map_type = '1', optimize_use_projections = '1', input_format_null_as_default = '1', input_format_ipv4_default_on_conversion_error = '0', input_format_ipv6_default_on_conversion_error = '0', output_format_json_named_tuples_as_objects = '1', output_format_write_statistics = '0', output_format_pretty_row_numbers = '1';
 
 DROP TABLE data_a_02187;

From a15d088fd2c615edcd5ac07b38f8d389b5c28ac5 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 10 May 2023 09:20:16 +0200
Subject: [PATCH 378/689] Fix 02417_opentelemetry_insert_on_distributed_table
 (wrong sed was used)

---
 ...entelemetry_insert_on_distributed_table.reference | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference
index 02df9aa06c2..fe2cd9d666a 100644
--- a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference
+++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference
@@ -1,6 +1,6 @@
 ===1===
-{"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const Block const DB::Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
-{"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const Block const DB::Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
+{"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
+{"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
 1
 ===2===
 {"operation_name":"void DB::DistributedAsyncInsertDirectoryQueue::processFile(const std::string &)","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
@@ -8,11 +8,11 @@
 3
 2
 ===3===
-{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
-{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
+{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
+{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
 1
 ===4===
-{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
-{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
+{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
+{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
 3
 2

From f6372dd3fc1ceb1256b7aec37e86559e405c7cf9 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 10 May 2023 12:14:44 +0200
Subject: [PATCH 379/689] Do not launch PRs w/o "can be tested"

---
 tests/ci/workflow_approve_rerun_lambda/app.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/ci/workflow_approve_rerun_lambda/app.py b/tests/ci/workflow_approve_rerun_lambda/app.py
index 0f6d10de751..53a08617256 100644
--- a/tests/ci/workflow_approve_rerun_lambda/app.py
+++ b/tests/ci/workflow_approve_rerun_lambda/app.py
@@ -313,11 +313,12 @@ def check_suspicious_changed_files(changed_files):
                 )
                 return True
 
-    print("No changed files match suspicious patterns, run will be approved")
+    print("No changed files match suspicious patterns, run could be approved")
     return False
 
 
 def approve_run(workflow_description: WorkflowDescription, token: str) -> None:
+    print("Approving run")
     url = f"{workflow_description.api_url}/approve"
     _exec_post_with_retry(url, token)
 
@@ -480,6 +481,11 @@ def main(event):
         approve_run(workflow_description, token)
         return
 
+    labels = {label["name"] for label in pull_request["labels"]}
+    if "can be tested" not in labels:
+        print("Label 'can be tested' is required for untrusted users")
+        return
+
     changed_files = get_changed_files_for_pull_request(pull_request, token)
     print(f"Totally have {len(changed_files)} changed files in PR:", changed_files)
     if check_suspicious_changed_files(changed_files):

From 2f9a7cba8a3669305b66a477b9c64386d655a9b5 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 10 May 2023 12:15:17 +0200
Subject: [PATCH 380/689] Fix stack-use-after-scope in KafkaProducer::produce()

ASan report:

    ==1==ERROR: AddressSanitizer: stack-use-after-scope on address 0x7f7540186b20 at pc 0x55656ef458ad bp 0x7f754114acf0 sp 0x7f754114a4b8
    READ of size 2 at 0x7f7540186b20 thread T2 (TCPHandler)
        0 0x55656ef458ac in __asan_memcpy (/usr/bin/clickhouse+0x11b028ac) (BuildId: 6ba63057df561f6dc16fb8fd7623fd858bf327d1)
        1 0x55659f62daf8 in rd_kafka_msg_new00 build_docker/./contrib/librdkafka/src/rdkafka_msg.c:192:3
        2 0x55659f62daf8 in rd_kafka_msg_new0 build_docker/./contrib/librdkafka/src/rdkafka_msg.c:257:8
        3 0x55659f630635 in rd_kafka_producev build_docker/./contrib/librdkafka/src/rdkafka_msg.c:642:23
        4 0x55659f4a865c in cppkafka::Producer::do_produce() build_docker/./contrib/cppkafka/src/producer.cpp:128:19
        5 0x55659f4a8306 in cppkafka::Producer::produce() build_docker/./contrib/cppkafka/src/producer.cpp:69:5
        6 0x5565961e748c in DB::KafkaProducer::produce() build_docker/./src/Storages/Kafka/KafkaProducer.cpp:70:23
        7 0x5565962155b2 in DB::MessageQueueSink::consume(DB::Chunk) build_docker/./src/Storages/MessageQueueSink.cpp:64:23
        8 0x5565980b9e8e in DB::SinkToStorage::onConsume(DB::Chunk) build_docker/./src/Processors/Sinks/SinkToStorage.cpp:18:5

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Storages/Kafka/KafkaProducer.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Storages/Kafka/KafkaProducer.cpp b/src/Storages/Kafka/KafkaProducer.cpp
index 5ef5d080ddf..c36f9b48315 100644
--- a/src/Storages/Kafka/KafkaProducer.cpp
+++ b/src/Storages/Kafka/KafkaProducer.cpp
@@ -52,8 +52,8 @@ void KafkaProducer::produce(const String & message, size_t rows_in_message, cons
     if (key_column_index)
     {
         const auto & key_column = assert_cast<const ColumnString &>(*columns[key_column_index.value()]);
-        const auto key_data = key_column.getDataAt(last_row).toString();
-        builder.key(cppkafka::Buffer(key_data.data(), key_data.size()));
+        const auto key_data = key_column.getDataAt(last_row);
+        builder.key(cppkafka::Buffer(key_data.data, key_data.size));
     }
 
     if (timestamp_column_index)

From d605f8279815c5cfb551e1899374bb5b7306785c Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 10 May 2023 12:36:04 +0200
Subject: [PATCH 381/689] Replace old release with new packages in suspicious
 patterns

---
 tests/ci/workflow_approve_rerun_lambda/app.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/ci/workflow_approve_rerun_lambda/app.py b/tests/ci/workflow_approve_rerun_lambda/app.py
index 53a08617256..7d234bdcfc4 100644
--- a/tests/ci/workflow_approve_rerun_lambda/app.py
+++ b/tests/ci/workflow_approve_rerun_lambda/app.py
@@ -12,12 +12,11 @@ import boto3  # type: ignore
 SUSPICIOUS_CHANGED_FILES_NUMBER = 200
 
 SUSPICIOUS_PATTERNS = [
-    "tests/ci/*",
-    "docs/tools/*",
     ".github/*",
-    "utils/release/*",
     "docker/*",
-    "release",
+    "docs/tools/*",
+    "packages/*",
+    "tests/ci/*",
 ]
 
 # Number of retries for API calls.

From acae08714b27f7e23ea484aed996be5f7abbd1ae Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Wed, 10 May 2023 13:03:34 +0200
Subject: [PATCH 382/689] Move assertions after logging

---
 src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
index 00d23183f6a..1aaf9c977a4 100644
--- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
+++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
@@ -389,14 +389,6 @@ CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegment & file_segme
     auto read_buffer_for_file_segment = getReadBufferForFileSegment(file_segment);
 
     watch.stop();
-    current_file_segment_counters.increment(
-        ProfileEvents::FileSegmentWaitReadBufferMicroseconds, watch.elapsedMicroseconds());
-
-    [[maybe_unused]] auto download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE;
-    chassert(download_current_segment == file_segment.isDownloader());
-
-    chassert(file_segment.range() == range);
-    chassert(file_offset_of_buffer_end >= range.left && file_offset_of_buffer_end <= range.right);
 
     LOG_TEST(
         log,
@@ -406,6 +398,15 @@ CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegment & file_segme
         read_buffer_for_file_segment->getFileOffsetOfBufferEnd(),
         file_segment.getInfoForLog());
 
+    current_file_segment_counters.increment(
+        ProfileEvents::FileSegmentWaitReadBufferMicroseconds, watch.elapsedMicroseconds());
+
+    [[maybe_unused]] auto download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE;
+    chassert(download_current_segment == file_segment.isDownloader());
+
+    chassert(file_segment.range() == range);
+    chassert(file_offset_of_buffer_end >= range.left && file_offset_of_buffer_end <= range.right);
+
     read_buffer_for_file_segment->setReadUntilPosition(range.right + 1); /// [..., range.right]
 
     switch (read_type)

From 405e371acadf3f15a0c56f5e6370713f1fc7b342 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Wed, 10 May 2023 11:18:15 +0000
Subject: [PATCH 383/689] Docs: Fix sidebar label for dictionary table function

---
 docs/en/sql-reference/table-functions/dictionary.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/table-functions/dictionary.md b/docs/en/sql-reference/table-functions/dictionary.md
index c4bdde4dce2..73d5039a64b 100644
--- a/docs/en/sql-reference/table-functions/dictionary.md
+++ b/docs/en/sql-reference/table-functions/dictionary.md
@@ -1,7 +1,7 @@
 ---
 slug: /en/sql-reference/table-functions/dictionary
 sidebar_position: 54
-sidebar_label: dictionary function
+sidebar_label: dictionary
 title: dictionary
 ---
 

From 505d9be7b22ed6be344593f12841c65e45aa634f Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 10 May 2023 13:56:17 +0200
Subject: [PATCH 384/689] address comments

---
 contrib/libfiu-cmake/CMakeLists.txt                |  2 +-
 docker/test/fasttest/run.sh                        |  2 +-
 src/Common/FailPoint.cpp                           | 14 +++++++++++++-
 src/Common/FailPoint.h                             | 12 ++++++++++--
 .../02718_insert_meet_hardware_error.sql           |  2 +-
 5 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/contrib/libfiu-cmake/CMakeLists.txt b/contrib/libfiu-cmake/CMakeLists.txt
index eb66d86c704..e805491edbb 100644
--- a/contrib/libfiu-cmake/CMakeLists.txt
+++ b/contrib/libfiu-cmake/CMakeLists.txt
@@ -1,5 +1,5 @@
 if (NOT ENABLE_FIU)
-  message (STATUS "Not using gwp-asan")
+  message (STATUS "Not using fiu")
   return ()
 endif ()
 
diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index 6b320a20665..135d33bf5cc 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -147,7 +147,7 @@ function clone_submodules
             contrib/xxHash
             contrib/simdjson
             contrib/liburing
-	    contrib/libfiu
+            contrib/libfiu
         )
 
         git submodule sync
diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp
index 85cc6853fdc..4f57e04ffea 100644
--- a/src/Common/FailPoint.cpp
+++ b/src/Common/FailPoint.cpp
@@ -25,6 +25,12 @@ static struct InitFiu
 } init_fiu;
 #endif
 
+/// We should define different types of failpoints here. There are four types of them:
+/// - ONCE: the failpoint will only be triggered once.
+/// - REGULAR: the failpoint will always be triggered util disableFailPoint is called.
+/// - PAUSAEBLE_ONCE: the failpoint will be blocked one time when pauseFailPoint is called, util disableFailPoint is called.
+/// - PAUSAEBLE: the failpoint will be blocked every time when pauseFailPoint is called, util disableFailPoint is called.
+
 #define APPLY_FOR_FAILPOINTS(ONCE, REGULAR, PAUSEABLE_ONCE, PAUSEABLE) \
     ONCE(replicated_merge_tree_commit_zk_fail_after_op) \
     REGULAR(dummy_failpoint) \
@@ -96,6 +102,11 @@ void FailPointInjection::enablePauseFailPoint(const String & fail_point_name, UI
     throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find fail point {}", fail_point_name);
 }
 
+void FailPointInjection::pauseFailPoint(const String & fail_point_name)
+{
+    fiu_do_on(fail_point_name.c_str(), FailPointInjection::wait(fail_point_name););
+}
+
 void FailPointInjection::enableFailPoint(const String & fail_point_name)
 {
 #if FIU_ENABLE
@@ -141,11 +152,12 @@ void FailPointInjection::disableFailPoint(const String & fail_point_name)
 
 void FailPointInjection::wait(const String & fail_point_name)
 {
-    std::lock_guard lock(mu);
+    std::unique_lock lock(mu);
     if (auto iter = fail_point_wait_channels.find(fail_point_name); iter == fail_point_wait_channels.end())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Can not find channel for fail point {}", fail_point_name);
     else
     {
+        lock.unlock();
         auto ptr = iter->second;
         ptr->wait();
     }
diff --git a/src/Common/FailPoint.h b/src/Common/FailPoint.h
index c7505766907..f61863b8a1c 100644
--- a/src/Common/FailPoint.h
+++ b/src/Common/FailPoint.h
@@ -23,13 +23,21 @@
 namespace DB
 {
 
-// When `fail_point` is enabled, wait till it is disabled
-#define FAIL_POINT_PAUSE(fail_point) fiu_do_on(fail_point, FailPointHelper::wait(fail_point);)
+/// This is a simple named failpoint library inspired by https://github.com/pingcap/tiflash
+/// The usage is simple:
+/// 1. define failpoint with a 'failpoint_name' in FailPoint.cpp
+/// 2. inject failpoint in normal code
+///   2.1 use fiu_do_on which can inject any code blocks, when it is a regular-triggered / once-triggered failpoint
+///   2.2 use pauseFailPoint when it is a pausable failpoint
+/// 3. in test file, we can use system failpoint enable/disable 'failpoint_name'
 
 class FailPointChannel;
 class FailPointInjection
 {
 public:
+
+    static void pauseFailPoint(const String & fail_point_name);
+
     static void enableFailPoint(const String & fail_point_name);
 
     static void enablePauseFailPoint(const String & fail_point_name, UInt64 time);
diff --git a/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql b/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql
index 65354e0624a..ab90bbecc04 100644
--- a/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql
+++ b/tests/queries/0_stateless/02718_insert_meet_hardware_error.sql
@@ -1,4 +1,4 @@
--- Tags: zookeeper
+-- Tags: zookeeper, no-parallel
 
 DROP TABLE IF EXISTS t_hardware_error NO DELAY;
 

From 632c0eeefcdcefbfa3fd8da7e1d21f84bffc2bd2 Mon Sep 17 00:00:00 2001
From: Roman Vasin <r.vasin@arenadata.io>
Date: Wed, 10 May 2023 15:25:12 +0300
Subject: [PATCH 385/689] Add support of temporary tables in FROM part of
 ATTACH|REPLACE PARTITION FROM (#49436)

* Add temp tables after FROM

* Resolve works fine with temp and non-temp tables

* Add usage of temporary table in  00626_replace_partition_from_table test

* Update documentation

* Add tests for ReplicatedMergeTree

* Move tests into 02731_replace_partition_from_temporary_table
---
 .../statements/alter/partition.md             | 13 ++++-
 .../statements/alter/partition.md             | 13 ++++-
 src/Storages/MergeTree/MergeTreeData.cpp      |  4 +-
 ...e_partition_from_temporary_table.reference |  9 ++++
 ...replace_partition_from_temporary_table.sql | 48 +++++++++++++++++++
 5 files changed, 81 insertions(+), 6 deletions(-)
 create mode 100644 tests/queries/0_stateless/02731_replace_partition_from_temporary_table.reference
 create mode 100644 tests/queries/0_stateless/02731_replace_partition_from_temporary_table.sql

diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md
index 52e99d93109..ce0bc1ea528 100644
--- a/docs/en/sql-reference/statements/alter/partition.md
+++ b/docs/en/sql-reference/statements/alter/partition.md
@@ -103,7 +103,11 @@ ALTER TABLE table2 [ON CLUSTER cluster] ATTACH PARTITION partition_expr FROM tab
 ```
 
 This query copies the data partition from `table1` to `table2`.
-Note that data will be deleted neither from `table1` nor from `table2`.
+
+Note that:
+
+- Data will be deleted neither from `table1` nor from `table2`.
+- `table1` may be a temporary table.
 
 For the query to run successfully, the following conditions must be met:
 
@@ -117,7 +121,12 @@ For the query to run successfully, the following conditions must be met:
 ALTER TABLE table2 [ON CLUSTER cluster] REPLACE PARTITION partition_expr FROM table1
 ```
 
-This query copies the data partition from the `table1` to `table2` and replaces existing partition in the `table2`. Note that data won’t be deleted from `table1`.
+This query copies the data partition from the `table1` to `table2` and replaces existing partition in the `table2`.
+
+Note that:
+
+- Data won’t be deleted from `table1`.
+- `table1` may be a temporary table.
 
 For the query to run successfully, the following conditions must be met:
 
diff --git a/docs/ru/sql-reference/statements/alter/partition.md b/docs/ru/sql-reference/statements/alter/partition.md
index 95d02c062bd..90688c9ece2 100644
--- a/docs/ru/sql-reference/statements/alter/partition.md
+++ b/docs/ru/sql-reference/statements/alter/partition.md
@@ -102,7 +102,11 @@ ALTER TABLE table2 [ON CLUSTER cluster] ATTACH PARTITION partition_expr FROM tab
 ```
 
 Копирует партицию из таблицы `table1` в таблицу `table2`.
-Обратите внимание, что данные не удаляются ни из `table1`, ни из `table2`.
+
+Обратите внимание, что:
+
+- Данные не удаляются ни из `table1`, ни из `table2`.
+- `table1` может быть временной таблицей.
 
 Следует иметь в виду:
 
@@ -118,7 +122,12 @@ ALTER TABLE table2 [ON CLUSTER cluster] ATTACH PARTITION partition_expr FROM tab
 ALTER TABLE table2 [ON CLUSTER cluster] REPLACE PARTITION partition_expr FROM table1
 ```
 
-Копирует партицию из таблицы `table1` в таблицу `table2` с заменой существующих данных в `table2`. Данные из `table1` не удаляются.
+Копирует партицию из таблицы `table1` в таблицу `table2` с заменой существующих данных в `table2`.
+
+Обратите внимание, что:
+
+- Данные из `table1` не удаляются.
+- `table1` может быть временной таблицей.
 
 Следует иметь в виду:
 
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index a2a3348eb42..d200d8bb19e 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -4985,8 +4985,8 @@ Pipe MergeTreeData::alterPartition(
                 if (command.replace)
                     checkPartitionCanBeDropped(command.partition, query_context);
 
-                String from_database = query_context->resolveDatabase(command.from_database);
-                auto from_storage = DatabaseCatalog::instance().getTable({from_database, command.from_table}, query_context);
+                auto resolved = query_context->resolveStorageID({command.from_database, command.from_table});
+                auto from_storage = DatabaseCatalog::instance().getTable(resolved, query_context);
 
                 auto * from_storage_merge_tree = dynamic_cast<MergeTreeData *>(from_storage.get());
                 if (!from_storage_merge_tree)
diff --git a/tests/queries/0_stateless/02731_replace_partition_from_temporary_table.reference b/tests/queries/0_stateless/02731_replace_partition_from_temporary_table.reference
new file mode 100644
index 00000000000..ced682dd94f
--- /dev/null
+++ b/tests/queries/0_stateless/02731_replace_partition_from_temporary_table.reference
@@ -0,0 +1,9 @@
+Initial
+6	12
+6	12
+REPLACE simple
+6	10
+6	10
+ATTACH FROM
+6	10
+6	10
diff --git a/tests/queries/0_stateless/02731_replace_partition_from_temporary_table.sql b/tests/queries/0_stateless/02731_replace_partition_from_temporary_table.sql
new file mode 100644
index 00000000000..b2f31230dfb
--- /dev/null
+++ b/tests/queries/0_stateless/02731_replace_partition_from_temporary_table.sql
@@ -0,0 +1,48 @@
+-- Tags: no-replicated-database
+
+DROP TEMPORARY TABLE IF EXISTS src;
+DROP TABLE IF EXISTS dst;
+DROP TABLE IF EXISTS rdst;
+
+CREATE TEMPORARY TABLE src (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;
+CREATE TABLE dst (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;
+CREATE TABLE rdst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_alter_attach_00626_rdst', 'r1') PARTITION BY p ORDER BY k;
+
+SELECT 'Initial';
+INSERT INTO src VALUES (0, '0', 1);
+INSERT INTO src VALUES (1, '0', 1);
+INSERT INTO src VALUES (1, '1', 1);
+INSERT INTO src VALUES (2, '0', 1);
+INSERT INTO src VALUES (3, '0', 1);
+INSERT INTO src VALUES (3, '1', 1);
+
+INSERT INTO dst VALUES (0, '1', 2);
+INSERT INTO dst VALUES (1, '1', 2), (1, '2', 2);
+INSERT INTO dst VALUES (2, '1', 2);
+INSERT INTO dst VALUES (3, '1', 2), (3, '2', 2);
+
+INSERT INTO rdst VALUES (0, '1', 2);
+INSERT INTO rdst VALUES (1, '1', 2), (1, '2', 2);
+INSERT INTO rdst VALUES (2, '1', 2);
+INSERT INTO rdst VALUES (3, '1', 2), (3, '2', 2);
+
+SELECT count(), sum(d) FROM dst;
+SELECT count(), sum(d) FROM rdst;
+
+SELECT 'REPLACE simple';
+ALTER TABLE dst REPLACE PARTITION 1 FROM src;
+SELECT count(), sum(d) FROM dst;
+ALTER TABLE rdst REPLACE PARTITION 3 FROM src;
+SELECT count(), sum(d) FROM rdst;
+
+SELECT 'ATTACH FROM';
+ALTER TABLE dst DROP PARTITION 1;
+ALTER TABLE dst ATTACH PARTITION 1 FROM src;
+SELECT count(), sum(d) FROM dst;
+ALTER TABLE rdst DROP PARTITION 3;
+ALTER TABLE rdst ATTACH PARTITION 1 FROM src;
+SELECT count(), sum(d) FROM rdst;
+
+DROP TEMPORARY TABLE IF EXISTS src;
+DROP TABLE IF EXISTS dst;
+DROP TABLE IF EXISTS rdst;

From 0cf6b9f1459175388f7e2a58338a839004e0d6b8 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Wed, 10 May 2023 12:27:13 +0000
Subject: [PATCH 386/689] Inherit context from parent fiber

---
 contrib/boost                            |  2 +-
 contrib/boost-cmake/CMakeLists.txt       | 15 --------
 src/CMakeLists.txt                       |  3 --
 src/Common/AsyncTaskExecutor.cpp         | 12 +++----
 src/Common/AsyncTaskExecutor.h           | 31 ++++-------------
 src/Common/OpenTelemetryTraceContext.cpp | 44 ++++++++++++++++++++++--
 6 files changed, 55 insertions(+), 52 deletions(-)

diff --git a/contrib/boost b/contrib/boost
index d6c95434acb..1035c8bfcc9 160000
--- a/contrib/boost
+++ b/contrib/boost
@@ -1 +1 @@
-Subproject commit d6c95434acbb1a02d0b9de52bf4f37cac6c00328
+Subproject commit 1035c8bfcc9a3c1cfa7f6e827db94dae1ce1a43a
diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt
index cb0db5622a8..c9a759eab9c 100644
--- a/contrib/boost-cmake/CMakeLists.txt
+++ b/contrib/boost-cmake/CMakeLists.txt
@@ -161,21 +161,6 @@ elseif (SANITIZE STREQUAL "thread")
     target_compile_definitions(_boost_context PUBLIC BOOST_USE_TSAN)
 endif()
 
-# fiber
-
-set (SRCS_FIBER
-    "${LIBRARY_DIR}/libs/fiber/src/context.cpp"
-    "${LIBRARY_DIR}/libs/fiber/src/fiber.cpp"
-    "${LIBRARY_DIR}/libs/fiber/src/scheduler.cpp"
-    "${LIBRARY_DIR}/libs/fiber/src/waker.cpp"
-    "${LIBRARY_DIR}/libs/fiber/src/algo/round_robin.cpp"
-)
-
-add_library (_boost_fiber ${SRCS_FIBER})
-add_library (boost::fiber ALIAS _boost_fiber)
-target_include_directories (_boost_fiber PRIVATE ${LIBRARY_DIR})
-target_link_libraries(_boost_fiber PRIVATE _boost_context)
-
 # coroutine
 
 set (SRCS_COROUTINE
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index c69ac885154..76a67ade99c 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -547,9 +547,6 @@ endif ()
 target_link_libraries(clickhouse_common_io PUBLIC boost::context)
 dbms_target_link_libraries(PUBLIC boost::context)
 
-target_link_libraries(clickhouse_common_io PUBLIC boost::fiber)
-dbms_target_link_libraries(PUBLIC boost::fiber)
-
 if (ENABLE_NLP)
     dbms_target_link_libraries (PUBLIC ch_contrib::stemmer)
     dbms_target_link_libraries (PUBLIC ch_contrib::wnb)
diff --git a/src/Common/AsyncTaskExecutor.cpp b/src/Common/AsyncTaskExecutor.cpp
index d0c6454a849..68af535b22a 100644
--- a/src/Common/AsyncTaskExecutor.cpp
+++ b/src/Common/AsyncTaskExecutor.cpp
@@ -3,16 +3,16 @@
 namespace DB
 {
 
-thread_local const Fiber * current_fiber = nullptr;
+thread_local FiberInfo current_fiber_info;
 
 AsyncTaskExecutor::AsyncTaskExecutor(std::unique_ptr<AsyncTask> task_) : task(std::move(task_))
 {
     createFiber();
 }
 
-const Fiber * AsyncTaskExecutor::getCurrentFiber()
+FiberInfo AsyncTaskExecutor::getCurrentFiberInfo()
 {
-    return current_fiber;
+    return current_fiber_info;
 }
 
 void AsyncTaskExecutor::resume()
@@ -38,10 +38,10 @@ void AsyncTaskExecutor::resume()
 
 void AsyncTaskExecutor::resumeUnlocked()
 {
-    const auto * parent_fiber = current_fiber;
-    current_fiber = &fiber;
+    auto parent_fiber_info = current_fiber_info;
+    current_fiber_info = FiberInfo{&fiber, &parent_fiber_info};
     fiber = std::move(fiber).resume();
-    current_fiber = parent_fiber;
+    current_fiber_info = parent_fiber_info;
 }
 
 void AsyncTaskExecutor::cancel()
diff --git a/src/Common/AsyncTaskExecutor.h b/src/Common/AsyncTaskExecutor.h
index cf7cdc5ad82..1c2f758504a 100644
--- a/src/Common/AsyncTaskExecutor.h
+++ b/src/Common/AsyncTaskExecutor.h
@@ -24,6 +24,11 @@ enum class AsyncEventTimeoutType
 using AsyncCallback = std::function<void(int, Poco::Timespan, AsyncEventTimeoutType, const std::string &, uint32_t)>;
 using ResumeCallback = std::function<void()>;
 
+struct FiberInfo
+{
+    const Fiber * fiber = nullptr;
+    const FiberInfo * parent_fiber_info = nullptr;
+};
 
 /// Base class for a task that will be executed in a fiber.
 /// It has only one method - run, that takes 2 callbacks:
@@ -74,7 +79,7 @@ public:
         ERROR = 4,
     };
 #endif
-    static const Fiber * getCurrentFiber();
+    static FiberInfo getCurrentFiberInfo();
 
 protected:
     /// Method that is called in resume() before actual fiber resuming.
@@ -119,30 +124,6 @@ private:
     std::unique_ptr<AsyncTask> task;
 };
 
-/// Simple class for storing fiber local variables.
-template <typename T>
-class FiberLocalVariable
-{
-public:
-    T & operator*()
-    {
-        return get();
-    }
-
-    T * operator->()
-    {
-        return &get();
-    }
-
-private:
-    T & get()
-    {
-        return data[AsyncTaskExecutor::getCurrentFiber()];
-    }
-
-    std::unordered_map<const Fiber *, T> data;
-};
-
 String getSocketTimeoutExceededMessageByTimeoutType(AsyncEventTimeoutType type, Poco::Timespan timeout, const String & socket_description);
 
 }
diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp
index f25acc571d8..178efa33817 100644
--- a/src/Common/OpenTelemetryTraceContext.cpp
+++ b/src/Common/OpenTelemetryTraceContext.cpp
@@ -14,8 +14,48 @@ namespace DB
 namespace OpenTelemetry
 {
 
-/// This code can be executed inside fiber, we should use fiber local context.
-thread_local FiberLocalVariable<TracingContextOnThread> current_fiber_trace_context;
+/// This code can be executed inside several fibers in one thread,
+/// we should use fiber local tracing context.
+struct FiberLocalTracingContextOnThread
+{
+public:
+    FiberLocalTracingContextOnThread()
+    {
+        /// Initialize main context for this thread.
+        /// Contexts for fibers will inherit this main context.
+        data[nullptr] = TracingContextOnThread();
+    }
+
+    TracingContextOnThread & operator*()
+    {
+        return get();
+    }
+
+    TracingContextOnThread * operator->()
+    {
+        return &get();
+    }
+
+private:
+    TracingContextOnThread & get()
+    {
+        /// Get context for current fiber.
+        return getContextForFiber(AsyncTaskExecutor::getCurrentFiberInfo());
+    }
+
+    TracingContextOnThread & getContextForFiber(FiberInfo info)
+    {
+        auto it = data.find(info.fiber);
+        /// If it's the first request, we need to initialize context for the fiber using context from parent fiber.
+        if (it == data.end())
+            it = data.insert({info.fiber, getContextForFiber(*info.parent_fiber_info)}).first;
+        return it->second;
+    }
+
+    std::unordered_map<const Fiber *, TracingContextOnThread> data;
+};
+
+thread_local FiberLocalTracingContextOnThread current_fiber_trace_context;
 
 bool Span::addAttribute(std::string_view name, UInt64 value) noexcept
 {

From 10e6f5b59a8f19bfab4b24f76c29b2b9a5324555 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Wed, 10 May 2023 12:31:34 +0000
Subject: [PATCH 387/689] Fix indents

---
 src/Common/OpenTelemetryTraceContext.cpp | 26 ++++++++++++------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp
index 178efa33817..037ada88f80 100644
--- a/src/Common/OpenTelemetryTraceContext.cpp
+++ b/src/Common/OpenTelemetryTraceContext.cpp
@@ -175,7 +175,7 @@ SpanHolder::SpanHolder(std::string_view _operation_name, SpanKind _kind)
     }
 
     /// Set current span as parent of other spans created later on this thread.
-   current_fiber_trace_context->span_id = this->span_id;
+    current_fiber_trace_context->span_id = this->span_id;
 }
 
 void SpanHolder::finish() noexcept
@@ -189,7 +189,7 @@ void SpanHolder::finish() noexcept
 
     try
     {
-        auto log =current_fiber_trace_context->span_log.lock();
+        auto log = current_fiber_trace_context->span_log.lock();
 
         /// The log might be disabled, check it before use
         if (log)
@@ -312,7 +312,7 @@ void TracingContext::serialize(WriteBuffer & buf) const
 
 const TracingContextOnThread & CurrentContext()
 {
-    return*current_fiber_trace_context;
+    return *current_fiber_trace_context;
 }
 
 void TracingContextOnThread::reset() noexcept
@@ -347,15 +347,15 @@ TracingContextHolder::TracingContextHolder(
             /// So this branch ensures this class can be instantiated multiple times on one same thread safely.
             ///
             this->is_context_owner = false;
-            this->root_span.trace_id =current_fiber_trace_context->trace_id;
-            this->root_span.parent_span_id =current_fiber_trace_context->span_id;
+            this->root_span.trace_id = current_fiber_trace_context->trace_id;
+            this->root_span.parent_span_id = current_fiber_trace_context->span_id;
             this->root_span.span_id = thread_local_rng();
             this->root_span.operation_name = _operation_name;
             this->root_span.start_time_us
                 = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
 
             /// Set the root span as parent of other spans created on current thread
-           current_fiber_trace_context->span_id = this->root_span.span_id;
+            current_fiber_trace_context->span_id = this->root_span.span_id;
             return;
         }
 
@@ -399,10 +399,10 @@ TracingContextHolder::TracingContextHolder(
     }
 
     /// Set up trace context on current thread only when the root span is successfully initialized.
-   *current_fiber_trace_context = _parent_trace_context;
-   current_fiber_trace_context->span_id = this->root_span.span_id;
-   current_fiber_trace_context->trace_flags = TRACE_FLAG_SAMPLED;
-   current_fiber_trace_context->span_log = _span_log;
+    *current_fiber_trace_context = _parent_trace_context;
+    current_fiber_trace_context->span_id = this->root_span.span_id;
+    current_fiber_trace_context->trace_flags = TRACE_FLAG_SAMPLED;
+    current_fiber_trace_context->span_log = _span_log;
 }
 
 TracingContextHolder::~TracingContextHolder()
@@ -414,7 +414,7 @@ TracingContextHolder::~TracingContextHolder()
 
     try
     {
-        auto shared_span_log =current_fiber_trace_context->span_log.lock();
+        auto shared_span_log = current_fiber_trace_context->span_log.lock();
         if (shared_span_log)
         {
             try
@@ -445,11 +445,11 @@ TracingContextHolder::~TracingContextHolder()
     if (this->is_context_owner)
     {
         /// Clear the context on current thread
-       current_fiber_trace_context->reset();
+        current_fiber_trace_context->reset();
     }
     else
     {
-       current_fiber_trace_context->span_id = this->root_span.parent_span_id;
+        current_fiber_trace_context->span_id = this->root_span.parent_span_id;
     }
 }
 

From 95bcccaf844e5fda606cbbd4f97a212ee4ed7dc2 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Wed, 10 May 2023 14:42:49 +0200
Subject: [PATCH 388/689] Do not allocate own buffer in
 CachedOnDiskReadBufferFromFile when `use_external_buffer == true`

---
 src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
index 00d23183f6a..1bbf5356924 100644
--- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
+++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
@@ -50,7 +50,7 @@ CachedOnDiskReadBufferFromFile::CachedOnDiskReadBufferFromFile(
     bool use_external_buffer_,
     std::optional<size_t> read_until_position_,
     std::shared_ptr<FilesystemCacheLog> cache_log_)
-    : ReadBufferFromFileBase(settings_.remote_fs_buffer_size, nullptr, 0, file_size_)
+    : ReadBufferFromFileBase(use_external_buffer_ ? 0 : settings_.remote_fs_buffer_size, nullptr, 0, file_size_)
 #ifndef NDEBUG
     , log(&Poco::Logger::get("CachedOnDiskReadBufferFromFile(" + source_file_path_ + ")"))
 #else
@@ -151,10 +151,8 @@ CachedOnDiskReadBufferFromFile::getCacheReadBuffer(const FileSegment & file_segm
     /// Do not allow to use asynchronous version of LocalFSReadMethod.
     local_read_settings.local_fs_method = LocalFSReadMethod::pread;
 
-    // The buffer will unnecessarily allocate a Memory of size local_fs_buffer_size, which will then
-    // most likely be unused because we're swap()ping our own internal_buffer into
-    // implementation_buffer before each read. But we can't just set local_fs_buffer_size = 0 here
-    // because some buffer implementations actually use that memory (e.g. for prefetching).
+    if (use_external_buffer)
+        local_read_settings.local_fs_buffer_size = 0;
 
     auto buf = createReadBufferFromFileBase(path, local_read_settings);
 

From a7aec49fbbcab364bc443787ec2c11d2bf52762d Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Wed, 10 May 2023 12:43:29 +0000
Subject: [PATCH 389/689] Fix indents

---
 src/Common/OpenTelemetryTraceContext.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp
index 037ada88f80..8cf4879c1e2 100644
--- a/src/Common/OpenTelemetryTraceContext.cpp
+++ b/src/Common/OpenTelemetryTraceContext.cpp
@@ -155,8 +155,8 @@ SpanHolder::SpanHolder(std::string_view _operation_name, SpanKind _kind)
     /// Use try-catch to make sure the ctor is exception safe.
     try
     {
-        this->trace_id =current_fiber_trace_context->trace_id;
-        this->parent_span_id =current_fiber_trace_context->span_id;
+        this->trace_id = current_fiber_trace_context->trace_id;
+        this->parent_span_id = current_fiber_trace_context->span_id;
         this->span_id = thread_local_rng(); // create a new id for this span
         this->operation_name = _operation_name;
         this->kind = _kind;

From 7770bc27568501748e9389ca48efa94c7909d1b1 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 10 May 2023 15:18:02 +0200
Subject: [PATCH 390/689] update submodule

---
 .gitmodules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitmodules b/.gitmodules
index f7cadb3d16c..1aa01509c3f 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -340,7 +340,7 @@
 	url = https://github.com/axboe/liburing
 [submodule "contrib/libfiu"]
 	path = contrib/libfiu
-	url = https://github.com/hanfei1991/libfiu.git
+	url = https://github.com/ClickHouse/libfiu.git
 [submodule "contrib/isa-l"]
 	path = contrib/isa-l
 	url = https://github.com/ClickHouse/isa-l.git

From 4c0abc84d4062d7c0d0f42dad31c11e5a366ea9d Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <440544+davenger@users.noreply.github.com>
Date: Wed, 10 May 2023 15:40:32 +0200
Subject: [PATCH 391/689] Repro

---
 ..._lightweight_delete_and_object_column.reference |  1 +
 .../02352_lightweight_delete_and_object_column.sql | 14 ++++++++++++++
 2 files changed, 15 insertions(+)
 create mode 100644 tests/queries/0_stateless/02352_lightweight_delete_and_object_column.reference
 create mode 100644 tests/queries/0_stateless/02352_lightweight_delete_and_object_column.sql

diff --git a/tests/queries/0_stateless/02352_lightweight_delete_and_object_column.reference b/tests/queries/0_stateless/02352_lightweight_delete_and_object_column.reference
new file mode 100644
index 00000000000..3ad5abd03ae
--- /dev/null
+++ b/tests/queries/0_stateless/02352_lightweight_delete_and_object_column.reference
@@ -0,0 +1 @@
+99
diff --git a/tests/queries/0_stateless/02352_lightweight_delete_and_object_column.sql b/tests/queries/0_stateless/02352_lightweight_delete_and_object_column.sql
new file mode 100644
index 00000000000..cd29d0ac445
--- /dev/null
+++ b/tests/queries/0_stateless/02352_lightweight_delete_and_object_column.sql
@@ -0,0 +1,14 @@
+DROP TABLE IF EXISTS t_obj SYNC;
+
+SET allow_experimental_object_type=1;
+
+CREATE TABLE t_obj(id Int32, name Object('json')) ENGINE = MergeTree() ORDER BY id;
+
+INSERT INTO t_obj select number, '{"a" : "' || toString(number) || '"}' FROM numbers(100);
+
+DELETE FROM t_obj WHERE id = 10;
+
+SELECT COUNT() FROM t_obj;
+
+DROP TABLE t_obj SYNC;
+

From 88b07d121cc98ce3fe3e9e00150ca766d61e7812 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 10 May 2023 16:00:04 +0200
Subject: [PATCH 392/689] Update vectorscan library with fixes for MSan

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .gitmodules        | 5 ++++-
 contrib/vectorscan | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index 56aeea2bfe8..b1ae25a531f 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -267,7 +267,10 @@
 	url = https://github.com/ClickHouse/nats.c
 [submodule "contrib/vectorscan"]
 	path = contrib/vectorscan
-	url = https://github.com/VectorCamp/vectorscan
+	# FIXME: update once upstream fixes will be merged:
+	# - https://github.com/VectorCamp/vectorscan/pull/148
+	# - https://github.com/VectorCamp/vectorscan/pull/149
+	url = https://github.com/azat-ch/vectorscan
 [submodule "contrib/c-ares"]
 	path = contrib/c-ares
 	url = https://github.com/ClickHouse/c-ares
diff --git a/contrib/vectorscan b/contrib/vectorscan
index b4bba94b1a2..aaca65aa210 160000
--- a/contrib/vectorscan
+++ b/contrib/vectorscan
@@ -1 +1 @@
-Subproject commit b4bba94b1a250603b0b198e0394946e32f6c3f30
+Subproject commit aaca65aa210ce3ec91bd2b249c4d59e55e80a869

From e6f4e5cd10e2f2e07a7c2e162c0b6e7a02f57acc Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <440544+davenger@users.noreply.github.com>
Date: Wed, 10 May 2023 16:10:04 +0200
Subject: [PATCH 393/689] Ignore LWD column in checkPartDynamicColumns

---
 src/Storages/MergeTree/MergeTreeData.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index a2a3348eb42..533ae2cadd5 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -3608,6 +3608,9 @@ void MergeTreeData::checkPartDynamicColumns(MutableDataPartPtr & part, DataParts
     const auto & part_columns = part->getColumns();
     for (const auto & part_column : part_columns)
     {
+        if (part_column.name == LightweightDeleteDescription::FILTER_COLUMN.name)
+            continue;
+
         auto storage_column = columns.getPhysical(part_column.name);
         if (!storage_column.type->hasDynamicSubcolumns())
             continue;

From 2273884ded830d1bac328c58724128d61579fb91 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 10 May 2023 17:42:08 +0200
Subject: [PATCH 394/689] address comments

---
 .../ExternalDictionariesLoader.cpp            | 32 +++----------------
 src/Interpreters/ExternalDictionariesLoader.h |  4 +--
 .../TableFunctionDictionary.cpp               | 31 +++++++++++++++++-
 3 files changed, 35 insertions(+), 32 deletions(-)

diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp
index 2129b3bbec6..21613806788 100644
--- a/src/Interpreters/ExternalDictionariesLoader.cpp
+++ b/src/Interpreters/ExternalDictionariesLoader.cpp
@@ -4,11 +4,11 @@
 #include <Dictionaries/DictionaryFactory.h>
 #include <Dictionaries/DictionaryStructure.h>
 #include <Databases/IDatabase.h>
-#include <DataTypes/DataTypeArray.h>
-#include <DataTypes/DataTypeString.h>
-#include <DataTypes/DataTypesNumber.h>
 #include <Storages/IStorage.h>
-#include <Storages/StorageDictionary.h>
+#include <Common/logger_useful.h>
+
+#include "Storages/ColumnsDescription.h"
+#include "config.h"
 
 #if USE_MYSQL
 #   include <mysqlxx/PoolFactory.h>
@@ -69,30 +69,6 @@ void ExternalDictionariesLoader::reloadDictionary(const std::string & dictionary
     loadOrReload(resolved_dictionary_name);
 }
 
-ColumnsDescription ExternalDictionariesLoader::getActualTableStructure(const std::string & dictionary_name, ContextPtr query_context) const
-{
-    /// if it is regexp tree dictionary, we have a special table structure.
-    std::string resolved_name = resolveDictionaryName(dictionary_name, query_context->getCurrentDatabase());
-    auto load_result = load(resolved_name);
-    if (load_result)
-    {
-        const auto dictionary = std::static_pointer_cast<const IDictionary>(load_result);
-        if (dictionary->getTypeName() == "RegExpTree")
-        {
-            return ColumnsDescription(NamesAndTypesList({
-                {"id", std::make_shared<DataTypeUInt64>()},
-                {"parent_id", std::make_shared<DataTypeUInt64>()},
-                {"regexp", std::make_shared<DataTypeString>()},
-                {"keys", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
-                {"values", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())}
-            }));
-        }
-    }
-    /// otherwise, we get table structure by dictionary structure.
-    auto dictionary_structure = getDictionaryStructure(dictionary_name, query_context);
-    return ColumnsDescription(StorageDictionary::getNamesAndTypes(dictionary_structure));
-}
-
 DictionaryStructure ExternalDictionariesLoader::getDictionaryStructure(const std::string & dictionary_name, ContextPtr query_context) const
 {
     std::string resolved_name = resolveDictionaryName(dictionary_name, query_context->getCurrentDatabase());
diff --git a/src/Interpreters/ExternalDictionariesLoader.h b/src/Interpreters/ExternalDictionariesLoader.h
index 717aefddd8f..5f27d695c8b 100644
--- a/src/Interpreters/ExternalDictionariesLoader.h
+++ b/src/Interpreters/ExternalDictionariesLoader.h
@@ -4,7 +4,6 @@
 #include <Interpreters/Context_fwd.h>
 #include <Interpreters/ExternalLoader.h>
 #include <Common/quoteString.h>
-#include <Storages/ColumnsDescription.h>
 
 #include <memory>
 
@@ -30,8 +29,6 @@ public:
 
     QualifiedTableName qualifyDictionaryNameWithDatabase(const std::string & dictionary_name, ContextPtr context) const;
 
-    ColumnsDescription getActualTableStructure(const std::string & dictionary_name, ContextPtr context) const;
-
     DictionaryStructure getDictionaryStructure(const std::string & dictionary_name, ContextPtr context) const;
 
     void assertDictionaryStructureExists(const std::string & dictionary_name, ContextPtr context) const;
@@ -53,6 +50,7 @@ protected:
 
     friend class StorageSystemDictionaries;
     friend class DatabaseDictionary;
+    friend class TableFunctionDictionary;
 };
 
 }
diff --git a/src/TableFunctions/TableFunctionDictionary.cpp b/src/TableFunctions/TableFunctionDictionary.cpp
index 5231e39dd8c..90db9550a72 100644
--- a/src/TableFunctions/TableFunctionDictionary.cpp
+++ b/src/TableFunctions/TableFunctionDictionary.cpp
@@ -2,6 +2,10 @@
 
 #include <Parsers/ASTLiteral.h>
 
+#include <DataTypes/DataTypeArray.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypesNumber.h>
+
 #include <Interpreters/Context.h>
 #include <Interpreters/ExternalDictionariesLoader.h>
 #include <Interpreters/evaluateConstantExpression.h>
@@ -42,8 +46,33 @@ void TableFunctionDictionary::parseArguments(const ASTPtr & ast_function, Contex
 ColumnsDescription TableFunctionDictionary::getActualTableStructure(ContextPtr context) const
 {
     const ExternalDictionariesLoader & external_loader = context->getExternalDictionariesLoader();
+    std::string resolved_name = external_loader.resolveDictionaryName(dictionary_name, context->getCurrentDatabase());
+    auto load_result = external_loader.load(resolved_name);
+    if (load_result)
+    {
+        /// for regexp tree dictionary, the table structure will be different with dictionary structure. it is:
+        /// - id. identifier of the tree node
+        /// - parent_id.
+        /// - regexp. the regular expression
+        /// - keys. the names of attributions of dictionary structure
+        /// - values. the values of each attribution
+        const auto dictionary = std::static_pointer_cast<const IDictionary>(load_result);
+        if (dictionary->getTypeName() == "RegExpTree")
+        {
+            return ColumnsDescription(NamesAndTypesList({
+                {"id", std::make_shared<DataTypeUInt64>()},
+                {"parent_id", std::make_shared<DataTypeUInt64>()},
+                {"regexp", std::make_shared<DataTypeString>()},
+                {"keys", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
+                {"values", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())}
+            }));
+        }
+    }
+
+    /// otherwise, we get table structure by dictionary structure.
+    auto dictionary_structure = external_loader.getDictionaryStructure(dictionary_name, context);
+    return ColumnsDescription(StorageDictionary::getNamesAndTypes(dictionary_structure));
 
-    return external_loader.getActualTableStructure(dictionary_name, context);
 }
 
 StoragePtr TableFunctionDictionary::executeImpl(

From 4be0372eb472b0abe77c49cd9f785adc2c1be46b Mon Sep 17 00:00:00 2001
From: Rich Raposa <richraposa@gmail.com>
Date: Wed, 10 May 2023 09:48:33 -0600
Subject: [PATCH 395/689] Update settings.md

Removed the mention that async_insert is  only useful for HTTP
---
 docs/en/operations/settings/settings.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index ad7ef52412a..a17313930c4 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -1630,7 +1630,7 @@ For not replicated tables see [non_replicated_deduplication_window](merge-tree-s
 
 ### async_insert {#async-insert}
 
-Enables or disables asynchronous inserts. This makes sense only for insertion over HTTP protocol. Note that deduplication isn't working for such inserts.
+Enables or disables asynchronous inserts. Note that deduplication isn't working for such inserts.
 
 If enabled, the data is combined into batches before the insertion into tables, so it is possible to do small and frequent insertions into ClickHouse (up to 15000 queries per second) without buffer tables.
 

From b80d5943977b727ea2597a4a198bd623b0c7960a Mon Sep 17 00:00:00 2001
From: Tyler Hannan <tyler@clickhouse.com>
Date: Wed, 10 May 2023 18:55:23 +0200
Subject: [PATCH 396/689] Update README.md

---
 README.md | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index c82c64cfd22..7a364795a02 100644
--- a/README.md
+++ b/README.md
@@ -21,11 +21,17 @@ curl https://clickhouse.com/ | sh
 * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
 
 ## Upcoming Events
-* [**ClickHouse Spring Meetup in Manhattan**](https://www.meetup.com/clickhouse-new-york-user-group/events/292517734) - April 26 - It's spring, and it's time to meet again in the city! Talks include: "Building a domain specific query language on top of Clickhouse", "A Galaxy of Information", "Our Journey to ClickHouse Cloud from Redshift", and a ClickHouse update!
+
 * [**v23.4 Release Webinar**](https://clickhouse.com/company/events/v23-4-release-webinar?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-04) - April 26 - 23.4 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
-* [**ClickHouse Meetup in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/292892466) - May 16 - Save the date! ClickHouse is coming back to Berlin. We’re excited to announce an upcoming ClickHouse Meetup that you won’t want to miss. Join us as we gather together to discuss the latest in the world of ClickHouse and share user stories.
+* [**ClickHouse Meetup in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/292892466) - May 16 
+* [**ClickHouse Meetup in Barcelona**](https://www.meetup.com/clickhouse-barcelona-user-group/events/292892669) - May 25 
+* [**ClickHouse Meetup in London**](https://www.meetup.com/clickhouse-london-user-group/events/292892824) - May 25 
+* [**ClickHouse Meetup in San Francisco**](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/293426725/) - Jun 7 
+* [**ClickHouse Meetup in Stockholm**](https://www.meetup.com/clickhouse-berlin-user-group/events/292892466) - Jun 13 
+
+Also, keep an eye out for upcoming meetups in Amsterdam, Boston, NYC, Beijing, and Toronto. Somewhere else you want us to be? Please feel free to reach out to tyler <at> clickhouse <dot> com.
 
 ## Recent Recordings
 * **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
-* **Recording available**: [**v23.3 Release Webinar**](https://www.youtube.com/watch?v=ISaGUjvBNao) UNDROP TABLE, server settings introspection, nested dynamic disks, MySQL compatibility, parseDate Time, Lightweight Deletes, Parallel Replicas, integrations updates, and so much more! Watch it now!
+* **Recording available**: [**v23.4 Release Webinar**]([https://www.youtube.com/watch?v=ISaGUjvBNao](https://www.youtube.com/watch?v=4rrf6bk_mOg)) UNDROP TABLE, server settings introspection, nested dynamic disks, MySQL compatibility, parseDate Time, Lightweight Deletes, Parallel Replicas, integrations updates, and so much more! Watch it now!
 * **All release webinar recordings**: [YouTube playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3jAlSy1JxyP8zluvXaN3nxU)

From 35c58f6255aead7ee7a796309f2bb4453fc22496 Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Wed, 10 May 2023 19:06:44 +0200
Subject: [PATCH 397/689] Update test

---
 .../02497_storage_file_reader_selection.sh    | 22 +++++--------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/tests/queries/0_stateless/02497_storage_file_reader_selection.sh b/tests/queries/0_stateless/02497_storage_file_reader_selection.sh
index 4d9336bc1a0..76f87633446 100755
--- a/tests/queries/0_stateless/02497_storage_file_reader_selection.sh
+++ b/tests/queries/0_stateless/02497_storage_file_reader_selection.sh
@@ -4,24 +4,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-USER_FILES_PATH=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}')
-
-DATA_FILE=$USER_FILES_PATH/test_02497_storage_file_reader.data
+DATA_FILE=test_02497_$CLICKHOUSE_TEST_UNIQUE_NAME.tsv
 echo -e 'key\nfoo\nbar' > $DATA_FILE
 
 QUERY_ID=$RANDOM
-$CLICKHOUSE_CLIENT -q "SELECT * FROM file('test_02497_storage_file_reader.data', 'TSV', 's String')" \
-  --query_id $QUERY_ID
+$CLICKHOUSE_LOCAL --storage_file_read_method=mmap --print-profile-events -q "SELECT * FROM file($DATA_FILE) FORMAT Null" 2>&1 | grep -F -c "CreatedReadBufferMMap"
+$CLICKHOUSE_LOCAL --storage_file_read_method=mmap --print-profile-events -q "SELECT * FROM file($DATA_FILE) FORMAT Null" 2>&1 | grep -F -c "CreatedReadBufferOrdinary"
 
-$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS"
-$CLICKHOUSE_CLIENT -q "SELECT sum(ProfileEvents['CreatedReadBufferMMap']) FROM system.query_log WHERE query_id='$QUERY_ID'"
-$CLICKHOUSE_CLIENT -q "SELECT sum(ProfileEvents['CreatedReadBufferOrdinary']) FROM system.query_log WHERE query_id='$QUERY_ID'"
+$CLICKHOUSE_LOCAL --storage_file_read_method=pread --print-profile-events -q "SELECT * FROM file($DATA_FILE) FORMAT Null" 2>&1 | grep -F -c "CreatedReadBufferMMap"
+$CLICKHOUSE_LOCAL --storage_file_read_method=pread --print-profile-events -q "SELECT * FROM file($DATA_FILE) FORMAT Null" 2>&1 | grep -F -c "CreatedReadBufferOrdinary"
 
-QUERY_ID=$RANDOM
-$CLICKHOUSE_CLIENT -q "SELECT * FROM file('test_02497_storage_file_reader.data', 'TSV', 's String')" \
-  --query_id $QUERY_ID \
-  --storage_file_read_method=pread
-
-$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS"
-$CLICKHOUSE_CLIENT -q "SELECT sum(ProfileEvents['CreatedReadBufferMMap']) FROM system.query_log WHERE query_id='$QUERY_ID'"
-$CLICKHOUSE_CLIENT -q "SELECT sum(ProfileEvents['CreatedReadBufferOrdinary']) FROM system.query_log WHERE query_id='$QUERY_ID'"
+rm $DATA_FILE

From 36a8b3acf18ce35ebc38e6f51effc84daec1d2d7 Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Wed, 10 May 2023 19:07:01 +0200
Subject: [PATCH 398/689] Update test reference

---
 .../02497_storage_file_reader_selection.reference           | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tests/queries/0_stateless/02497_storage_file_reader_selection.reference b/tests/queries/0_stateless/02497_storage_file_reader_selection.reference
index 8da37e4219c..680eb502aaa 100644
--- a/tests/queries/0_stateless/02497_storage_file_reader_selection.reference
+++ b/tests/queries/0_stateless/02497_storage_file_reader_selection.reference
@@ -1,10 +1,4 @@
-key
-foo
-bar
 1
 0
-key
-foo
-bar
 0
 1

From dde535797e39d09bba25c429314d47aa1d127ca3 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 10 May 2023 17:16:06 +0200
Subject: [PATCH 399/689] Update libfiu to fix the build

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 contrib/libfiu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/libfiu b/contrib/libfiu
index 7ae4328050c..b85edbde4cf 160000
--- a/contrib/libfiu
+++ b/contrib/libfiu
@@ -1 +1 @@
-Subproject commit 7ae4328050ccad8867a05a37af941886f717b6fc
+Subproject commit b85edbde4cf974b1b40d27828a56f0505f4e2ee5

From 374dbd9c3951263ea111a5fe9e262e807fd39ee7 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Wed, 10 May 2023 17:25:29 +0000
Subject: [PATCH 400/689] Fix msan issue in randomStringUTF8(<uneven number>)

---
 src/Common/Volnitsky.h                        |  6 +-
 src/Common/format.h                           | 13 ++--
 src/Functions/randomStringUTF8.cpp            | 61 +++++++++----------
 .../01278_random_string_utf8.reference        |  1 +
 .../0_stateless/01278_random_string_utf8.sql  |  1 +
 5 files changed, 39 insertions(+), 43 deletions(-)

diff --git a/src/Common/Volnitsky.h b/src/Common/Volnitsky.h
index a0fce5531cd..64c5bf4c0d3 100644
--- a/src/Common/Volnitsky.h
+++ b/src/Common/Volnitsky.h
@@ -386,8 +386,6 @@ protected:
     FallbackSearcher fallback_searcher;
 
 public:
-    using Searcher = FallbackSearcher;
-
     /** haystack_size_hint - the expected total size of the haystack for `search` calls. Optional (zero means unspecified).
       * If you specify it small enough, the fallback algorithm will be used,
       *  since it is considered that it's useless to waste time initializing the hash table.
@@ -729,7 +727,7 @@ public:
 
 
 using Volnitsky = VolnitskyBase<true, true, ASCIICaseSensitiveStringSearcher>;
-using VolnitskyUTF8 = VolnitskyBase<true, false, ASCIICaseSensitiveStringSearcher>; /// exactly same as Volnitsky
+using VolnitskyUTF8 = VolnitskyBase<true, false, UTF8CaseSensitiveStringSearcher>;
 using VolnitskyCaseInsensitive = VolnitskyBase<false, true, ASCIICaseInsensitiveStringSearcher>; /// ignores non-ASCII bytes
 using VolnitskyCaseInsensitiveUTF8 = VolnitskyBase<false, false, UTF8CaseInsensitiveStringSearcher>;
 
@@ -737,7 +735,7 @@ using VolnitskyCaseSensitiveToken = VolnitskyBase<true, true, ASCIICaseSensitive
 using VolnitskyCaseInsensitiveToken = VolnitskyBase<false, true, ASCIICaseInsensitiveTokenSearcher>;
 
 using MultiVolnitsky = MultiVolnitskyBase<true, true, ASCIICaseSensitiveStringSearcher>;
-using MultiVolnitskyUTF8 = MultiVolnitskyBase<true, false, ASCIICaseSensitiveStringSearcher>;
+using MultiVolnitskyUTF8 = MultiVolnitskyBase<true, false, UTF8CaseSensitiveStringSearcher>;
 using MultiVolnitskyCaseInsensitive = MultiVolnitskyBase<false, true, ASCIICaseInsensitiveStringSearcher>;
 using MultiVolnitskyCaseInsensitiveUTF8 = MultiVolnitskyBase<false, false, UTF8CaseInsensitiveStringSearcher>;
 
diff --git a/src/Common/format.h b/src/Common/format.h
index 1a02a064c09..b72c4f15125 100644
--- a/src/Common/format.h
+++ b/src/Common/format.h
@@ -17,18 +17,18 @@ namespace Format
 {
     using IndexPositions = PODArrayWithStackMemory<UInt64, 64>;
 
-    static inline void parseNumber(const String & description, UInt64 l, UInt64 r, UInt64 & res, UInt64 argument_number)
+    static inline UInt64 parseNumber(const String & description, UInt64 l, UInt64 r, UInt64 argument_number)
     {
-        res = 0;
+        UInt64 res = 0;
         for (UInt64 pos = l; pos < r; ++pos)
         {
             if (!isNumericASCII(description[pos]))
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not a number in curly braces at position {}", std::to_string(pos));
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not a number in curly braces at position {}", pos);
             res = res * 10 + description[pos] - '0';
             if (res >= argument_number)
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Too big number for arguments, must be at most {}",
-                    argument_number - 1);
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Too big number for arguments, must be at most {}", argument_number - 1);
         }
+        return res;
     }
 
     static inline void init(
@@ -132,8 +132,7 @@ namespace Format
                         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot switch from automatic field numbering to manual field specification");
                     is_plain_numbering = false;
 
-                    UInt64 arg;
-                    parseNumber(pattern, last_open, i, arg, argument_number);
+                    UInt64 arg = parseNumber(pattern, last_open, i, argument_number);
 
                     if (arg >= argument_number)
                         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Argument is too big for formatting. Note that indexing starts from zero");
diff --git a/src/Functions/randomStringUTF8.cpp b/src/Functions/randomStringUTF8.cpp
index 14860338d4a..6aabb81c661 100644
--- a/src/Functions/randomStringUTF8.cpp
+++ b/src/Functions/randomStringUTF8.cpp
@@ -61,26 +61,25 @@ public:
 
         offsets_to.resize(input_rows_count);
 
-        const IColumn & length_column = *arguments[0].column;
-        size_t summary_utf8_len = 0;
+        const IColumn & col_length = *arguments[0].column;
+        size_t total_codepoints = 0;
         for (size_t row_num = 0; row_num < input_rows_count; ++row_num)
         {
-            size_t utf8_len = length_column.getUInt(row_num);
-            summary_utf8_len += utf8_len;
+            size_t codepoints = col_length.getUInt(row_num);
+            total_codepoints += codepoints;
         }
 
         /* As we generate only assigned planes, the mathematical expectation of the number of bytes
          * per generated code point ~= 3.85. So, reserving for coefficient 4 will not be an overhead
          */
 
-        if (summary_utf8_len > (1 << 29))
+        if (total_codepoints > (1 << 29))
             throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "Too large string size in function {}", getName());
 
-        size_t size_in_bytes_with_margin = summary_utf8_len * 4 + input_rows_count;
-        data_to.resize(size_in_bytes_with_margin);
-        pcg64_fast rng(randomSeed()); // TODO It is inefficient. We should use SIMD PRNG instead.
+        size_t max_byte_size = total_codepoints * 4 + input_rows_count;
+        data_to.resize(max_byte_size);
 
-        const auto generate_code_point = [](UInt32 rand) -> UInt32 {
+        const auto generate_code_point = [](UInt32 rand) {
             /// We want to generate number in [0x0, 0x70000) and shift it if need
 
             /// Generate highest byte in [0, 6]
@@ -104,43 +103,41 @@ public:
             return code_point;
         };
 
+        pcg64_fast rng(randomSeed());
         IColumn::Offset offset = 0;
+
         for (size_t row_num = 0; row_num < input_rows_count; ++row_num)
         {
-            size_t utf8_len = length_column.getUInt(row_num);
+            size_t codepoints = col_length.getUInt(row_num);
             auto * pos = data_to.data() + offset;
 
-            size_t last_writen_bytes = 0;
-            size_t i = 0;
-            for (; i < utf8_len; i += 2)
+            for (size_t i = 0; i < codepoints; i +=2)
             {
-                UInt64 rand = rng();
+                UInt64 rand = rng(); /// that's the bottleneck
 
                 UInt32 code_point1 = generate_code_point(static_cast<UInt32>(rand));
-                UInt32 code_point2 = generate_code_point(static_cast<UInt32>(rand >> 32u));
 
-                /// We have padding in column buffers that we can overwrite.
-                size_t length1 = UTF8::convertCodePointToUTF8(code_point1, pos, sizeof(int));
-                assert(length1 <= 4);
-                pos += length1;
+                size_t bytes1 = UTF8::convertCodePointToUTF8(code_point1, pos, 4);
+                chassert(bytes1 <= 4);
+                pos += bytes1;
 
-                size_t length2 = UTF8::convertCodePointToUTF8(code_point2, pos, sizeof(int));
-                assert(length2 <= 4);
-                last_writen_bytes = length2;
-                pos += last_writen_bytes;
-            }
-            offset = pos - data_to.data() + 1;
-            if (i > utf8_len)
-            {
-                offset -= last_writen_bytes;
+                if (i + 1 != codepoints)
+                {
+                    UInt32 code_point2 = generate_code_point(static_cast<UInt32>(rand >> 32u));
+                    size_t bytes2 = UTF8::convertCodePointToUTF8(code_point2, pos, 4);
+                    chassert(bytes2 <= 4);
+                    pos += bytes2;
+                }
             }
+
+            *pos = 0;
+            ++pos;
+
+            offset = pos - data_to.data();
             offsets_to[row_num] = offset;
         }
 
-        /// Put zero bytes in between.
-        auto * pos = data_to.data();
-        for (size_t row_num = 0; row_num < input_rows_count; ++row_num)
-            pos[offsets_to[row_num] - 1] = 0;
+        data_to.resize(offset);
 
         return col_to;
     }
diff --git a/tests/queries/0_stateless/01278_random_string_utf8.reference b/tests/queries/0_stateless/01278_random_string_utf8.reference
index 36ae0ace76a..45efb26db75 100644
--- a/tests/queries/0_stateless/01278_random_string_utf8.reference
+++ b/tests/queries/0_stateless/01278_random_string_utf8.reference
@@ -2,3 +2,4 @@
 String
 1
 
+99
diff --git a/tests/queries/0_stateless/01278_random_string_utf8.sql b/tests/queries/0_stateless/01278_random_string_utf8.sql
index f2c0a6c41c7..76349d9d814 100644
--- a/tests/queries/0_stateless/01278_random_string_utf8.sql
+++ b/tests/queries/0_stateless/01278_random_string_utf8.sql
@@ -3,3 +3,4 @@ SELECT lengthUTF8(randomStringUTF8(100));
 SELECT toTypeName(randomStringUTF8(10));
 SELECT isValidUTF8(randomStringUTF8(100000));
 SELECT randomStringUTF8(0);
+SELECT lengthUTF8(lowerUTF8(randomStringUTF8(99))); -- bug #49672: msan assert

From f2ad1122a17d9a235f34e531a1b113a77d3c21aa Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Wed, 10 May 2023 17:30:30 +0000
Subject: [PATCH 401/689] fix convertation

---
 programs/server/Server.cpp                | 8 ++++----
 src/Bridge/IBridge.cpp                    | 8 ++++----
 src/Server/KeeperTCPHandler.cpp           | 6 +++---
 src/Server/TCPHandler.h                   | 4 ++--
 src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 4 ++--
 src/Storages/StorageDistributed.cpp       | 4 ++--
 6 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 3f415462178..05fadd33555 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -1370,8 +1370,8 @@ try
                 {
                     Poco::Net::ServerSocket socket;
                     auto address = socketBindListen(config(), socket, listen_host, port);
-                    socket.setReceiveTimeout(config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC));
-                    socket.setSendTimeout(config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC));
+                    socket.setReceiveTimeout(Poco::Timespan(config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0));
+                    socket.setSendTimeout(Poco::Timespan(config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0));
                     return ProtocolServerAdapter(
                         listen_host,
                         port_name,
@@ -1393,8 +1393,8 @@ try
 #if USE_SSL
                     Poco::Net::SecureServerSocket socket;
                     auto address = socketBindListen(config(), socket, listen_host, port, /* secure = */ true);
-                    socket.setReceiveTimeout(config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC));
-                    socket.setSendTimeout(config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC));
+                    socket.setReceiveTimeout(Poco::Timespan(config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0));
+                    socket.setSendTimeout(Poco::Timespan(config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0));
                     return ProtocolServerAdapter(
                         listen_host,
                         secure_port_name,
diff --git a/src/Bridge/IBridge.cpp b/src/Bridge/IBridge.cpp
index 4ba53fd4435..3c84e22478a 100644
--- a/src/Bridge/IBridge.cpp
+++ b/src/Bridge/IBridge.cpp
@@ -214,14 +214,14 @@ int IBridge::main(const std::vector<std::string> & /*args*/)
 
     Poco::Net::ServerSocket socket;
     auto address = socketBindListen(socket, hostname, port, log);
-    socket.setReceiveTimeout(http_timeout);
-    socket.setSendTimeout(http_timeout);
+    socket.setReceiveTimeout(Poco::Timespan(http_timeout, 0));
+    socket.setSendTimeout(Poco::Timespan(http_timeout, 0));
 
     Poco::ThreadPool server_pool(3, max_server_connections);
 
     Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams;
-    http_params->setTimeout(http_timeout);
-    http_params->setKeepAliveTimeout(keep_alive_timeout);
+    http_params->setTimeout(Poco::Timespan(http_timeout, 0));
+    http_params->setKeepAliveTimeout(Poco::Timespan(keep_alive_timeout, 0));
 
     auto shared_context = Context::createShared();
     auto context = Context::createGlobal(shared_context.get());
diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp
index f9e11062906..e3edc281e83 100644
--- a/src/Server/KeeperTCPHandler.cpp
+++ b/src/Server/KeeperTCPHandler.cpp
@@ -293,7 +293,7 @@ Poco::Timespan KeeperTCPHandler::receiveHandshake(int32_t handshake_length)
     if (handshake_length == Coordination::CLIENT_HANDSHAKE_LENGTH_WITH_READONLY)
         Coordination::read(readonly, *in);
 
-    return Poco::Timespan(0, timeout_ms * 1000);
+    return Poco::Timespan(timeout_ms * 1000);
 }
 
 
@@ -342,8 +342,8 @@ void KeeperTCPHandler::runImpl()
         int32_t handshake_length = header;
         auto client_timeout = receiveHandshake(handshake_length);
 
-        if (client_timeout == 0)
-            client_timeout = Coordination::DEFAULT_SESSION_TIMEOUT_MS;
+        if (client_timeout.totalMilliseconds() == 0)
+            client_timeout = Poco::Timespan(Coordination::DEFAULT_SESSION_TIMEOUT_MS * Poco::Timespan::MILLISECONDS);
         session_timeout = std::max(client_timeout, min_session_timeout);
         session_timeout = std::min(session_timeout, max_session_timeout);
     }
diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h
index cb50b149629..d18ffc5afe8 100644
--- a/src/Server/TCPHandler.h
+++ b/src/Server/TCPHandler.h
@@ -173,8 +173,8 @@ private:
 
     /// Connection settings, which are extracted from a context.
     bool send_exception_with_stack_trace = true;
-    Poco::Timespan send_timeout = DBMS_DEFAULT_SEND_TIMEOUT_SEC;
-    Poco::Timespan receive_timeout = DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC;
+    Poco::Timespan send_timeout = Poco::Timespan(DBMS_DEFAULT_SEND_TIMEOUT_SEC, 0);
+    Poco::Timespan receive_timeout = Poco::Timespan(DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, 0);
     UInt64 poll_interval = DBMS_DEFAULT_POLL_INTERVAL;
     UInt64 idle_connection_timeout = 3600;
     UInt64 interactive_delay = 100000;
diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp
index f53a7afedd1..e84f5c963a8 100644
--- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp
+++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp
@@ -724,7 +724,7 @@ void StorageRabbitMQ::read(
 
     uint64_t max_execution_time_ms = rabbitmq_settings->rabbitmq_flush_interval_ms.changed
         ? rabbitmq_settings->rabbitmq_flush_interval_ms
-        : static_cast<UInt64>(Poco::Timespan(getContext()->getSettingsRef().stream_flush_interval_ms).milliseconds());
+        : static_cast<UInt64>(getContext()->getSettingsRef().stream_flush_interval_ms.totalMilliseconds());
 
     for (size_t i = 0; i < num_created_consumers; ++i)
     {
@@ -1053,7 +1053,7 @@ bool StorageRabbitMQ::tryStreamToViews()
 
     uint64_t max_execution_time_ms = rabbitmq_settings->rabbitmq_flush_interval_ms.changed
         ? rabbitmq_settings->rabbitmq_flush_interval_ms
-        : static_cast<UInt64>(Poco::Timespan(getContext()->getSettingsRef().stream_flush_interval_ms).milliseconds());
+        : static_cast<UInt64>(getContext()->getSettingsRef().stream_flush_interval_ms.totalMilliseconds());
 
     for (size_t i = 0; i < num_created_consumers; ++i)
     {
diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp
index 0cc826e674d..b2091d7305f 100644
--- a/src/Storages/StorageDistributed.cpp
+++ b/src/Storages/StorageDistributed.cpp
@@ -1986,9 +1986,9 @@ void registerStorageDistributed(StorageFactory & factory)
         if (!distributed_settings.monitor_split_batch_on_failure.changed)
             distributed_settings.monitor_split_batch_on_failure = context->getSettingsRef().distributed_directory_monitor_split_batch_on_failure;
         if (!distributed_settings.monitor_sleep_time_ms.changed)
-            distributed_settings.monitor_sleep_time_ms = Poco::Timespan(context->getSettingsRef().distributed_directory_monitor_sleep_time_ms);
+            distributed_settings.monitor_sleep_time_ms = context->getSettingsRef().distributed_directory_monitor_sleep_time_ms;
         if (!distributed_settings.monitor_max_sleep_time_ms.changed)
-            distributed_settings.monitor_max_sleep_time_ms = Poco::Timespan(context->getSettingsRef().distributed_directory_monitor_max_sleep_time_ms);
+            distributed_settings.monitor_max_sleep_time_ms = context->getSettingsRef().distributed_directory_monitor_max_sleep_time_ms;
 
         return std::make_shared<StorageDistributed>(
             args.table_id,

From 8f573aba2d4c3d80779665fdbaaf547449ae52d7 Mon Sep 17 00:00:00 2001
From: Dan Roscigno <dan@roscigno.com>
Date: Wed, 10 May 2023 14:00:05 -0400
Subject: [PATCH 402/689] docs for connect_timeout_with_failover_secure_ms

closes #47267
---
 docs/en/operations/settings/settings.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index ad7ef52412a..2cb84203d50 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -1125,6 +1125,12 @@ If unsuccessful, several attempts are made to connect to various replicas.
 
 Default value: 1000.
 
+## connect_timeout_with_failover_secure_ms
+
+Connection timeout for selecting first healthy replica (for secure connections)
+
+Default value: 1000.
+
 ## connection_pool_max_wait_ms {#connection-pool-max-wait-ms}
 
 The wait time in milliseconds for a connection when the connection pool is full.

From 7fbf87be176081411918ae35f040f338892d1416 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Tue, 18 Apr 2023 13:11:42 +0200
Subject: [PATCH 403/689] rework WriteBufferFromS3, squashed

---
 contrib/googletest-cmake/CMakeLists.txt       |   25 +-
 src/Backups/BackupIO_S3.cpp                   |    1 -
 src/CMakeLists.txt                            |    1 +
 src/Disks/DiskLocal.cpp                       |    1 +
 .../IO/CachedOnDiskWriteBufferFromFile.cpp    |   13 +-
 .../ObjectStorages/S3/S3ObjectStorage.cpp     |    3 +-
 src/IO/S3/Client.cpp                          |   33 +-
 src/IO/S3/Client.h                            |    7 +
 src/IO/SwapHelper.cpp                         |   17 +
 src/IO/SwapHelper.h                           |   21 +-
 src/IO/WriteBuffer.h                          |    5 +-
 src/IO/WriteBufferFromFileDecorator.cpp       |   38 +-
 src/IO/WriteBufferFromFileDecorator.h         |    7 +-
 src/IO/WriteBufferFromS3.cpp                  |  946 +++++++-------
 src/IO/WriteBufferFromS3.h                    |   95 +-
 src/IO/WriteBufferFromS3MemoryStream.cpp      |   68 +
 src/IO/WriteBufferFromS3MemoryStream.h        |   39 +
 src/IO/WriteBufferFromS3TaskTracker.cpp       |  137 ++
 src/IO/WriteBufferFromS3TaskTracker.h         |   37 +
 src/IO/tests/gtest_writebuffer_s3.cpp         | 1114 +++++++++++++++++
 src/Storages/MergeTree/MergeTreeData.cpp      |    1 +
 .../MergeTree/MergeTreeDeduplicationLog.cpp   |    3 +
 .../MergeTree/MergeTreeMutationEntry.cpp      |    1 +
 src/Storages/StorageS3.cpp                    |    1 -
 .../02240_filesystem_query_cache.reference    |    1 +
 .../02240_filesystem_query_cache.sql          |    2 +-
 ...system_cache_on_write_operations.reference |   36 +-
 ...41_filesystem_cache_on_write_operations.sh |   19 +-
 ...ilesystem_cache_persistent_files.reference |   17 +-
 ...events_from_query_log_and_client.reference |    4 +-
 30 files changed, 2102 insertions(+), 591 deletions(-)
 create mode 100644 src/IO/SwapHelper.cpp
 create mode 100644 src/IO/WriteBufferFromS3MemoryStream.cpp
 create mode 100644 src/IO/WriteBufferFromS3MemoryStream.h
 create mode 100644 src/IO/WriteBufferFromS3TaskTracker.cpp
 create mode 100644 src/IO/WriteBufferFromS3TaskTracker.h
 create mode 100644 src/IO/tests/gtest_writebuffer_s3.cpp

diff --git a/contrib/googletest-cmake/CMakeLists.txt b/contrib/googletest-cmake/CMakeLists.txt
index 90fdde0c185..3905df03155 100644
--- a/contrib/googletest-cmake/CMakeLists.txt
+++ b/contrib/googletest-cmake/CMakeLists.txt
@@ -1,15 +1,30 @@
-set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest")
+set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/googletest")
 
-add_library(_gtest "${SRC_DIR}/src/gtest-all.cc")
+add_library(_gtest "${SRC_DIR}/googletest/src/gtest-all.cc")
 set_target_properties(_gtest PROPERTIES VERSION "1.0.0")
 target_compile_definitions (_gtest PUBLIC GTEST_HAS_POSIX_RE=0)
-target_include_directories(_gtest SYSTEM PUBLIC "${SRC_DIR}/include")
-target_include_directories(_gtest PRIVATE "${SRC_DIR}")
+target_include_directories(_gtest SYSTEM PUBLIC "${SRC_DIR}/googletest/include")
+target_include_directories(_gtest PRIVATE "${SRC_DIR}/googletest")
 
-add_library(_gtest_main "${SRC_DIR}/src/gtest_main.cc")
+add_library(_gtest_main "${SRC_DIR}/googletest/src/gtest_main.cc")
 set_target_properties(_gtest_main PROPERTIES VERSION "1.0.0")
 target_link_libraries(_gtest_main PUBLIC _gtest)
 
 add_library(_gtest_all INTERFACE)
 target_link_libraries(_gtest_all INTERFACE _gtest _gtest_main)
 add_library(ch_contrib::gtest_all ALIAS _gtest_all)
+
+
+add_library(_gmock "${SRC_DIR}/googlemock/src/gmock-all.cc")
+set_target_properties(_gmock PROPERTIES VERSION "1.0.0")
+target_compile_definitions (_gmock PUBLIC GTEST_HAS_POSIX_RE=0)
+target_include_directories(_gmock SYSTEM PUBLIC "${SRC_DIR}/googlemock/include" "${SRC_DIR}/googletest/include")
+target_include_directories(_gmock PRIVATE "${SRC_DIR}/googlemock")
+
+add_library(_gmock_main "${SRC_DIR}/googlemock/src/gmock_main.cc")
+set_target_properties(_gmock_main PROPERTIES VERSION "1.0.0")
+target_link_libraries(_gmock_main PUBLIC _gmock)
+
+add_library(_gmock_all INTERFACE)
+target_link_libraries(_gmock_all INTERFACE _gmock _gmock_main)
+add_library(ch_contrib::gmock_all ALIAS _gmock_all)
diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp
index 90333900d4a..84dba63ae4e 100644
--- a/src/Backups/BackupIO_S3.cpp
+++ b/src/Backups/BackupIO_S3.cpp
@@ -253,7 +253,6 @@ std::unique_ptr<WriteBuffer> BackupWriterS3::writeFile(const String & file_name)
         fs::path(s3_uri.key) / file_name,
         request_settings,
         std::nullopt,
-        DBMS_DEFAULT_BUFFER_SIZE,
         threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupWriterS3"));
 }
 
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index b3f4fbb7420..ac99a7c3669 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -612,6 +612,7 @@ if (ENABLE_TESTS)
 
     target_link_libraries(unit_tests_dbms PRIVATE
         ch_contrib::gtest_all
+        ch_contrib::gmock_all
         clickhouse_functions
         clickhouse_aggregate_functions
         clickhouse_parsers
diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp
index 69b70da272a..1abecb7af4e 100644
--- a/src/Disks/DiskLocal.cpp
+++ b/src/Disks/DiskLocal.cpp
@@ -544,6 +544,7 @@ try
         auto tmp_file = std::make_unique<TemporaryFileOnDisk>(disk_ptr);
         auto buf = std::make_unique<WriteBufferFromTemporaryFile>(std::move(tmp_file));
         buf->write(data.data, data.PAGE_SIZE_IN_BYTES);
+        buf->finalize();
         buf->sync();
     }
     return true;
diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp
index af2226ea6ca..9153af90312 100644
--- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp
+++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp
@@ -211,10 +211,16 @@ void CachedOnDiskWriteBufferFromFile::nextImpl()
 {
     size_t size = offset();
 
+    /// Write data to cache.
+    cacheData(working_buffer.begin(), size, throw_on_error_from_cache);
+    current_download_offset += size;
+
     try
     {
         SwapHelper swap(*this, *impl);
         /// Write data to the underlying buffer.
+        /// Actually here WriteBufferFromFileDecorator::nextImpl has to be called, but it is pivate method.
+        /// In particular WriteBufferFromFileDecorator introduces logic with swaps in order to achieve delegation.
         impl->next();
     }
     catch (...)
@@ -225,10 +231,6 @@ void CachedOnDiskWriteBufferFromFile::nextImpl()
 
         throw;
     }
-
-    /// Write data to cache.
-    cacheData(working_buffer.begin(), size, throw_on_error_from_cache);
-    current_download_offset += size;
 }
 
 void CachedOnDiskWriteBufferFromFile::cacheData(char * data, size_t size, bool throw_on_error)
@@ -292,8 +294,7 @@ void CachedOnDiskWriteBufferFromFile::finalizeImpl()
 {
     try
     {
-        SwapHelper swap(*this, *impl);
-        impl->finalize();
+        WriteBufferFromFileDecorator::finalizeImpl();
     }
     catch (...)
     {
diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
index 2eee8bf5693..79b3d3a2b8b 100644
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
@@ -161,7 +161,7 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
     WriteMode mode, // S3 doesn't support append, only rewrite
     std::optional<ObjectAttributes> attributes,
     FinalizeCallback && finalize_callback,
-    size_t buf_size,
+    size_t buf_size [[maybe_unused]],
     const WriteSettings & write_settings)
 {
     WriteSettings disk_write_settings = IObjectStorage::patchSettings(write_settings);
@@ -180,7 +180,6 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
         object.remote_path,
         settings_ptr->request_settings,
         attributes,
-        buf_size,
         std::move(scheduler),
         disk_write_settings);
 
diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp
index 3c0a8122a91..3c39893b44e 100644
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@@ -255,7 +255,7 @@ Model::HeadObjectOutcome Client::HeadObject(const HeadObjectRequest & request) c
     if (auto uri = getURIForBucket(bucket); uri.has_value())
         request.overrideURI(std::move(*uri));
 
-    auto result = Aws::S3::S3Client::HeadObject(request);
+    auto result = HeadObject(static_cast<const Model::HeadObjectRequest&>(request));
     if (result.IsSuccess())
         return result;
 
@@ -312,70 +312,75 @@ Model::HeadObjectOutcome Client::HeadObject(const HeadObjectRequest & request) c
 
     request.overrideURI(std::move(*bucket_uri));
 
-    return Aws::S3::S3Client::HeadObject(request);
+    /// The next call is NOT a recurcive call
+    /// This is a virtuall call Aws::S3::S3Client::HeadObject(const Model::HeadObjectRequest&)
+    return HeadObject(static_cast<const Model::HeadObjectRequest&>(request));
 }
 
+/// For each request, we wrap the request functions from Aws::S3::Client with doRequest
+/// doRequest calls virtuall function from Aws::S3::Client while DB::S3::Client has not virtual calls for each request type
+
 Model::ListObjectsV2Outcome Client::ListObjectsV2(const ListObjectsV2Request & request) const
 {
-    return doRequest(request, [this](const Model::ListObjectsV2Request & req) { return Aws::S3::S3Client::ListObjectsV2(req); });
+    return doRequest(request, [this](const Model::ListObjectsV2Request & req) { return ListObjectsV2(req); });
 }
 
 Model::ListObjectsOutcome Client::ListObjects(const ListObjectsRequest & request) const
 {
-    return doRequest(request, [this](const Model::ListObjectsRequest & req) { return Aws::S3::S3Client::ListObjects(req); });
+    return doRequest(request, [this](const Model::ListObjectsRequest & req) { return ListObjects(req); });
 }
 
 Model::GetObjectOutcome Client::GetObject(const GetObjectRequest & request) const
 {
-    return doRequest(request, [this](const Model::GetObjectRequest & req) { return Aws::S3::S3Client::GetObject(req); });
+    return doRequest(request, [this](const Model::GetObjectRequest & req) { return GetObject(req); });
 }
 
 Model::AbortMultipartUploadOutcome Client::AbortMultipartUpload(const AbortMultipartUploadRequest & request) const
 {
     return doRequest(
-        request, [this](const Model::AbortMultipartUploadRequest & req) { return Aws::S3::S3Client::AbortMultipartUpload(req); });
+        request, [this](const Model::AbortMultipartUploadRequest & req) { return AbortMultipartUpload(req); });
 }
 
 Model::CreateMultipartUploadOutcome Client::CreateMultipartUpload(const CreateMultipartUploadRequest & request) const
 {
     return doRequest(
-        request, [this](const Model::CreateMultipartUploadRequest & req) { return Aws::S3::S3Client::CreateMultipartUpload(req); });
+        request, [this](const Model::CreateMultipartUploadRequest & req) { return CreateMultipartUpload(req); });
 }
 
 Model::CompleteMultipartUploadOutcome Client::CompleteMultipartUpload(const CompleteMultipartUploadRequest & request) const
 {
     return doRequest(
-        request, [this](const Model::CompleteMultipartUploadRequest & req) { return Aws::S3::S3Client::CompleteMultipartUpload(req); });
+        request, [this](const Model::CompleteMultipartUploadRequest & req) { return CompleteMultipartUpload(req); });
 }
 
 Model::CopyObjectOutcome Client::CopyObject(const CopyObjectRequest & request) const
 {
-    return doRequest(request, [this](const Model::CopyObjectRequest & req) { return Aws::S3::S3Client::CopyObject(req); });
+    return doRequest(request, [this](const Model::CopyObjectRequest & req) { return CopyObject(req); });
 }
 
 Model::PutObjectOutcome Client::PutObject(const PutObjectRequest & request) const
 {
-    return doRequest(request, [this](const Model::PutObjectRequest & req) { return Aws::S3::S3Client::PutObject(req); });
+    return doRequest(request, [this](const Model::PutObjectRequest & req) { return PutObject(req); });
 }
 
 Model::UploadPartOutcome Client::UploadPart(const UploadPartRequest & request) const
 {
-    return doRequest(request, [this](const Model::UploadPartRequest & req) { return Aws::S3::S3Client::UploadPart(req); });
+    return doRequest(request, [this](const Model::UploadPartRequest & req) { return UploadPart(req); });
 }
 
 Model::UploadPartCopyOutcome Client::UploadPartCopy(const UploadPartCopyRequest & request) const
 {
-    return doRequest(request, [this](const Model::UploadPartCopyRequest & req) { return Aws::S3::S3Client::UploadPartCopy(req); });
+    return doRequest(request, [this](const Model::UploadPartCopyRequest & req) { return UploadPartCopy(req); });
 }
 
 Model::DeleteObjectOutcome Client::DeleteObject(const DeleteObjectRequest & request) const
 {
-    return doRequest(request, [this](const Model::DeleteObjectRequest & req) { return Aws::S3::S3Client::DeleteObject(req); });
+    return doRequest(request, [this](const Model::DeleteObjectRequest & req) { return DeleteObject(req); });
 }
 
 Model::DeleteObjectsOutcome Client::DeleteObjects(const DeleteObjectsRequest & request) const
 {
-    return doRequest(request, [this](const Model::DeleteObjectsRequest & req) { return Aws::S3::S3Client::DeleteObjects(req); });
+    return doRequest(request, [this](const Model::DeleteObjectsRequest & req) { return DeleteObjects(req); });
 }
 
 template <typename RequestType, typename RequestFn>
diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h
index 63feb94e593..330c85c418a 100644
--- a/src/IO/S3/Client.h
+++ b/src/IO/S3/Client.h
@@ -40,6 +40,11 @@ struct ServerSideEncryptionKMSConfig
 #include <aws/core/client/AWSErrorMarshaller.h>
 #include <aws/core/client/RetryStrategy.h>
 
+namespace MockS3
+{
+    struct Client;
+}
+
 namespace DB::S3
 {
 
@@ -195,6 +200,8 @@ public:
 
     bool supportsMultiPartCopy() const;
 private:
+    friend struct ::MockS3::Client;
+
     Client(size_t max_redirects_,
            ServerSideEncryptionKMSConfig sse_kms_config_,
            const std::shared_ptr<Aws::Auth::AWSCredentialsProvider>& credentials_provider,
diff --git a/src/IO/SwapHelper.cpp b/src/IO/SwapHelper.cpp
new file mode 100644
index 00000000000..4a1cc8acf4c
--- /dev/null
+++ b/src/IO/SwapHelper.cpp
@@ -0,0 +1,17 @@
+#include <IO/SwapHelper.h>
+
+namespace DB
+{
+
+SwapHelper::SwapHelper(BufferBase & b1_, BufferBase & b2_)
+    : b1(b1_), b2(b2_)
+{
+    b1.swap(b2);
+}
+
+SwapHelper::~SwapHelper()
+{
+    b1.swap(b2);
+}
+
+}
diff --git a/src/IO/SwapHelper.h b/src/IO/SwapHelper.h
index cedbf5f78fe..fcf32927f23 100644
--- a/src/IO/SwapHelper.h
+++ b/src/IO/SwapHelper.h
@@ -1,16 +1,19 @@
 #pragma once
+
 #include <IO/BufferBase.h>
 
 namespace DB
 {
-    class SwapHelper
-    {
-    public:
-        SwapHelper(BufferBase & b1_, BufferBase & b2_) : b1(b1_), b2(b2_) { b1.swap(b2); }
-        ~SwapHelper() { b1.swap(b2); }
 
-    private:
-        BufferBase & b1;
-        BufferBase & b2;
-    };
+class SwapHelper
+{
+public:
+    SwapHelper(BufferBase & b1_, BufferBase & b2_);
+    ~SwapHelper();
+
+private:
+    BufferBase & b1;
+    BufferBase & b2;
+};
+
 }
diff --git a/src/IO/WriteBuffer.h b/src/IO/WriteBuffer.h
index 436d07515a3..2c891e17d9a 100644
--- a/src/IO/WriteBuffer.h
+++ b/src/IO/WriteBuffer.h
@@ -42,7 +42,8 @@ public:
     {
         if (!offset())
             return;
-        bytes += offset();
+
+        auto bytes_in_buffer = offset();
 
         try
         {
@@ -54,9 +55,11 @@ public:
               * so that later (for example, when the stack was expanded) there was no second attempt to write data.
               */
             pos = working_buffer.begin();
+            bytes += bytes_in_buffer;
             throw;
         }
 
+        bytes += bytes_in_buffer;
         pos = working_buffer.begin();
     }
 
diff --git a/src/IO/WriteBufferFromFileDecorator.cpp b/src/IO/WriteBufferFromFileDecorator.cpp
index ac801534b4f..4cc881f177f 100644
--- a/src/IO/WriteBufferFromFileDecorator.cpp
+++ b/src/IO/WriteBufferFromFileDecorator.cpp
@@ -1,6 +1,7 @@
 #include "WriteBufferFromFileDecorator.h"
 
 #include <IO/WriteBuffer.h>
+#include <IO/SwapHelper.h>
 
 namespace DB
 {
@@ -13,12 +14,18 @@ WriteBufferFromFileDecorator::WriteBufferFromFileDecorator(std::unique_ptr<Write
 
 void WriteBufferFromFileDecorator::finalizeImpl()
 {
-    next();
 
+    /// In case of exception in preFinalize as a part of finalize call
+    /// WriteBufferFromFileDecorator.finalized is set as true
+    /// but impl->finalized is remain false
+    /// That leads to situation when the destructor of impl is called with impl->finalized equal false.
     if (!is_prefinalized)
         WriteBufferFromFileDecorator::preFinalize();
 
-    impl->finalize();
+    {
+        SwapHelper swap(*this, *impl);
+        impl->finalize();
+    }
 }
 
 WriteBufferFromFileDecorator::~WriteBufferFromFileDecorator()
@@ -31,11 +38,21 @@ WriteBufferFromFileDecorator::~WriteBufferFromFileDecorator()
     {
         tryLogCurrentException(__PRETTY_FUNCTION__);
     }
+
+    /// It is not a mistake that swap is called here
+    /// Swap has been called at constructor, it should be called at destructor
+    /// In oreder to provide valid buffer for impl's d-tor call
+    swap(*impl);
 }
 
 void WriteBufferFromFileDecorator::sync()
 {
-    impl->sync();
+    next();
+
+    {
+        SwapHelper swap(*this, *impl);
+        impl->sync();
+    }
 }
 
 std::string WriteBufferFromFileDecorator::getFileName() const
@@ -45,11 +62,22 @@ std::string WriteBufferFromFileDecorator::getFileName() const
     return std::string();
 }
 
+void WriteBufferFromFileDecorator::preFinalize()
+{
+    next();
+
+    {
+        SwapHelper swap(*this, *impl);
+        impl->preFinalize();
+    }
+
+    is_prefinalized = true;
+}
+
 void WriteBufferFromFileDecorator::nextImpl()
 {
-    swap(*impl);
+    SwapHelper swap(*this, *impl);
     impl->next();
-    swap(*impl);
 }
 
 }
diff --git a/src/IO/WriteBufferFromFileDecorator.h b/src/IO/WriteBufferFromFileDecorator.h
index dde05276c28..5344bb1425c 100644
--- a/src/IO/WriteBufferFromFileDecorator.h
+++ b/src/IO/WriteBufferFromFileDecorator.h
@@ -17,12 +17,7 @@ public:
 
     std::string getFileName() const override;
 
-    void preFinalize() override
-    {
-        next();
-        impl->preFinalize();
-        is_prefinalized = true;
-    }
+    void preFinalize() override;
 
     const WriteBuffer & getImpl() const { return *impl; }
 
diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp
index 5a25cb89107..5630ed2cb68 100644
--- a/src/IO/WriteBufferFromS3.cpp
+++ b/src/IO/WriteBufferFromS3.cpp
@@ -2,13 +2,16 @@
 
 #if USE_AWS_S3
 
+#include "WriteBufferFromS3.h"
+#include "WriteBufferFromS3MemoryStream.h"
+#include "WriteBufferFromS3TaskTracker.h"
+
 #include <Common/logger_useful.h>
 #include <Common/ProfileEvents.h>
 #include <Common/Throttler.h>
 #include <Interpreters/Cache/FileCache.h>
 
 #include <IO/ResourceGuard.h>
-#include <IO/WriteBufferFromS3.h>
 #include <IO/WriteHelpers.h>
 #include <IO/S3Common.h>
 #include <IO/S3/Requests.h>
@@ -29,11 +32,13 @@ namespace ProfileEvents
 
     extern const Event S3CreateMultipartUpload;
     extern const Event S3CompleteMultipartUpload;
+    extern const Event S3AbortMultipartUpload;
     extern const Event S3UploadPart;
     extern const Event S3PutObject;
 
     extern const Event DiskS3CreateMultipartUpload;
     extern const Event DiskS3CompleteMultipartUpload;
+    extern const Event DiskS3AbortMultipartUpload;
     extern const Event DiskS3UploadPart;
     extern const Event DiskS3PutObject;
 
@@ -43,30 +48,105 @@ namespace ProfileEvents
 
 namespace DB
 {
-// S3 protocol does not allow to have multipart upload with more than 10000 parts.
-// In case server does not return an error on exceeding that number, we print a warning
-// because custom S3 implementation may allow relaxed requirements on that.
-const int S3_WARN_MAX_PARTS = 10000;
 
 namespace ErrorCodes
 {
     extern const int S3_ERROR;
     extern const int INVALID_CONFIG_PARAMETER;
+    extern const int LOGICAL_ERROR;
 }
 
-struct WriteBufferFromS3::UploadPartTask
+struct WriteBufferFromS3::PartData
 {
-    S3::UploadPartRequest req;
-    bool is_finished = false;
-    std::string tag;
-    std::exception_ptr exception;
+    Memory<> memory;
+    size_t data_size = 0;
+
+    std::shared_ptr<std::iostream> createAwsBuffer()
+    {
+        auto buffer = std::make_shared<MemoryStream>(memory.data(), data_size);
+        buffer->exceptions(std::ios::badbit);
+        return buffer;
+    }
+
+    bool isEmpty() const
+    {
+        return data_size == 0;
+    }
 };
 
-struct WriteBufferFromS3::PutObjectTask
+struct WriteBufferFromS3::BufferAllocationPolicy
 {
-    S3::PutObjectRequest req;
-    bool is_finished = false;
-    std::exception_ptr exception;
+    size_t first_size = 0;
+    size_t second_size = 0;
+
+    size_t multiply_factor = 0;
+    size_t multiply_threshold = 0;
+    size_t max_size = 0;
+
+    size_t current_size = 0;
+    size_t buffer_number = 0;
+
+    explicit BufferAllocationPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings_)
+        : first_size(std::max(settings_.max_single_part_upload_size, settings_.min_upload_part_size))
+        , second_size(settings_.min_upload_part_size)
+        , multiply_factor(settings_.upload_part_size_multiply_factor)
+        , multiply_threshold(settings_.upload_part_size_multiply_parts_count_threshold)
+        , max_size(settings_.max_upload_part_size)
+    {
+        if (settings_.strict_upload_part_size > 0)
+        {
+            first_size = settings_.strict_upload_part_size;
+            second_size = settings_.strict_upload_part_size;
+            multiply_factor = 1;
+            multiply_threshold = 10000;
+            max_size = settings_.max_upload_part_size;
+        }
+        else
+        {
+            first_size = std::max(settings_.max_single_part_upload_size, settings_.min_upload_part_size);
+            second_size = settings_.min_upload_part_size;
+            multiply_factor = settings_.upload_part_size_multiply_factor;
+            multiply_threshold = settings_.upload_part_size_multiply_parts_count_threshold;
+            max_size = settings_.max_upload_part_size;
+        }
+
+        chassert(first_size > 0);
+        chassert(second_size > 0);
+        chassert(multiply_factor >= 1);
+        chassert(multiply_threshold > 0);
+        chassert(max_size > 0);
+    }
+
+    size_t getNumber() const
+    {
+        return buffer_number;
+    }
+
+    size_t getSize() const
+    {
+        chassert(buffer_number > 0);
+        return current_size;
+    }
+
+    void next()
+    {
+        ++buffer_number;
+
+        if (1 == buffer_number)
+        {
+            current_size = first_size;
+            return;
+        }
+
+        if (2 == buffer_number)
+            current_size = second_size;
+
+        if (0 == ((buffer_number-1) % multiply_threshold))
+        {
+            current_size *= multiply_factor;
+            current_size = std::min(current_size, max_size);
+        }
+    }
 };
 
 WriteBufferFromS3::WriteBufferFromS3(
@@ -75,146 +155,88 @@ WriteBufferFromS3::WriteBufferFromS3(
     const String & key_,
     const S3Settings::RequestSettings & request_settings_,
     std::optional<std::map<String, String>> object_metadata_,
-    size_t buffer_size_,
     ThreadPoolCallbackRunner<void> schedule_,
     const WriteSettings & write_settings_)
-    : BufferWithOwnMemory<WriteBuffer>(buffer_size_, nullptr, 0)
-    , bucket(bucket_)
+    : bucket(bucket_)
     , key(key_)
     , request_settings(request_settings_)
     , upload_settings(request_settings.getUploadSettings())
+    , write_settings(write_settings_)
     , client_ptr(std::move(client_ptr_))
     , object_metadata(std::move(object_metadata_))
-    , strict_upload_part_size(upload_settings.strict_upload_part_size)
-    , current_upload_part_size(upload_settings.min_upload_part_size)
-    , schedule(std::move(schedule_))
-    , write_settings(write_settings_)
+    , buffer_allocation_policy(std::make_unique<BufferAllocationPolicy>(request_settings_.getUploadSettings()))
+    , task_tracker(std::make_unique<WriteBufferFromS3::TaskTracker>(std::move(schedule_)))
 {
+    LOG_TRACE(log, "Create WriteBufferFromS3, {}", getLogDetails());
+
     allocateBuffer();
 }
 
 void WriteBufferFromS3::nextImpl()
 {
-    if (!offset())
+    LOG_TRACE(log, "nextImpl with incoming data size {}, memory buffer size {}. {}", offset(), memory.size(), getLogDetails());
+
+    if (is_prefinalized)
+        throw Exception(
+                ErrorCodes::LOGICAL_ERROR,
+                "Cannot write to prefinalized buffer for S3, the file could have been created with PutObjectRequest");
+
+    /// Make sense to call to before adding new async task to check if there is an exception
+    task_tracker->getReady();
+
+    hidePartialData();
+
+    reallocateFirstBuffer();
+
+    if (available() > 0)
         return;
 
-    /// Buffer in a bad state after exception
-    if (temporary_buffer->tellp() == -1)
-        allocateBuffer();
-    else
-        chassert(temporary_buffer->tellp() == static_cast<std::streamoff>(last_part_size));
+    detachBuffer();
 
-    if (strict_upload_part_size)
-        processWithStrictParts();
-    else
-        processWithDynamicParts();
+    if (!multipart_upload_id.empty() || detached_part_data.size() > 1)
+        writeMultipartUpload();
 
-    waitForReadyBackgroundTasks();
-}
-
-void WriteBufferFromS3::processWithStrictParts()
-{
-    chassert(strict_upload_part_size > 0);
-
-    size_t buffer_size = offset();
-    size_t left_in_buffer = buffer_size;
-    size_t new_size = last_part_size + buffer_size;
-    size_t buffer_offset = 0;
-
-    if (new_size > strict_upload_part_size)
-    {
-        /// Data size will exceed fixed part size threshold for multipart upload, need to use multipart upload.
-        if (multipart_upload_id.empty())
-            createMultipartUpload();
-
-        while (new_size > strict_upload_part_size)
-        {
-            size_t to_write = strict_upload_part_size - last_part_size;
-            temporary_buffer->write(working_buffer.begin() + buffer_offset, to_write);
-            buffer_offset += to_write;
-
-            writePart();
-            allocateBuffer();
-
-            new_size -= strict_upload_part_size;
-            left_in_buffer -= to_write;
-        }
-    }
-
-    if (left_in_buffer)
-    {
-        temporary_buffer->write(working_buffer.begin() + buffer_offset, left_in_buffer);
-        last_part_size += left_in_buffer;
-    }
-
-    ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Bytes, buffer_size);
-
-    if (write_settings.remote_throttler)
-        write_settings.remote_throttler->add(buffer_size, ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds);
-}
-
-void WriteBufferFromS3::processWithDynamicParts()
-{
-    chassert(current_upload_part_size > 0);
-
-    size_t size = offset();
-    temporary_buffer->write(working_buffer.begin(), size);
-    ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Bytes, size);
-    last_part_size += size;
-
-    if (write_settings.remote_throttler)
-        write_settings.remote_throttler->add(size, ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds);
-
-    /// Data size exceeds singlepart upload threshold, need to use multipart upload.
-    if (multipart_upload_id.empty() && last_part_size > upload_settings.max_single_part_upload_size)
-        createMultipartUpload();
-
-    if (!multipart_upload_id.empty() && last_part_size > current_upload_part_size)
-    {
-        writePart();
-        allocateBuffer();
-    }
-}
-
-void WriteBufferFromS3::allocateBuffer()
-{
-    temporary_buffer = Aws::MakeShared<Aws::StringStream>("temporary buffer");
-    temporary_buffer->exceptions(std::ios::badbit);
-    last_part_size = 0;
-}
-
-WriteBufferFromS3::~WriteBufferFromS3()
-{
-#ifndef NDEBUG
-    if (!finalized)
-    {
-        LOG_ERROR(log, "WriteBufferFromS3 is not finalized in destructor. It's a bug");
-        std::terminate();
-    }
-#else
-    try
-    {
-        finalize();
-    }
-    catch (...)
-    {
-        tryLogCurrentException(__PRETTY_FUNCTION__);
-    }
-#endif
+    allocateBuffer();
 }
 
 void WriteBufferFromS3::preFinalize()
 {
-    next();
+    if (is_prefinalized)
+        return;
 
-    if (multipart_upload_id.empty())
+    LOG_TRACE(log, "preFinalize WriteBufferFromS3. {}", getLogDetails());
+
+    task_tracker->getReady();
+
+    hidePartialData();
+
+    if (hidden_size > 0)
+        detachBuffer();
+    setFakeBufferWhenPreFinalized();
+
+    bool do_single_part_upload = false;
+
+    if (multipart_upload_id.empty() && detached_part_data.size() <= 1)
     {
-        makeSinglepartUpload();
+        if (detached_part_data.empty() || detached_part_data.front().data_size <= upload_settings.max_single_part_upload_size)
+            do_single_part_upload = true;
+    }
+
+    if (do_single_part_upload)
+    {
+        if (detached_part_data.empty())
+        {
+            makeSinglepartUpload({});
+        }
+        else
+        {
+            makeSinglepartUpload(std::move(detached_part_data.front()));
+            detached_part_data.pop_front();
+        }
     }
     else
     {
-        /// Write rest of the data as last part.
-        writePart();
+        writeMultipartUpload();
     }
 
     is_prefinalized = true;
@@ -222,24 +244,182 @@ void WriteBufferFromS3::preFinalize()
 
 void WriteBufferFromS3::finalizeImpl()
 {
+    LOG_TRACE(log, "finalizeImpl WriteBufferFromS3. {}.", getLogDetails());
+
     if (!is_prefinalized)
         preFinalize();
 
-    waitForAllBackgroundTasks();
+    chassert(offset() == 0);
+    chassert(hidden_size == 0);
+
+    task_tracker->getAll();
 
     if (!multipart_upload_id.empty())
+    {
         completeMultipartUpload();
+        multipart_upload_finished = true;
+    }
 
     if (request_settings.check_objects_after_upload)
     {
         LOG_TRACE(log, "Checking object {} exists after upload", key);
         S3::checkObjectExists(*client_ptr, bucket, key, {}, request_settings, /* for_disk_s3= */ write_settings.for_object_storage, "Immediately after upload");
+
+        LOG_TRACE(log, "Checking object {} has size as expected {}", key, total_size);
+        size_t actual_size = S3::getObjectSize(*client_ptr, bucket, key, {}, request_settings, /* for_disk_s3= */ write_settings.for_object_storage);
+        if (actual_size != total_size)
+            throw Exception(
+                    ErrorCodes::S3_ERROR,
+                    "Object {} from bucket {} has unexpected size {} after upload, expected size {}, it's a bug in S3 or S3 API.",
+                    key, bucket, actual_size, total_size);
+
         LOG_TRACE(log, "Object {} exists after upload", key);
     }
 }
 
-void WriteBufferFromS3::fillCreateMultipartRequest(DB::S3::CreateMultipartUploadRequest & req)
+String WriteBufferFromS3::getLogDetails() const
 {
+    String multipart_upload_details;
+    if (!multipart_upload_id.empty())
+        multipart_upload_details = fmt::format(", upload id {}, upload has finished {}"
+                                       , multipart_upload_id, multipart_upload_finished);
+
+    return fmt::format("Details: bucket {}, key {}, total size {}, count {}, hidden_size {}, offset {}, with pool: {}, finalized {}{}",
+                       bucket, key, total_size, count(), hidden_size, offset(), task_tracker->isAsync(), finalized, multipart_upload_details);
+}
+
+void WriteBufferFromS3::tryToAbortMultipartUpload()
+{
+    try
+    {
+        task_tracker->safeWaitAll();
+        abortMultipartUpload();
+    }
+    catch (...)
+    {
+        LOG_ERROR(log, "Multipart upload hasn't aborted. {}", getLogDetails());
+        tryLogCurrentException(__PRETTY_FUNCTION__);
+    }
+}
+
+WriteBufferFromS3::~WriteBufferFromS3()
+{
+    LOG_TRACE(log, "Close WriteBufferFromS3. {}.", getLogDetails());
+
+    // That descructor could be call with finalized=false in case of exceptions
+    if (!finalized)
+    {
+        LOG_ERROR(log, "WriteBufferFromS3 is not finalized in destructor. It could be if an exception occurs. File is not written to S3. {}.", getLogDetails());
+    }
+
+    task_tracker->safeWaitAll();
+
+    if (!multipart_upload_id.empty() && !multipart_upload_finished)
+    {
+        LOG_WARNING(log, "WriteBufferFromS3 was neither finished nor aborted, try to abort upload in destructor. {}.", getLogDetails());
+        tryToAbortMultipartUpload();
+    }
+}
+
+void WriteBufferFromS3::hidePartialData()
+{
+    if (write_settings.remote_throttler)
+            write_settings.remote_throttler->add(offset(), ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds);
+
+    chassert(memory.size() >= hidden_size + offset());
+
+    hidden_size += offset();
+    chassert(memory.data() + hidden_size == working_buffer.begin() + offset());
+    chassert(memory.data() + hidden_size == position());
+
+    WriteBuffer::set(memory.data() + hidden_size, memory.size() - hidden_size);
+    chassert(offset() == 0);
+}
+
+void WriteBufferFromS3::reallocateFirstBuffer()
+{
+    chassert(offset() == 0);
+
+    if (buffer_allocation_policy->getNumber() > 1 || available() > 0)
+        return;
+
+    const size_t max_first_buffer = buffer_allocation_policy->getSize();
+    if (memory.size() == max_first_buffer)
+        return;
+
+    size_t size = std::min(memory.size() * 2, max_first_buffer);
+    memory.resize(size);
+
+    WriteBuffer::set(memory.data() + hidden_size, memory.size() - hidden_size);
+
+    chassert(offset() == 0);
+
+    LOG_TRACE(log, "Reallocated first buffer with size {}. {}", memory.size(), getLogDetails());
+}
+
+void WriteBufferFromS3::detachBuffer()
+{
+    size_t data_size = size_t(position() - memory.data());
+    chassert(data_size == hidden_size);
+
+    auto buf = std::move(memory);
+
+    WriteBuffer::set(nullptr, 0);
+    total_size += hidden_size;
+    hidden_size = 0;
+
+    detached_part_data.push_back({std::move(buf), data_size});
+}
+
+void WriteBufferFromS3::allocateFirstBuffer()
+{
+    const auto max_first_buffer = buffer_allocation_policy->getSize();
+    const auto size = std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), max_first_buffer);
+    memory = Memory(size);
+    WriteBuffer::set(memory.data(), memory.size());
+
+    LOG_TRACE(log, "Allocated first buffer with size {}. {}", memory.size(), getLogDetails());
+}
+
+void WriteBufferFromS3::allocateBuffer()
+{
+    buffer_allocation_policy->next();
+    chassert(0 == hidden_size);
+
+    if (buffer_allocation_policy->getNumber() == 1)
+        return allocateFirstBuffer();
+
+    memory = Memory(buffer_allocation_policy->getSize());
+    WriteBuffer::set(memory.data(), memory.size());
+
+    LOG_TRACE(log, "Allocated buffer with size {}. {}", buffer_allocation_policy->getSize(), getLogDetails());
+}
+
+void WriteBufferFromS3::setFakeBufferWhenPreFinalized()
+{
+    WriteBuffer::set(fake_buffer_when_prefinalized, sizeof(fake_buffer_when_prefinalized));
+}
+
+void WriteBufferFromS3::writeMultipartUpload()
+{
+    if (multipart_upload_id.empty())
+    {
+        createMultipartUpload();
+    }
+
+    while (!detached_part_data.empty())
+    {
+        writePart(std::move(detached_part_data.front()));
+        detached_part_data.pop_front();
+    }
+}
+
+void WriteBufferFromS3::createMultipartUpload()
+{
+    LOG_TRACE(log, "Create multipart upload. Bucket: {}, Key: {}, Upload id: {}", bucket, key, multipart_upload_id);
+
+    S3::CreateMultipartUploadRequest req;
+
     req.SetBucket(bucket);
     req.SetKey(key);
 
@@ -250,12 +430,6 @@ void WriteBufferFromS3::fillCreateMultipartRequest(DB::S3::CreateMultipartUpload
         req.SetMetadata(object_metadata.value());
 
     client_ptr->setKMSHeaders(req);
-}
-
-void WriteBufferFromS3::createMultipartUpload()
-{
-    DB::S3::CreateMultipartUploadRequest req;
-    fillCreateMultipartRequest(req);
 
     ProfileEvents::increment(ProfileEvents::S3CreateMultipartUpload);
     if (write_settings.for_object_storage)
@@ -267,184 +441,164 @@ void WriteBufferFromS3::createMultipartUpload()
 
     ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, watch.elapsedMicroseconds());
 
-    if (outcome.IsSuccess())
-    {
-        multipart_upload_id = outcome.GetResult().GetUploadId();
-        LOG_TRACE(log, "Multipart upload has created. Bucket: {}, Key: {}, Upload id: {}", bucket, key, multipart_upload_id);
-    }
-    else
+    if (!outcome.IsSuccess())
     {
         ProfileEvents::increment(ProfileEvents::WriteBufferFromS3RequestsErrors, 1);
         throw S3Exception(outcome.GetError().GetMessage(), outcome.GetError().GetErrorType());
     }
+
+    multipart_upload_id = outcome.GetResult().GetUploadId();
+    LOG_TRACE(log, "Multipart upload has created. {}", getLogDetails());
 }
 
-void WriteBufferFromS3::writePart()
+void WriteBufferFromS3::abortMultipartUpload()
 {
-    auto size = temporary_buffer->tellp();
-
-    LOG_TRACE(log, "Writing part. Bucket: {}, Key: {}, Upload_id: {}, Size: {}", bucket, key, multipart_upload_id, size);
-
-    if (size < 0)
+    if (multipart_upload_id.empty())
     {
-        LOG_WARNING(log, "Skipping part upload. Buffer is in bad state, it means that we have tried to upload something, but got an exception.");
+        LOG_WARNING(log, "Nothing to abort. {}", getLogDetails());
         return;
     }
 
-    if (size == 0)
+    LOG_WARNING(log, "Abort multipart upload. {}", getLogDetails());
+
+    S3::AbortMultipartUploadRequest req;
+    req.SetBucket(bucket);
+    req.SetKey(key);
+    req.SetUploadId(multipart_upload_id);
+
+    ProfileEvents::increment(ProfileEvents::S3AbortMultipartUpload);
+    if (write_settings.for_object_storage)
+        ProfileEvents::increment(ProfileEvents::DiskS3AbortMultipartUpload);
+
+    Stopwatch watch;
+    auto outcome = client_ptr->AbortMultipartUpload(req);
+    watch.stop();
+
+    ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, watch.elapsedMicroseconds());
+
+    if (!outcome.IsSuccess())
     {
-        LOG_TRACE(log, "Skipping writing part. Buffer is empty.");
-        return;
+        ProfileEvents::increment(ProfileEvents::WriteBufferFromS3RequestsErrors, 1);
+        throw S3Exception(outcome.GetError().GetMessage(), outcome.GetError().GetErrorType());
     }
 
-    if (TSA_SUPPRESS_WARNING_FOR_READ(part_tags).size() == S3_WARN_MAX_PARTS)
-    {
-        // Don't throw exception here by ourselves but leave the decision to take by S3 server.
-        LOG_WARNING(log, "Maximum part number in S3 protocol has reached (too many parts). Server may not accept this whole upload.");
-    }
-
-    if (schedule)
-    {
-        UploadPartTask * task = nullptr;
-
-        {
-            std::lock_guard lock(bg_tasks_mutex);
-            task = &upload_object_tasks.emplace_back();
-            ++num_added_bg_tasks;
-        }
-
-        /// Notify waiting thread when task finished
-        auto task_finish_notify = [&, task]()
-        {
-            std::lock_guard lock(bg_tasks_mutex);
-            task->is_finished = true;
-            ++num_finished_bg_tasks;
-
-            /// Notification under mutex is important here.
-            /// Otherwise, WriteBuffer could be destroyed in between
-            /// Releasing lock and condvar notification.
-            bg_tasks_condvar.notify_one();
-        };
-
-        try
-        {
-            fillUploadRequest(task->req);
-
-            schedule([this, task, task_finish_notify]()
-            {
-                try
-                {
-                    processUploadRequest(*task);
-                }
-                catch (...)
-                {
-                    task->exception = std::current_exception();
-                }
-
-                task_finish_notify();
-            }, 0);
-        }
-        catch (...)
-        {
-            task_finish_notify();
-            throw;
-        }
-    }
-    else
-    {
-        UploadPartTask task;
-        auto & tags = TSA_SUPPRESS_WARNING_FOR_WRITE(part_tags); /// Suppress warning because schedule == false.
-
-        fillUploadRequest(task.req);
-        processUploadRequest(task);
-        tags.push_back(task.tag);
-    }
+    LOG_WARNING(log, "Multipart upload has aborted successfully. {}", getLogDetails());
 }
 
-void WriteBufferFromS3::fillUploadRequest(S3::UploadPartRequest & req)
+S3::UploadPartRequest WriteBufferFromS3::getUploadRequest(size_t part_number, PartData & data)
 {
-    /// Increase part number.
-    ++part_number;
+    ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Bytes, data.data_size);
+    LOG_TRACE(log, "fillUploadRequest, size {}, key: {}", data.data_size, key);
 
-    auto max_part_number = upload_settings.max_part_number;
-
-    if (!multipart_upload_id.empty() && (part_number > max_part_number))
-    {
-        throw Exception(
-                        ErrorCodes::INVALID_CONFIG_PARAMETER,
-                        "Part number exceeded {} while writing {} bytes to S3. "
-                        "Check min_upload_part_size = {}, max_upload_part_size = {}, "
-                        "upload_part_size_multiply_factor = {}, upload_part_size_multiply_parts_count_threshold = {}, "
-                        "max_single_part_upload_size = {}",
-                        max_part_number, count(),
-                        upload_settings.min_upload_part_size, upload_settings.max_upload_part_size,
-                        upload_settings.upload_part_size_multiply_factor,
-                        upload_settings.upload_part_size_multiply_parts_count_threshold,
-                        upload_settings.max_single_part_upload_size);
-    }
+    S3::UploadPartRequest req;
 
     /// Setup request.
     req.SetBucket(bucket);
     req.SetKey(key);
     req.SetPartNumber(static_cast<int>(part_number));
     req.SetUploadId(multipart_upload_id);
-    req.SetContentLength(temporary_buffer->tellp());
-    req.SetBody(temporary_buffer);
-
+    req.SetContentLength(data.data_size);
+    req.SetBody(data.createAwsBuffer());
     /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840
     req.SetContentType("binary/octet-stream");
 
-    if (!strict_upload_part_size)
-    {
-        /// Maybe increase `current_upload_part_size` (we need to increase it sometimes to keep `part_number` less or equal than `max_part_number`).
-        auto threshold = upload_settings.upload_part_size_multiply_parts_count_threshold;
-        if (!multipart_upload_id.empty() && (part_number % threshold == 0))
-        {
-            auto max_upload_part_size = upload_settings.max_upload_part_size;
-            auto upload_part_size_multiply_factor = upload_settings.upload_part_size_multiply_factor;
-            current_upload_part_size *= upload_part_size_multiply_factor;
-            current_upload_part_size = std::min(current_upload_part_size, max_upload_part_size);
-        }
-    }
+    return req;
 }
 
-void WriteBufferFromS3::processUploadRequest(UploadPartTask & task)
+void WriteBufferFromS3::writePart(WriteBufferFromS3::PartData && data)
 {
-    ProfileEvents::increment(ProfileEvents::S3UploadPart);
-    if (write_settings.for_object_storage)
-        ProfileEvents::increment(ProfileEvents::DiskS3UploadPart);
-
-    ResourceCost cost = task.req.GetContentLength();
-    ResourceGuard rlock(write_settings.resource_link, cost);
-    Stopwatch watch;
-    auto outcome = client_ptr->UploadPart(task.req);
-    watch.stop();
-    rlock.unlock(); // Avoid acquiring other locks under resource lock
-
-    ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, watch.elapsedMicroseconds());
-
-    if (outcome.IsSuccess())
+    if (data.data_size == 0)
     {
-        task.tag = outcome.GetResult().GetETag();
-        std::lock_guard lock(bg_tasks_mutex); /// Protect part_tags from race
-        LOG_TRACE(log, "Writing part finished. Bucket: {}, Key: {}, Upload_id: {}, Etag: {}, Parts: {}", bucket, key, multipart_upload_id, task.tag, part_tags.size());
+        LOG_TRACE(log, "Skipping writing part as empty.");
+        return;
     }
-    else
+
+    multipart_tags.push_back({});
+    size_t part_number = multipart_tags.size();
+    LOG_TRACE(log, "WritePart. {}, part size: {}, part number: {}", getLogDetails(), data.data_size, part_number);
+
+    if (multipart_upload_id.empty())
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "Unable to write a part without multipart_upload_id, details: WriteBufferFromS3 created for bucket {}, key {}",
+            bucket, key);
+
+    if (part_number > upload_settings.max_part_number)
     {
-        ProfileEvents::increment(ProfileEvents::WriteBufferFromS3RequestsErrors, 1);
-        write_settings.resource_link.accumulate(cost); // We assume no resource was used in case of failure
-        throw S3Exception(outcome.GetError().GetMessage(), outcome.GetError().GetErrorType());
+        throw Exception(
+            ErrorCodes::INVALID_CONFIG_PARAMETER,
+            "Part number exceeded {} while writing {} bytes to S3. Check min_upload_part_size = {}, max_upload_part_size = {}, "
+            "upload_part_size_multiply_factor = {}, upload_part_size_multiply_parts_count_threshold = {}, max_single_part_upload_size = {}",
+            upload_settings.max_part_number, count(), upload_settings.min_upload_part_size, upload_settings.max_upload_part_size,
+            upload_settings.upload_part_size_multiply_factor, upload_settings.upload_part_size_multiply_parts_count_threshold,
+            upload_settings.max_single_part_upload_size);
     }
+
+    if (data.data_size > upload_settings.max_upload_part_size)
+    {
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "Part size exceeded max_upload_part_size, part number: {}, part size {}, max_upload_part_size {}, {}",
+            part_number,
+            data.data_size,
+            upload_settings.max_upload_part_size,
+            getLogDetails());
+    }
+
+    auto req = getUploadRequest(part_number, data);
+    auto worker_data = std::make_shared<std::tuple<S3::UploadPartRequest, WriteBufferFromS3::PartData>>(std::move(req), std::move(data));
+
+    auto upload_worker = [&, worker_data, part_number] ()
+    {
+        LOG_TEST(log, "Writing part started. bucket {}, key {}, part id {}", bucket, key, part_number);
+
+        ProfileEvents::increment(ProfileEvents::S3UploadPart);
+        if (write_settings.for_object_storage)
+            ProfileEvents::increment(ProfileEvents::DiskS3UploadPart);
+
+        auto & request = std::get<0>(*worker_data);
+
+        ResourceCost cost = request.GetContentLength();
+        ResourceGuard rlock(write_settings.resource_link, cost);
+        Stopwatch watch;
+        auto outcome = client_ptr->UploadPart(request);
+        watch.stop();
+        rlock.unlock(); // Avoid acquiring other locks under resource lock
+
+        ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, watch.elapsedMicroseconds());
+
+        if (!outcome.IsSuccess())
+        {
+            ProfileEvents::increment(ProfileEvents::WriteBufferFromS3RequestsErrors, 1);
+            write_settings.resource_link.accumulate(cost); // We assume no resource was used in case of failure
+            throw S3Exception(outcome.GetError().GetMessage(), outcome.GetError().GetErrorType());
+        }
+
+        multipart_tags[part_number-1] = outcome.GetResult().GetETag();
+
+        LOG_TEST(log, "Writing part finished. bucket {}, key{}, part id {}, etag {}", bucket, key, part_number, multipart_tags[part_number-1]);
+    };
+
+    task_tracker->add(std::move(upload_worker));
 }
 
 void WriteBufferFromS3::completeMultipartUpload()
 {
-    const auto & tags = TSA_SUPPRESS_WARNING_FOR_READ(part_tags);
+    LOG_TRACE(log, "Completing multipart upload. {}, Parts: {}", getLogDetails(), multipart_tags.size());
 
-    LOG_TRACE(log, "Completing multipart upload. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, tags.size());
+    if (multipart_tags.empty())
+        throw Exception(
+                ErrorCodes::LOGICAL_ERROR,
+                "Failed to complete multipart upload. No parts have uploaded");
 
-    if (tags.empty())
-        throw Exception(ErrorCodes::S3_ERROR, "Failed to complete multipart upload. No parts have uploaded");
+    for (size_t i = 0; i < multipart_tags.size(); ++i)
+    {
+        const auto tag = multipart_tags.at(i);
+        if (tag.empty())
+            throw Exception(
+                    ErrorCodes::LOGICAL_ERROR,
+                    "Failed to complete multipart upload. Part {} haven't been uploaded.", i);
+    }
 
     S3::CompleteMultipartUploadRequest req;
     req.SetBucket(bucket);
@@ -452,10 +606,10 @@ void WriteBufferFromS3::completeMultipartUpload()
     req.SetUploadId(multipart_upload_id);
 
     Aws::S3::Model::CompletedMultipartUpload multipart_upload;
-    for (size_t i = 0; i < tags.size(); ++i)
+    for (size_t i = 0; i < multipart_tags.size(); ++i)
     {
         Aws::S3::Model::CompletedPart part;
-        multipart_upload.AddParts(part.WithETag(tags[i]).WithPartNumber(static_cast<int>(i + 1)));
+        multipart_upload.AddParts(part.WithETag(multipart_tags[i]).WithPartNumber(static_cast<int>(i + 1)));
     }
 
     req.SetMultipartUpload(multipart_upload);
@@ -475,26 +629,24 @@ void WriteBufferFromS3::completeMultipartUpload()
 
         if (outcome.IsSuccess())
         {
-            LOG_TRACE(log, "Multipart upload has completed. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, tags.size());
+            LOG_TRACE(log, "Multipart upload has completed. {}, Parts: {}", getLogDetails(), multipart_tags.size());
             return;
         }
+
+        ProfileEvents::increment(ProfileEvents::WriteBufferFromS3RequestsErrors, 1);
+
+        if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY)
+        {
+            /// For unknown reason, at least MinIO can respond with NO_SUCH_KEY for put requests
+            /// BTW, NO_SUCH_UPLOAD is expected error and we shouldn't retry it
+            LOG_INFO(log, "Multipart upload failed with NO_SUCH_KEY error, will retry. {}, Parts: {}", getLogDetails(), multipart_tags.size());
+        }
         else
         {
-            ProfileEvents::increment(ProfileEvents::WriteBufferFromS3RequestsErrors, 1);
-
-            if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY)
-            {
-                /// For unknown reason, at least MinIO can respond with NO_SUCH_KEY for put requests
-                /// BTW, NO_SUCH_UPLOAD is expected error and we shouldn't retry it
-                LOG_INFO(log, "Multipart upload failed with NO_SUCH_KEY error for Bucket: {}, Key: {}, Upload_id: {}, Parts: {}, will retry", bucket, key, multipart_upload_id, tags.size());
-            }
-            else
-            {
-                throw S3Exception(
-                    outcome.GetError().GetErrorType(),
-                    "Message: {}, Key: {}, Bucket: {}, Tags: {}",
-                    outcome.GetError().GetMessage(), key, bucket, fmt::join(tags.begin(), tags.end(), " "));
-            }
+            throw S3Exception(
+                outcome.GetError().GetErrorType(),
+                "Message: {}, Key: {}, Bucket: {}, Tags: {}",
+                outcome.GetError().GetMessage(), key, bucket, fmt::join(multipart_tags.begin(), multipart_tags.end(), " "));
         }
     }
 
@@ -504,73 +656,17 @@ void WriteBufferFromS3::completeMultipartUpload()
         max_retry, key, bucket);
 }
 
-void WriteBufferFromS3::makeSinglepartUpload()
+S3::PutObjectRequest WriteBufferFromS3::getPutRequest(PartData & data)
 {
-    auto size = temporary_buffer->tellp();
-    bool with_pool = static_cast<bool>(schedule);
+    ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Bytes, data.data_size);
+    LOG_TRACE(log, "getPutRequest, size {}, key {}", data.data_size, key);
 
-    LOG_TRACE(log, "Making single part upload. Bucket: {}, Key: {}, Size: {}, WithPool: {}", bucket, key, size, with_pool);
+    S3::PutObjectRequest req;
 
-    if (size < 0)
-    {
-        LOG_WARNING(log, "Skipping single part upload. Buffer is in bad state, it mean that we have tried to upload something, but got an exception.");
-        return;
-    }
-
-    if (schedule)
-    {
-        put_object_task = std::make_unique<PutObjectTask>();
-
-        /// Notify waiting thread when put object task finished
-        auto task_notify_finish = [&]()
-        {
-            std::lock_guard lock(bg_tasks_mutex);
-            put_object_task->is_finished = true;
-
-            /// Notification under mutex is important here.
-            /// Othervies, WriteBuffer could be destroyed in between
-            /// Releasing lock and condvar notification.
-            bg_tasks_condvar.notify_one();
-        };
-
-        try
-        {
-            fillPutRequest(put_object_task->req);
-
-            schedule([this, task_notify_finish]()
-            {
-                try
-                {
-                    processPutRequest(*put_object_task);
-                }
-                catch (...)
-                {
-                    put_object_task->exception = std::current_exception();
-                }
-
-                task_notify_finish();
-            }, 0);
-        }
-        catch (...)
-        {
-            task_notify_finish();
-            throw;
-        }
-    }
-    else
-    {
-        PutObjectTask task;
-        fillPutRequest(task.req);
-        processPutRequest(task);
-    }
-}
-
-void WriteBufferFromS3::fillPutRequest(S3::PutObjectRequest & req)
-{
     req.SetBucket(bucket);
     req.SetKey(key);
-    req.SetContentLength(temporary_buffer->tellp());
-    req.SetBody(temporary_buffer);
+    req.SetContentLength(data.data_size);
+    req.SetBody(data.createAwsBuffer());
     if (object_metadata.has_value())
         req.SetMetadata(object_metadata.value());
     if (!upload_settings.storage_class_name.empty())
@@ -580,121 +676,73 @@ void WriteBufferFromS3::fillPutRequest(S3::PutObjectRequest & req)
     req.SetContentType("binary/octet-stream");
 
     client_ptr->setKMSHeaders(req);
+
+    return req;
 }
 
-void WriteBufferFromS3::processPutRequest(const PutObjectTask & task)
+void WriteBufferFromS3::makeSinglepartUpload(WriteBufferFromS3::PartData && data)
 {
-    size_t max_retry = std::max(request_settings.max_unexpected_write_error_retries, 1UL);
-    for (size_t i = 0; i < max_retry; ++i)
+    LOG_TRACE(log, "Making single part upload. {}.", getLogDetails());
+
+    auto req = getPutRequest(data);
+    auto worker_data = std::make_shared<std::tuple<S3::PutObjectRequest, WriteBufferFromS3::PartData>>(std::move(req), std::move(data));
+
+    auto upload_worker = [&, worker_data] ()
     {
-        ProfileEvents::increment(ProfileEvents::S3PutObject);
-        if (write_settings.for_object_storage)
-            ProfileEvents::increment(ProfileEvents::DiskS3PutObject);
+        LOG_TEST(log, "writing single part upload started. bucket {}, key {}", bucket, key);
 
-        ResourceCost cost = task.req.GetContentLength();
-        ResourceGuard rlock(write_settings.resource_link, cost);
-        Stopwatch watch;
-        auto outcome = client_ptr->PutObject(task.req);
-        watch.stop();
-        rlock.unlock();
+        auto & request = std::get<0>(*worker_data);
+        size_t content_length = request.GetContentLength();
 
-        ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, watch.elapsedMicroseconds());
-
-        bool with_pool = static_cast<bool>(schedule);
-        if (outcome.IsSuccess())
-        {
-            LOG_TRACE(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool);
-            return;
-        }
-        else
+        size_t max_retry = std::max(request_settings.max_unexpected_write_error_retries, 1UL);
+        for (size_t i = 0; i < max_retry; ++i)
         {
+            ProfileEvents::increment(ProfileEvents::S3PutObject);
+            if (write_settings.for_object_storage)
+                ProfileEvents::increment(ProfileEvents::DiskS3PutObject);
+
+            ResourceCost cost = request.GetContentLength();
+            ResourceGuard rlock(write_settings.resource_link, cost);
+            Stopwatch watch;
+            auto outcome = client_ptr->PutObject(request);
+            watch.stop();
+            rlock.unlock();
+
+            ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, watch.elapsedMicroseconds());
+
+            if (outcome.IsSuccess())
+            {
+                LOG_TRACE(log, "Single part upload has completed. bucket {}, key {}, object size {}", bucket, key, content_length);
+                return;
+            }
+
             ProfileEvents::increment(ProfileEvents::WriteBufferFromS3RequestsErrors, 1);
+            write_settings.resource_link.accumulate(cost); // We assume no resource was used in case of failure
+
             if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY)
             {
-                write_settings.resource_link.accumulate(cost); // We assume no resource was used in case of failure
+
                 /// For unknown reason, at least MinIO can respond with NO_SUCH_KEY for put requests
-                LOG_INFO(log, "Single part upload failed with NO_SUCH_KEY error for Bucket: {}, Key: {}, Object size: {}, WithPool: {}, will retry", bucket, key, task.req.GetContentLength(), with_pool);
+                LOG_INFO(log, "Single part upload failed with NO_SUCH_KEY error for  bucket {}, key {}, object size {}, will retry", bucket, key, content_length);
             }
             else
             {
-                write_settings.resource_link.accumulate(cost); // We assume no resource was used in case of failure
+                LOG_ERROR(log, "S3Exception name {}, Message: {}, bucket {}, key {}, object size {}",
+                          outcome.GetError().GetExceptionName(), outcome.GetError().GetMessage(), bucket, key, content_length);
                 throw S3Exception(
                     outcome.GetError().GetErrorType(),
-                    "Message: {}, Key: {}, Bucket: {}, Object size: {}, WithPool: {}",
-                    outcome.GetError().GetMessage(), key, bucket, task.req.GetContentLength(), with_pool);
+                    "Message: {}, bucket {}, key {}, object size {}",
+                    outcome.GetError().GetMessage(), bucket, key, content_length);
             }
         }
-    }
 
-    throw S3Exception(
-        Aws::S3::S3Errors::NO_SUCH_KEY,
-        "Message: Single part upload failed with NO_SUCH_KEY error, retries {}, Key: {}, Bucket: {}",
-        max_retry, key, bucket);
-}
+        throw S3Exception(
+            Aws::S3::S3Errors::NO_SUCH_KEY,
+            "Message: Single part upload failed with NO_SUCH_KEY error, retries {}, Key: {}, Bucket: {}",
+            max_retry, key, bucket);
+    };
 
-void WriteBufferFromS3::waitForReadyBackgroundTasks()
-{
-    if (schedule)
-    {
-        std::unique_lock lock(bg_tasks_mutex);
-
-        /// Suppress warnings because bg_tasks_mutex is actually hold, but tsa annotations do not understand std::unique_lock
-        auto & tasks = TSA_SUPPRESS_WARNING_FOR_WRITE(upload_object_tasks);
-
-        while (!tasks.empty() && tasks.front().is_finished)
-        {
-            auto & task = tasks.front();
-            auto exception = task.exception;
-            auto tag = std::move(task.tag);
-            tasks.pop_front();
-
-            if (exception)
-            {
-                waitForAllBackgroundTasksUnlocked(lock);
-                std::rethrow_exception(exception);
-            }
-
-            TSA_SUPPRESS_WARNING_FOR_WRITE(part_tags).push_back(tag);
-        }
-    }
-}
-
-void WriteBufferFromS3::waitForAllBackgroundTasks()
-{
-    if (schedule)
-    {
-        std::unique_lock lock(bg_tasks_mutex);
-        waitForAllBackgroundTasksUnlocked(lock);
-    }
-}
-
-void WriteBufferFromS3::waitForAllBackgroundTasksUnlocked(std::unique_lock<std::mutex> & bg_tasks_lock)
-{
-    if (schedule)
-    {
-        bg_tasks_condvar.wait(bg_tasks_lock, [this]() {return TSA_SUPPRESS_WARNING_FOR_READ(num_added_bg_tasks) == TSA_SUPPRESS_WARNING_FOR_READ(num_finished_bg_tasks); });
-
-        /// Suppress warnings because bg_tasks_mutex is actually hold, but tsa annotations do not understand std::unique_lock
-        auto & tasks = TSA_SUPPRESS_WARNING_FOR_WRITE(upload_object_tasks);
-        while (!tasks.empty())
-        {
-            auto & task = tasks.front();
-
-            if (task.exception)
-                std::rethrow_exception(task.exception);
-
-            TSA_SUPPRESS_WARNING_FOR_WRITE(part_tags).push_back(task.tag);
-
-            tasks.pop_front();
-        }
-
-        if (put_object_task)
-        {
-            bg_tasks_condvar.wait(bg_tasks_lock, [this]() { return put_object_task->is_finished; });
-            if (put_object_task->exception)
-                std::rethrow_exception(put_object_task->exception);
-        }
-    }
+    task_tracker->add(std::move(upload_worker));
 }
 
 }
diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h
index 2374f1502f5..13ed151ad57 100644
--- a/src/IO/WriteBufferFromS3.h
+++ b/src/IO/WriteBufferFromS3.h
@@ -4,20 +4,16 @@
 
 #if USE_AWS_S3
 
-#include <memory>
-#include <vector>
-#include <list>
-
 #include <base/types.h>
 #include <IO/BufferWithOwnMemory.h>
 #include <IO/WriteBuffer.h>
 #include <IO/WriteSettings.h>
-#include <IO/S3/Requests.h>
 #include <Storages/StorageS3Settings.h>
 #include <Interpreters/threadPoolCallbackRunner.h>
 
-#include <aws/core/utils/memory/stl/AWSStringStream.h>
-
+#include <memory>
+#include <vector>
+#include <list>
 
 namespace Aws::S3
 {
@@ -27,8 +23,6 @@ class Client;
 namespace DB
 {
 
-class WriteBufferFromFile;
-
 /**
  * Buffer to write a data to a S3 object with specified bucket and key.
  * If data size written to the buffer is less than 'max_single_part_upload_size' write is performed using singlepart upload.
@@ -45,81 +39,74 @@ public:
         const String & key_,
         const S3Settings::RequestSettings & request_settings_,
         std::optional<std::map<String, String>> object_metadata_ = std::nullopt,
-        size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE,
         ThreadPoolCallbackRunner<void> schedule_ = {},
         const WriteSettings & write_settings_ = {});
 
     ~WriteBufferFromS3() override;
-
     void nextImpl() override;
-
     void preFinalize() override;
 
 private:
-    void allocateBuffer();
-
-    void processWithStrictParts();
-    void processWithDynamicParts();
-
-    void fillCreateMultipartRequest(S3::CreateMultipartUploadRequest & req);
-    void createMultipartUpload();
-    void writePart();
-    void completeMultipartUpload();
-
-    void makeSinglepartUpload();
-
     /// Receives response from the server after sending all data.
     void finalizeImpl() override;
 
-    struct UploadPartTask;
-    void fillUploadRequest(S3::UploadPartRequest & req);
-    void processUploadRequest(UploadPartTask & task);
+    String getLogDetails() const;
 
-    struct PutObjectTask;
-    void fillPutRequest(S3::PutObjectRequest & req);
-    void processPutRequest(const PutObjectTask & task);
+    struct PartData;
+    void hidePartialData();
+    void allocateFirstBuffer();
+    void reallocateFirstBuffer();
+    void detachBuffer();
+    void allocateBuffer();
+    void setFakeBufferWhenPreFinalized();
 
-    void waitForReadyBackgroundTasks();
-    void waitForAllBackgroundTasks();
-    void waitForAllBackgroundTasksUnlocked(std::unique_lock<std::mutex> & bg_tasks_lock);
+    S3::UploadPartRequest getUploadRequest(size_t part_number, PartData & data);
+    void writePart(PartData && data);
+    void writeMultipartUpload();
+    void createMultipartUpload();
+    void completeMultipartUpload();
+    void abortMultipartUpload();
+    void tryToAbortMultipartUpload();
+
+    S3::PutObjectRequest getPutRequest(PartData & data);
+    void makeSinglepartUpload(PartData && data);
 
     const String bucket;
     const String key;
     const S3Settings::RequestSettings request_settings;
     const S3Settings::RequestSettings::PartUploadSettings & upload_settings;
+    const WriteSettings write_settings;
     const std::shared_ptr<const S3::Client> client_ptr;
     const std::optional<std::map<String, String>> object_metadata;
+    Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3");
 
-    /// Strict/static Part size, no adjustments will be done on fly.
-    size_t strict_upload_part_size = 0;
-    /// Part size will be adjusted on fly (for bigger uploads)
-    size_t current_upload_part_size = 0;
-    std::shared_ptr<Aws::StringStream> temporary_buffer; /// Buffer to accumulate data.
-    size_t last_part_size = 0;
-    size_t part_number = 0;
+    struct BufferAllocationPolicy;
+    std::unique_ptr<BufferAllocationPolicy> buffer_allocation_policy;
 
     /// Upload in S3 is made in parts.
     /// We initiate upload, then upload each part and get ETag as a response, and then finalizeImpl() upload with listing all our parts.
     String multipart_upload_id;
-    std::vector<String> TSA_GUARDED_BY(bg_tasks_mutex) part_tags;
+    std::deque<String> multipart_tags;
+    bool multipart_upload_finished = false;
 
+    /// Track that prefinalize() is called only once
     bool is_prefinalized = false;
 
-    /// Following fields are for background uploads in thread pool (if specified).
-    /// We use std::function to avoid dependency of Interpreters
-    const ThreadPoolCallbackRunner<void> schedule;
+    /// First fully filled buffer has to be delayed
+    /// There are two ways after:
+    /// First is to call prefinalize/finalize, which leads to single part upload
+    /// Second is to write more data, which leads to multi part upload
+    std::deque<PartData> detached_part_data;
+    char fake_buffer_when_prefinalized[1] = {};
 
-    std::unique_ptr<PutObjectTask> put_object_task; /// Does not need protection by mutex because of the logic around is_finished field.
-    std::list<UploadPartTask> TSA_GUARDED_BY(bg_tasks_mutex) upload_object_tasks;
-    int num_added_bg_tasks TSA_GUARDED_BY(bg_tasks_mutex) = 0;
-    int num_finished_bg_tasks TSA_GUARDED_BY(bg_tasks_mutex) = 0;
+    /// offset() and count() are unstable inside nextImpl
+    /// For example nextImpl changes position hence offset() and count() is changed
+    /// This vars are dedicated to store information about sizes when offset() and count() are unstable
+    size_t total_size = 0;
+    size_t hidden_size = 0;
 
-    std::mutex bg_tasks_mutex;
-    std::condition_variable bg_tasks_condvar;
-
-    Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3");
-
-    WriteSettings write_settings;
+    class TaskTracker;
+    std::unique_ptr<TaskTracker> task_tracker;
 };
 
 }
diff --git a/src/IO/WriteBufferFromS3MemoryStream.cpp b/src/IO/WriteBufferFromS3MemoryStream.cpp
new file mode 100644
index 00000000000..6271f15f055
--- /dev/null
+++ b/src/IO/WriteBufferFromS3MemoryStream.cpp
@@ -0,0 +1,68 @@
+#include "config.h"
+
+#if USE_AWS_S3
+
+#include <IO/WriteBufferFromS3MemoryStream.h>
+
+namespace DB
+{
+
+MemoryStream::MemoryBuf::MemoryBuf(char * begin_, size_t size_)
+    : begin(begin_)
+    , size(size_)
+{
+    this->setg(begin, begin, begin + size);
+}
+
+MemoryStream::MemoryBuf::int_type MemoryStream::MemoryBuf::underflow()
+{
+    if (gptr() < egptr())
+            return traits_type::to_int_type(*gptr());
+    return traits_type::eof();
+}
+
+MemoryStream::MemoryBuf::pos_type MemoryStream::MemoryBuf::seekoff(off_type off, std::ios_base::seekdir way,
+                 std::ios_base::openmode mode)
+{
+    bool out_mode = (std::ios_base::out & mode) != 0;
+    if (out_mode)
+        return off_type(-1);
+
+    off_type ret(-1);
+
+    if (way == std::ios_base::beg)
+        ret = 0;
+    else if (way == std::ios_base::cur)
+        ret = gptr() - begin;
+    else if (way == std::ios_base::end)
+        ret = size;
+
+    if (ret == off_type(-1))
+        return ret;
+
+    ret += off;
+    if (!(ret >= 0 && size_t(ret) <= size))
+        return off_type(-1);
+
+    this->setg(begin, begin + ret, begin + size);
+
+    return pos_type(ret);
+}
+
+MemoryStream::MemoryBuf::pos_type MemoryStream::MemoryBuf::seekpos(pos_type sp,
+                 std::ios_base::openmode mode)
+{
+    return seekoff(off_type(sp), std::ios_base::beg, mode);
+}
+
+MemoryStream::MemoryStream(char * begin_, size_t size_)
+    : std::iostream(nullptr)
+    , mem_buf(begin_, size_)
+{
+    init(&mem_buf);
+}
+
+}
+
+#endif
+
diff --git a/src/IO/WriteBufferFromS3MemoryStream.h b/src/IO/WriteBufferFromS3MemoryStream.h
new file mode 100644
index 00000000000..5a7cc17705d
--- /dev/null
+++ b/src/IO/WriteBufferFromS3MemoryStream.h
@@ -0,0 +1,39 @@
+#pragma once
+
+#include "config.h"
+
+#if USE_AWS_S3
+
+#include "WriteBufferFromS3.h"
+
+#include <iostream>
+
+namespace DB
+{
+
+struct MemoryStream: std::iostream
+{
+    struct MemoryBuf: std::streambuf
+    {
+            MemoryBuf(char * begin_, size_t size_);
+
+            int_type underflow() override;
+
+            pos_type seekoff(off_type off, std::ios_base::seekdir way,
+                             std::ios_base::openmode mode) override;
+
+            pos_type seekpos(pos_type sp,
+                             std::ios_base::openmode mode) override;
+
+            char * begin = nullptr;
+            size_t size = 0;
+    };
+
+    MemoryStream(char * begin_, size_t size_);
+
+    MemoryBuf mem_buf;
+};
+
+}
+
+#endif
diff --git a/src/IO/WriteBufferFromS3TaskTracker.cpp b/src/IO/WriteBufferFromS3TaskTracker.cpp
new file mode 100644
index 00000000000..0769f7731c2
--- /dev/null
+++ b/src/IO/WriteBufferFromS3TaskTracker.cpp
@@ -0,0 +1,137 @@
+#include "config.h"
+
+#if USE_AWS_S3
+
+#include <IO/WriteBufferFromS3TaskTracker.h>
+
+namespace DB
+{
+
+WriteBufferFromS3::TaskTracker::TaskTracker(ThreadPoolCallbackRunner<void> scheduler_)
+    : is_async(bool(scheduler_))
+    , scheduler(scheduler_ ? std::move(scheduler_) : syncRunner())
+{}
+
+WriteBufferFromS3::TaskTracker::~TaskTracker()
+{
+    safeWaitAll();
+}
+
+ThreadPoolCallbackRunner<void> WriteBufferFromS3::TaskTracker::syncRunner()
+{
+    return [](Callback && callback, int64_t) mutable -> std::future<void>
+    {
+        auto package = std::packaged_task<void()>(std::move(callback));
+        /// No exceptions are propagated, exceptions are packed to future
+        package();
+        return  package.get_future();
+    };
+}
+
+void WriteBufferFromS3::TaskTracker::getReady()
+{
+    LOG_TEST(log, "getReady, in queue {}", futures.size());
+
+    /// Exceptions are propagated
+    auto it = futures.begin();
+    while (it != futures.end())
+    {
+        chassert(it->valid());
+        if (it->wait_for(std::chrono::seconds(0)) != std::future_status::ready)
+        {
+            ++it;
+            continue;
+        }
+
+        try
+        {
+            it->get();
+        } catch (...)
+        {
+            tryLogCurrentException(__PRETTY_FUNCTION__);
+            throw;
+        }
+
+        it = futures.erase(it);
+    }
+
+    LOG_TEST(log, "getReady ended, in queue {}", futures.size());
+}
+
+void WriteBufferFromS3::TaskTracker::getAll()
+{
+    LOG_TEST(log, "getAll, in queue {}", futures.size());
+
+    /// Exceptions are propagated
+    for (auto & future : futures)
+    {
+        try
+        {
+            future.get();
+        } catch (...)
+        {
+            tryLogCurrentException(__PRETTY_FUNCTION__);
+            throw;
+        }
+    }
+    futures.clear();
+}
+
+void WriteBufferFromS3::TaskTracker::safeWaitAll()
+{
+    LOG_TEST(log, "safeWaitAll, wait in queue {}", futures.size());
+
+    /// Exceptions are not propagated
+    for (auto & future : futures)
+    {
+        LOG_TEST(log, "safeWaitAll, wait future");
+
+        if (future.valid())
+            future.wait();
+    }
+
+    LOG_TEST(log, "safeWaitAll, get in queue {}", futures.size());
+
+    for (auto & future : futures)
+    {
+        if (future.valid())
+        {
+            try
+            {
+                future.get();
+            } catch (...)
+            {
+                tryLogCurrentException(__PRETTY_FUNCTION__);
+            }
+        }
+    }
+    futures.clear();
+    LOG_TEST(log, "safeWaitAll ended, get in queue {}", futures.size());
+}
+
+void WriteBufferFromS3::TaskTracker::add(Callback && func)
+{
+    LOG_TEST(log, "add, in queue {}", futures.size());
+
+    auto future = scheduler(std::move(func), 0);
+    auto exit_scope = scope_guard(
+        [&future]()
+        {
+            future.wait();
+        }
+    );
+
+    futures.push_back(std::move(future));
+
+    exit_scope.release();
+    LOG_TEST(log, "add ended, in queue {}", futures.size());
+}
+
+bool WriteBufferFromS3::TaskTracker::isAsync() const
+{
+    return is_async;
+}
+
+}
+
+#endif
diff --git a/src/IO/WriteBufferFromS3TaskTracker.h b/src/IO/WriteBufferFromS3TaskTracker.h
new file mode 100644
index 00000000000..fa214a4f8c5
--- /dev/null
+++ b/src/IO/WriteBufferFromS3TaskTracker.h
@@ -0,0 +1,37 @@
+#pragma once
+
+#include "config.h"
+
+#if USE_AWS_S3
+
+#include "WriteBufferFromS3.h"
+
+namespace DB
+{
+
+class WriteBufferFromS3::TaskTracker
+{
+public:
+    using Callback = std::function<void()>;
+
+    explicit TaskTracker(ThreadPoolCallbackRunner<void> scheduler_);
+    ~TaskTracker();
+
+    static ThreadPoolCallbackRunner<void> syncRunner();
+
+    bool isAsync() const;
+    void getReady();
+    void getAll();
+    void safeWaitAll();
+    void add(Callback && func);
+
+private:
+    bool is_async;
+    ThreadPoolCallbackRunner<void> scheduler;
+    std::list<std::future<void>> futures;
+    Poco::Logger * log = &Poco::Logger::get("TaskTracker");
+};
+
+}
+
+#endif
diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp
new file mode 100644
index 00000000000..d7661d3e3d0
--- /dev/null
+++ b/src/IO/tests/gtest_writebuffer_s3.cpp
@@ -0,0 +1,1114 @@
+#include <gtest/gtest.h>
+
+#include "config.h"
+
+#if USE_AWS_S3
+
+#include <gmock/gmock.h>
+
+#include <aws/core/Aws.h>
+#include <aws/core/auth/AWSCredentials.h>
+#include <aws/core/auth/AWSCredentialsProvider.h>
+#include <aws/core/config/AWSProfileConfigLoader.h>
+
+#include <aws/s3/model/CreateMultipartUploadRequest.h>
+#include <aws/s3/model/CompleteMultipartUploadRequest.h>
+#include <aws/s3/model/AbortMultipartUploadRequest.h>
+#include <aws/s3/model/PutObjectRequest.h>
+#include <aws/s3/model/UploadPartRequest.h>
+#include <aws/s3/model/HeadObjectRequest.h>
+#include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/S3Client.h>
+#include <aws/s3/S3Errors.h>
+
+#include <IO/WriteBufferFromS3.h>
+#include <IO/S3Common.h>
+
+#include <Common/filesystemHelpers.h>
+#include <IO/S3/Client.h>
+#include <Core/Settings.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+    extern const int S3_ERROR;
+}
+
+}
+
+namespace MockS3
+{
+
+class Sequencer
+{
+public:
+    size_t next() { return counter++; }
+    std::string next_id()
+    {
+        std::stringstream ss;
+        ss << "id-" << next();
+        return ss.str();
+    }
+
+private:
+    size_t counter = 0;
+};
+
+class BucketMemStore
+{
+public:
+    typedef std::string Key;
+    typedef std::string Data;
+    typedef std::string ETag;
+    typedef std::string MPU_ID;
+    typedef std::map<ETag, Data> MPUPartsInProgress;
+    typedef std::vector<Data> MPUParts;
+
+
+    std::map<Key, Data> objects;
+    std::map<MPU_ID, MPUPartsInProgress> multiPartUploads;
+    std::vector<std::pair<MPU_ID, MPUParts>> CompletedPartUploads;
+
+    Sequencer sequencer;
+
+    std::string CreateMPU()
+    {
+        auto id = sequencer.next_id();
+        multiPartUploads.emplace(id, MPUPartsInProgress{});
+        return id;
+    }
+
+    std::string UploadPart(const std::string & upload_id, const std::string & part)
+    {
+        auto etag = sequencer.next_id();
+        auto & parts = multiPartUploads.at(upload_id);
+        parts.emplace(etag, part);
+        return etag;
+    }
+
+    void PutObject(const std::string & key, const std::string & data)
+    {
+        objects[key] = data;
+    }
+
+    void CompleteMPU(const std::string & key, const std::string & upload_id, const std::vector<std::string> & etags)
+    {
+        MPUParts completedParts;
+        completedParts.reserve(etags.size());
+
+        auto & parts = multiPartUploads.at(upload_id);
+        for (const auto & tag: etags) {
+            completedParts.push_back(parts.at(tag));
+        }
+
+        std::stringstream file_data;
+        for (const auto & part_data: completedParts) {
+            file_data << part_data;
+        }
+
+        CompletedPartUploads.emplace_back(upload_id, std::move(completedParts));
+        objects[key] = file_data.str();
+        multiPartUploads.erase(upload_id);
+    }
+
+    void AbortMPU(const std::string & upload_id)
+    {
+        multiPartUploads.erase(upload_id);
+    }
+
+
+    const std::vector<std::pair<MPU_ID, MPUParts>> & GetCompletedPartUploads() const
+    {
+        return CompletedPartUploads;
+    }
+
+    static std::vector<size_t> GetPartSizes(const MPUParts & parts)
+    {
+        std::vector<size_t> result;
+        result.reserve(parts.size());
+        for (auto & part_data : parts)
+            result.push_back(part_data.size());
+
+        return result;
+    }
+
+};
+
+class S3MemStrore
+{
+public:
+    void CreateBucket(const std::string & bucket)
+    {
+        assert(buckets.count(bucket) == 0);
+        buckets.emplace(bucket, BucketMemStore{});
+    }
+
+    BucketMemStore& GetBucketStore(const std::string & bucket) {
+        return buckets.at(bucket);
+    }
+
+private:
+    std::map<std::string, BucketMemStore> buckets;
+};
+
+struct EventCounts
+{
+    size_t headObject = 0;
+    size_t getObject = 0;
+    size_t putObject = 0;
+    size_t multiUploadCreate = 0;
+    size_t multiUploadComplete = 0;
+    size_t multiUploadAbort = 0;
+    size_t uploadParts = 0;
+    size_t writtenSize = 0;
+
+    size_t totalRequestsCount() const
+    {
+        return headObject + getObject + putObject + multiUploadCreate + multiUploadComplete + uploadParts;
+    }
+};
+
+struct Client;
+
+struct InjectionModel
+{
+    virtual ~InjectionModel() = default;
+
+#define DeclareInjectCall(ObjectTypePart) \
+    virtual std::optional<Aws::S3::Model::ObjectTypePart##Outcome> call(const Aws::S3::Model::ObjectTypePart##Request & /*request*/) \
+    { \
+        return std::nullopt; \
+    }
+    DeclareInjectCall(PutObject)
+    DeclareInjectCall(HeadObject)
+    DeclareInjectCall(CreateMultipartUpload)
+    DeclareInjectCall(CompleteMultipartUpload)
+    DeclareInjectCall(AbortMultipartUpload)
+    DeclareInjectCall(UploadPart)
+#undef DeclareInjectCall
+};
+
+struct Client : DB::S3::Client
+{
+    Client(std::shared_ptr<S3MemStrore> mock_s3_store)
+        : DB::S3::Client(
+               100,
+               DB::S3::ServerSideEncryptionKMSConfig(),
+               std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>("", ""),
+               GetClientConfiguration(),
+               Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
+               /* useVirtualAddressing = */ true)
+        , store(mock_s3_store)
+    { }
+
+    static std::shared_ptr<Client> CreateClient(String bucket = "mock-s3-bucket")
+    {
+        auto s3store = std::make_shared<S3MemStrore>();
+        s3store->CreateBucket(bucket);
+        return std::make_shared<Client>(s3store);
+    }
+
+    static DB::S3::PocoHTTPClientConfiguration GetClientConfiguration()
+    {
+        DB::RemoteHostFilter remote_host_filter;
+        return DB::S3::ClientFactory::instance().createClientConfiguration(
+            "some-region",
+            remote_host_filter,
+            /* s3_max_redirects = */ 100,
+            /* enable_s3_requests_logging = */ true,
+            /* for_disk_s3 = */ false,
+            /* get_request_throttler = */ {},
+            /* put_request_throttler = */ {}
+        );
+    }
+
+    void setInjectionModel(std::shared_ptr<MockS3::InjectionModel> injections_)
+    {
+        injections = injections_;
+    }
+
+    Aws::S3::Model::PutObjectOutcome PutObject(const Aws::S3::Model::PutObjectRequest & request) const override
+    {
+        ++counters.putObject;
+
+        if (injections)
+        {
+            if (auto opt_val = injections->call(request))
+            {
+                return *opt_val;
+            }
+        }
+
+        auto & bStore = store->GetBucketStore(request.GetBucket());
+        std::stringstream data;
+        data << request.GetBody()->rdbuf();
+        bStore.PutObject(request.GetKey(), data.str());
+        counters.writtenSize += data.str().length();
+
+        Aws::S3::Model::PutObjectOutcome outcome;
+        Aws::S3::Model::PutObjectResult result(outcome.GetResultWithOwnership());
+        return result;
+    }
+
+    Aws::S3::Model::GetObjectOutcome GetObject(const Aws::S3::Model::GetObjectRequest & request) const override
+    {
+        ++counters.getObject;
+
+        auto & bStore = store->GetBucketStore(request.GetBucket());
+
+        auto factory = request.GetResponseStreamFactory();
+        Aws::Utils::Stream::ResponseStream responseStream(factory);
+        responseStream.GetUnderlyingStream() << std::stringstream(bStore.objects[request.GetKey()]).rdbuf();
+
+        Aws::AmazonWebServiceResult<Aws::Utils::Stream::ResponseStream> awsStream(std::move(responseStream), Aws::Http::HeaderValueCollection());
+        Aws::S3::Model::GetObjectResult getObjectResult(std::move(awsStream));
+        return Aws::S3::Model::GetObjectOutcome(std::move(getObjectResult));
+    }
+
+    Aws::S3::Model::HeadObjectOutcome HeadObject(const Aws::S3::Model::HeadObjectRequest & request) const override
+    {
+        ++counters.headObject;
+
+        if (injections)
+        {
+            if (auto opt_val = injections->call(request))
+            {
+                return std::move(*opt_val);
+            }
+        }
+
+        auto & bStore = store->GetBucketStore(request.GetBucket());
+        auto obj = bStore.objects[request.GetKey()];
+        Aws::S3::Model::HeadObjectOutcome outcome;
+        Aws::S3::Model::HeadObjectResult result(outcome.GetResultWithOwnership());
+        result.SetContentLength(obj.length());
+        return result;
+    }
+
+    Aws::S3::Model::CreateMultipartUploadOutcome CreateMultipartUpload(const Aws::S3::Model::CreateMultipartUploadRequest & request) const override
+    {
+        ++counters.multiUploadCreate;
+
+        if (injections)
+        {
+            if (auto opt_val = injections->call(request))
+            {
+                return std::move(*opt_val);
+            }
+        }
+
+        auto & bStore = store->GetBucketStore(request.GetBucket());
+        auto mpu_id = bStore.CreateMPU();
+
+        Aws::S3::Model::CreateMultipartUploadResult result;
+        result.SetUploadId(mpu_id.c_str());
+        return Aws::S3::Model::CreateMultipartUploadOutcome(result);
+    }
+
+    Aws::S3::Model::UploadPartOutcome UploadPart(const Aws::S3::Model::UploadPartRequest & request) const override
+    {
+        ++counters.uploadParts;
+
+        if (injections)
+        {
+            if (auto opt_val = injections->call(request))
+            {
+                return std::move(*opt_val);
+            }
+        }
+
+        std::stringstream data;
+        data << request.GetBody()->rdbuf();
+        counters.writtenSize += data.str().length();
+
+        auto & bStore = store->GetBucketStore(request.GetBucket());
+        auto etag = bStore.UploadPart(request.GetUploadId(), data.str());
+
+        Aws::S3::Model::UploadPartResult result;
+        result.SetETag(etag);
+        return Aws::S3::Model::UploadPartOutcome(result);
+    }
+
+    Aws::S3::Model::CompleteMultipartUploadOutcome CompleteMultipartUpload(const Aws::S3::Model::CompleteMultipartUploadRequest & request) const override
+    {
+        ++counters.multiUploadComplete;
+
+        if (injections)
+        {
+            if (auto opt_val = injections->call(request))
+            {
+                return std::move(*opt_val);
+            }
+        }
+
+        auto & bStore = store->GetBucketStore(request.GetBucket());
+
+        std::vector<std::string> etags;
+        for (const auto & x: request.GetMultipartUpload().GetParts()) {
+            etags.push_back(x.GetETag());
+        }
+        bStore.CompleteMPU(request.GetKey(), request.GetUploadId(), etags);
+
+        Aws::S3::Model::CompleteMultipartUploadResult result;
+        return Aws::S3::Model::CompleteMultipartUploadOutcome(result);
+    }
+
+    Aws::S3::Model::AbortMultipartUploadOutcome AbortMultipartUpload(const Aws::S3::Model::AbortMultipartUploadRequest & request) const override
+    {
+        ++counters.multiUploadAbort;
+
+        if (injections)
+        {
+            if (auto opt_val = injections->call(request))
+            {
+                return std::move(*opt_val);
+            }
+        }
+
+        auto & bStore = store->GetBucketStore(request.GetBucket());
+        bStore.AbortMPU(request.GetUploadId());
+
+        Aws::S3::Model::AbortMultipartUploadResult result;
+        return Aws::S3::Model::AbortMultipartUploadOutcome(result);
+    }
+
+    std::shared_ptr<S3MemStrore> store;
+    mutable EventCounts counters;
+    mutable std::shared_ptr<InjectionModel> injections;
+    void resetCounters() const { counters = {}; }
+};
+
+struct PutObjectFailIngection: InjectionModel
+{
+    std::optional<Aws::S3::Model::PutObjectOutcome> call(const Aws::S3::Model::PutObjectRequest & /*request*/) override
+    {
+        return Aws::Client::AWSError<Aws::Client::CoreErrors>(Aws::Client::CoreErrors::VALIDATION, "FailInjection", "PutObjectFailIngection", false);
+    }
+};
+
+struct HeadObjectFailIngection: InjectionModel
+{
+    std::optional<Aws::S3::Model::HeadObjectOutcome> call(const Aws::S3::Model::HeadObjectRequest & /*request*/) override
+    {
+        return Aws::Client::AWSError<Aws::Client::CoreErrors>(Aws::Client::CoreErrors::VALIDATION, "FailInjection", "HeadObjectFailIngection", false);
+    }
+};
+
+struct CreateMPUFailIngection: InjectionModel
+{
+    std::optional<Aws::S3::Model::CreateMultipartUploadOutcome> call(const Aws::S3::Model::CreateMultipartUploadRequest & /*request*/) override
+    {
+        return Aws::Client::AWSError<Aws::Client::CoreErrors>(Aws::Client::CoreErrors::VALIDATION, "FailInjection", "CreateMPUFailIngection", false);
+    }
+};
+
+struct CompleteMPUFailIngection: InjectionModel
+{
+    std::optional<Aws::S3::Model::CompleteMultipartUploadOutcome> call(const Aws::S3::Model::CompleteMultipartUploadRequest & /*request*/) override
+    {
+        return Aws::Client::AWSError<Aws::Client::CoreErrors>(Aws::Client::CoreErrors::VALIDATION, "FailInjection", "CompleteMPUFailIngection", false);
+    }
+};
+
+struct UploadPartFailIngection: InjectionModel
+{
+    std::optional<Aws::S3::Model::UploadPartOutcome> call(const Aws::S3::Model::UploadPartRequest & /*request*/) override
+    {
+        return Aws::Client::AWSError<Aws::Client::CoreErrors>(Aws::Client::CoreErrors::VALIDATION, "FailInjection", "UploadPartFailIngection", false);
+    }
+};
+
+struct BaseSyncPolicy
+{
+    virtual ~BaseSyncPolicy() = default;
+    virtual DB::ThreadPoolCallbackRunner<void> getScheduler() { return {}; }
+    virtual void execute(size_t = 0) {}
+    virtual void setAutoExecute(bool = true) {}
+
+    virtual size_t size() const { return 0; }
+    virtual bool empty() const { return size() == 0; }
+};
+
+struct SimpleAsyncTasks : BaseSyncPolicy
+{
+    bool auto_execute = false;
+    std::deque<std::packaged_task<void()>> queue;
+
+    virtual DB::ThreadPoolCallbackRunner<void> getScheduler() override
+    {
+        return [this] (std::function<void()> && operation, size_t /*priority*/)
+        {
+            if (auto_execute)
+            {
+                auto task = std::packaged_task<void()>(std::move(operation));
+                task();
+                return task.get_future();
+            }
+
+            queue.emplace_back(std::move(operation));
+            return queue.back().get_future();
+        };
+    }
+
+    virtual void execute(size_t limit = 0) override
+    {
+        if (limit == 0)
+            limit = queue.size();
+
+        while (!queue.empty() && limit)
+        {
+            auto & request = queue.front();
+            request();
+
+            queue.pop_front();
+            --limit;
+        }
+    }
+
+    virtual void setAutoExecute(bool value = true) override
+    {
+        auto_execute = value;
+        if (auto_execute)
+            execute();
+    }
+
+    virtual size_t size() const override { return queue.size(); }
+};
+
+}
+
+using namespace DB;
+
+void writeAsOneBlock(WriteBuffer& buf, size_t size)
+{
+    std::vector<char> data(size, 'a');
+    buf.write(data.data(), data.size());
+}
+
+void writeAsPieces(WriteBuffer& buf, size_t size)
+{
+    size_t ceil = 15ull*1024*1024*1024;
+    size_t piece = 1;
+    size_t written = 0;
+    while (written < size) {
+        size_t len = std::min({piece, size-written, ceil});
+        writeAsOneBlock(buf, len);
+        written += len;
+        piece *= 2;
+    }
+}
+
+class WBS3Test : public ::testing::Test
+{
+public:
+    const String bucket = "WBS3Test-bucket";
+
+    Settings & getSettings()
+    {
+        return settings;
+    }
+
+    MockS3::BaseSyncPolicy & getAsyncPolicy()
+    {
+        return *async_policy;
+    }
+
+    std::unique_ptr<WriteBufferFromS3> getWriteBuffer(String file_name = "file")
+    {
+        S3Settings::RequestSettings request_settings;
+        request_settings.updateFromSettings(settings);
+
+        client->resetCounters();
+
+        getAsyncPolicy().setAutoExecute(false);
+
+        return std::make_unique<WriteBufferFromS3>(
+                    client,
+                    bucket,
+                    file_name,
+                    request_settings,
+                    std::nullopt,
+                    getAsyncPolicy().getScheduler());
+    }
+
+    void setInjectionModel(std::shared_ptr<MockS3::InjectionModel> injections_)
+    {
+        client->setInjectionModel(injections_);
+    }
+
+    void runSimpleScenario(MockS3::EventCounts expected_counters, size_t size)
+    {
+        auto scenario = [&] (std::function<void(WriteBuffer& buf, size_t size)> writeMethod) {
+            auto buffer = getWriteBuffer("file");
+            writeMethod(*buffer, size);
+
+            getAsyncPolicy().setAutoExecute();
+            buffer->finalize();
+
+            expected_counters.writtenSize = size;
+            assertCountersEQ(expected_counters);
+
+            auto & bStore = client->store->GetBucketStore(bucket);
+            auto & data = bStore.objects["file"];
+            ASSERT_EQ(size, data.size());
+            for (char c : data)
+               ASSERT_EQ('a', c);
+        };
+
+        scenario(writeAsOneBlock);
+        scenario(writeAsPieces);
+    }
+
+    void assertCountersEQ(const MockS3::EventCounts & canonical) {
+        const auto & actual = client->counters;
+        ASSERT_EQ(canonical.headObject, actual.headObject);
+        ASSERT_EQ(canonical.getObject, actual.getObject);
+        ASSERT_EQ(canonical.putObject, actual.putObject);
+        ASSERT_EQ(canonical.multiUploadCreate, actual.multiUploadCreate);
+        ASSERT_EQ(canonical.multiUploadComplete, actual.multiUploadComplete);
+        ASSERT_EQ(canonical.multiUploadAbort, actual.multiUploadAbort);
+        ASSERT_EQ(canonical.uploadParts, actual.uploadParts);
+        ASSERT_EQ(canonical.writtenSize, actual.writtenSize);
+    }
+
+    auto getCompletedPartUploads ()
+    {
+         return client->store->GetBucketStore(bucket).GetCompletedPartUploads();
+    }
+
+protected:
+    Settings settings;
+
+    std::shared_ptr<MockS3::Client> client;
+    std::unique_ptr<MockS3::BaseSyncPolicy> async_policy;
+
+    virtual void SetUp() override
+    {
+        client = MockS3::Client::CreateClient(bucket);
+        async_policy = std::make_unique<MockS3::BaseSyncPolicy>();
+    }
+
+    virtual void TearDown() override
+    {
+        client.reset();
+        async_policy.reset();
+    }
+};
+
+class SyncAsync : public WBS3Test, public ::testing::WithParamInterface<bool>
+{
+protected:
+    bool test_with_pool = false;
+
+    virtual void SetUp() override
+    {
+        test_with_pool = GetParam();
+        client = MockS3::Client::CreateClient(bucket);
+        if (test_with_pool)
+            async_policy = std::make_unique<MockS3::SimpleAsyncTasks>();
+         else
+            async_policy = std::make_unique<MockS3::BaseSyncPolicy>();
+    }
+};
+
+INSTANTIATE_TEST_SUITE_P(WBS3
+    , SyncAsync
+    , ::testing::Values(true, false)
+    , [] (const ::testing::TestParamInfo<SyncAsync::ParamType>& info_param) {
+        std::string name = info_param.param ? "async" : "sync";
+        return name;
+  });
+
+TEST_P(SyncAsync, exception_on_head) {
+    setInjectionModel(std::make_shared<MockS3::HeadObjectFailIngection>());
+
+    getSettings().s3_check_objects_after_upload = true;
+
+    EXPECT_THROW({
+        try {
+            auto buffer = getWriteBuffer("exception_on_head_1");
+            buffer->write('A');
+            buffer->next();
+
+            getAsyncPolicy().setAutoExecute();
+            buffer->finalize();
+        }
+        catch( const DB::Exception& e )
+        {
+            ASSERT_EQ(ErrorCodes::S3_ERROR, e.code());
+            EXPECT_THAT(e.what(), testing::HasSubstr("Immediately after upload:"));
+            throw;
+        }
+    }, DB::S3Exception);
+}
+
+TEST_P(SyncAsync, exception_on_put) {
+    setInjectionModel(std::make_shared<MockS3::PutObjectFailIngection>());
+
+    EXPECT_THROW({
+        try
+        {
+            auto buffer = getWriteBuffer("exception_on_put_1");
+            buffer->write('A');
+            buffer->next();
+
+            getAsyncPolicy().setAutoExecute();
+            buffer->finalize();
+        }
+        catch( const DB::Exception& e )
+        {
+            ASSERT_EQ(ErrorCodes::S3_ERROR, e.code());
+            EXPECT_THAT(e.what(), testing::HasSubstr("PutObjectFailIngection"));
+            throw;
+        }
+      }, DB::S3Exception);
+
+    EXPECT_THROW({
+        try {
+            auto buffer = getWriteBuffer("exception_on_put_2");
+            buffer->write('A');
+
+            getAsyncPolicy().setAutoExecute();
+            buffer->finalize();
+        }
+        catch( const DB::Exception& e )
+        {
+            ASSERT_EQ(ErrorCodes::S3_ERROR, e.code());
+            EXPECT_THAT(e.what(), testing::HasSubstr("PutObjectFailIngection"));
+            throw;
+        }
+      }, DB::S3Exception);
+
+    EXPECT_THROW({
+        try {
+            auto buffer = getWriteBuffer("exception_on_put_3");
+            buffer->write('A');
+            getAsyncPolicy().setAutoExecute();
+            buffer->preFinalize();
+
+            getAsyncPolicy().setAutoExecute();
+            buffer->finalize();
+        }
+        catch( const DB::Exception& e )
+        {
+            ASSERT_EQ(ErrorCodes::S3_ERROR, e.code());
+            EXPECT_THAT(e.what(), testing::HasSubstr("PutObjectFailIngection"));
+            throw;
+        }
+      }, DB::S3Exception);
+
+}
+
+TEST_P(SyncAsync, exception_on_create_mpu) {
+    setInjectionModel(std::make_shared<MockS3::CreateMPUFailIngection>());
+
+    getSettings().s3_max_single_part_upload_size = 0; // no single part
+    getSettings().s3_min_upload_part_size = 1; // small parts ara ok
+
+    EXPECT_THROW({
+        try {
+            auto buffer = getWriteBuffer("exception_on_create_mpu_1");
+            buffer->write('A');
+            buffer->next();
+            buffer->write('A');
+            buffer->next();
+
+            getAsyncPolicy().setAutoExecute();
+            buffer->finalize();
+        }
+        catch( const DB::Exception& e )
+        {
+            ASSERT_EQ(ErrorCodes::S3_ERROR, e.code());
+            EXPECT_THAT(e.what(), testing::HasSubstr("CreateMPUFailIngection"));
+            throw;
+        }
+      }, DB::S3Exception);
+
+    EXPECT_THROW({
+        try {
+            auto buffer = getWriteBuffer("exception_on_create_mpu_2");
+            buffer->write('A');
+            buffer->preFinalize();
+
+            getAsyncPolicy().setAutoExecute();
+            buffer->finalize();
+        }
+        catch( const DB::Exception& e )
+        {
+            ASSERT_EQ(ErrorCodes::S3_ERROR, e.code());
+            EXPECT_THAT(e.what(), testing::HasSubstr("CreateMPUFailIngection"));
+            throw;
+        }
+      }, DB::S3Exception);
+
+    EXPECT_THROW({
+        try {
+            auto buffer = getWriteBuffer("exception_on_create_mpu_2");
+            buffer->write('A');
+
+            getAsyncPolicy().setAutoExecute();
+            buffer->finalize();
+        }
+        catch( const DB::Exception& e )
+        {
+            ASSERT_EQ(ErrorCodes::S3_ERROR, e.code());
+            EXPECT_THAT(e.what(), testing::HasSubstr("CreateMPUFailIngection"));
+            throw;
+        }
+      }, DB::S3Exception);
+}
+
+
+TEST_P(SyncAsync, exception_on_complete_mpu) {
+    setInjectionModel(std::make_shared<MockS3::CompleteMPUFailIngection>());
+
+    getSettings().s3_max_single_part_upload_size = 0; // no single part
+    getSettings().s3_min_upload_part_size = 1; // small parts ara ok
+
+    EXPECT_THROW({
+        try {
+            auto buffer = getWriteBuffer("exception_on_complete_mpu_1");
+            buffer->write('A');
+
+            getAsyncPolicy().setAutoExecute();
+            buffer->finalize();
+        }
+        catch(const DB::Exception & e)
+        {
+            ASSERT_EQ(ErrorCodes::S3_ERROR, e.code());
+            EXPECT_THAT(e.what(), testing::HasSubstr("CompleteMPUFailIngection"));
+            throw;
+        }
+      }, DB::S3Exception);
+}
+
+TEST_P(SyncAsync, exception_on_upload_part) {
+    setInjectionModel(std::make_shared<MockS3::UploadPartFailIngection>());
+
+    getSettings().s3_max_single_part_upload_size = 0; // no single part
+    getSettings().s3_min_upload_part_size = 1; // small parts ara ok
+
+    MockS3::EventCounts counters = {.multiUploadCreate = 1, .multiUploadAbort = 1};
+
+    counters.uploadParts = 2;
+
+    EXPECT_THROW({
+        try {
+            auto buffer = getWriteBuffer("exception_on_upload_part_1");
+
+            buffer->write('A');
+            buffer->next();
+            buffer->write('A');
+            buffer->next();
+
+            getAsyncPolicy().setAutoExecute();
+
+            buffer->finalize();
+        }
+        catch(const DB::Exception & e)
+        {
+            assertCountersEQ(counters);
+            ASSERT_EQ(ErrorCodes::S3_ERROR, e.code());
+            EXPECT_THAT(e.what(), testing::HasSubstr("UploadPartFailIngection"));
+            throw;
+        }
+      }, DB::S3Exception);
+
+    EXPECT_THROW({
+        try {
+            auto buffer = getWriteBuffer("exception_on_upload_part_2");
+            getAsyncPolicy().setAutoExecute();
+
+            buffer->write('A');
+            buffer->next();
+
+            buffer->write('A');
+            buffer->next();
+
+            buffer->finalize();
+        }
+        catch(const DB::Exception & e)
+        {
+            assertCountersEQ(counters);
+            ASSERT_EQ(ErrorCodes::S3_ERROR, e.code());
+            EXPECT_THAT(e.what(), testing::HasSubstr("UploadPartFailIngection"));
+            throw;
+        }
+      }, DB::S3Exception);
+
+    counters.uploadParts = 1;
+
+    EXPECT_THROW({
+        try {
+            auto buffer = getWriteBuffer("exception_on_upload_part_3");
+            buffer->write('A');
+
+            buffer->preFinalize();
+
+            getAsyncPolicy().setAutoExecute();
+            buffer->finalize();
+        }
+        catch(const DB::Exception & e)
+        {
+            assertCountersEQ(counters);
+            ASSERT_EQ(ErrorCodes::S3_ERROR, e.code());
+            EXPECT_THAT(e.what(), testing::HasSubstr("UploadPartFailIngection"));
+            throw;
+        }
+      }, DB::S3Exception);
+
+    EXPECT_THROW({
+        try {
+            auto buffer = getWriteBuffer("exception_on_upload_part_4");
+            buffer->write('A');
+
+            getAsyncPolicy().setAutoExecute();
+            buffer->finalize();
+        }
+        catch(const DB::Exception & e)
+        {
+            assertCountersEQ(counters);
+            ASSERT_EQ(ErrorCodes::S3_ERROR, e.code());
+            EXPECT_THAT(e.what(), testing::HasSubstr("UploadPartFailIngection"));
+            throw;
+        }
+      }, DB::S3Exception);
+}
+
+
+TEST_F(WBS3Test, prefinalize_called_multiple_times) {
+#ifdef ABORT_ON_LOGICAL_ERROR
+    GTEST_SKIP() << "this test trigger LOGICAL_ERROR, runs only if ABORT_ON_LOGICAL_ERROR is not defined";
+#else
+    EXPECT_THROW({
+        try {
+            auto buffer = getWriteBuffer("prefinalize_called_multiple_times");
+            buffer->write('A');
+            buffer->next();
+            buffer->preFinalize();
+            buffer->write('A');
+            buffer->next();
+            buffer->preFinalize();
+            buffer->finalize();
+        }
+        catch(const DB::Exception & e)
+        {
+            ASSERT_EQ(ErrorCodes::LOGICAL_ERROR, e.code());
+            EXPECT_THAT(e.what(), testing::HasSubstr("write to prefinalized buffer for S3"));
+            throw;
+        }
+    }, DB::Exception);
+#endif
+}
+
+TEST_P(SyncAsync, empty_file) {
+    getSettings().s3_check_objects_after_upload = true;
+
+    MockS3::EventCounts counters = {.headObject = 2, .putObject = 1};
+    runSimpleScenario(counters, 0);
+}
+
+TEST_P(SyncAsync, manual_next_calls) {
+    getSettings().s3_check_objects_after_upload = true;
+
+    {
+        MockS3::EventCounts counters = {.headObject = 2, .putObject = 1};
+
+        auto buffer = getWriteBuffer("manual_next_calls_1");
+        buffer->next();
+
+        getAsyncPolicy().setAutoExecute();
+        buffer->finalize();
+
+        assertCountersEQ(counters);
+    }
+
+    {
+        MockS3::EventCounts counters = {.headObject = 2, .putObject = 1};
+
+        auto buffer = getWriteBuffer("manual_next_calls_2");
+        buffer->next();
+        buffer->next();
+
+        getAsyncPolicy().setAutoExecute();
+        buffer->finalize();
+
+        assertCountersEQ(counters);
+    }
+
+    {
+        MockS3::EventCounts counters = {.headObject = 2, .putObject = 1, .writtenSize = 1};
+
+        auto buffer = getWriteBuffer("manual_next_calls_3");
+        buffer->next();
+        buffer->write('A');
+        buffer->next();
+
+        getAsyncPolicy().setAutoExecute();
+        buffer->finalize();
+
+        assertCountersEQ(counters);
+    }
+
+    {
+        MockS3::EventCounts counters = {.headObject = 2, .putObject = 1, .writtenSize = 2};
+
+        auto buffer = getWriteBuffer("manual_next_calls_4");
+        buffer->write('A');
+        buffer->next();
+        buffer->write('A');
+        buffer->next();
+        buffer->next();
+
+        getAsyncPolicy().setAutoExecute();
+        buffer->finalize();
+
+        assertCountersEQ(counters);
+     }
+}
+
+TEST_P(SyncAsync, small_file_is_one_put_request) {
+    getSettings().s3_check_objects_after_upload = true;
+
+    {
+        getSettings().s3_max_single_part_upload_size = 1000;
+        getSettings().s3_min_upload_part_size = 10;
+
+        MockS3::EventCounts counters = {.headObject = 2, .putObject = 1};
+
+        runSimpleScenario(counters, 1);
+        runSimpleScenario(counters, getSettings().s3_max_single_part_upload_size-1);
+        runSimpleScenario(counters, getSettings().s3_max_single_part_upload_size);
+        runSimpleScenario(counters, getSettings().s3_max_single_part_upload_size/2);
+    }
+
+    {
+
+        getSettings().s3_max_single_part_upload_size = 10;
+        getSettings().s3_min_upload_part_size = 1000;
+
+        MockS3::EventCounts counters = {.headObject = 2, .putObject = 1};
+
+        runSimpleScenario(counters, 1);
+        runSimpleScenario(counters, getSettings().s3_max_single_part_upload_size-1);
+        runSimpleScenario(counters, getSettings().s3_max_single_part_upload_size);
+        runSimpleScenario(counters, getSettings().s3_max_single_part_upload_size/2);
+    }
+}
+
+TEST_P(SyncAsync, little_bigger_file_is_multi_part_upload) {
+    getSettings().s3_check_objects_after_upload = true;
+
+    {
+        getSettings().s3_max_single_part_upload_size = 1000;
+        getSettings().s3_min_upload_part_size = 10;
+
+        MockS3::EventCounts counters = {.headObject = 2, .multiUploadCreate = 1, .multiUploadComplete = 1, .uploadParts = 2};
+        runSimpleScenario(counters, settings.s3_max_single_part_upload_size + 1);
+
+        counters.uploadParts = 101;
+        runSimpleScenario(counters, 2*settings.s3_max_single_part_upload_size);
+    }
+
+    {
+        getSettings().s3_max_single_part_upload_size = 10;
+        getSettings().s3_min_upload_part_size = 1000;
+
+        MockS3::EventCounts counters = {.headObject = 2, .multiUploadCreate = 1, .multiUploadComplete = 1, .uploadParts = 1};
+
+        runSimpleScenario(counters, settings.s3_max_single_part_upload_size + 1);
+        runSimpleScenario(counters, 2*settings.s3_max_single_part_upload_size);
+        runSimpleScenario(counters, settings.s3_min_upload_part_size-1);
+        runSimpleScenario(counters, settings.s3_min_upload_part_size);
+    }
+}
+
+TEST_P(SyncAsync, bigger_file_is_multi_part_upload) {
+    getSettings().s3_check_objects_after_upload = true;
+
+    {
+        getSettings().s3_max_single_part_upload_size = 1000;
+        getSettings().s3_min_upload_part_size = 10;
+
+        auto counters = MockS3::EventCounts{.headObject = 2, .multiUploadCreate = 1, .multiUploadComplete = 1, .uploadParts = 2};
+        runSimpleScenario(counters, settings.s3_max_single_part_upload_size + settings.s3_min_upload_part_size);
+
+        counters.uploadParts = 3;
+        runSimpleScenario(counters, settings.s3_max_single_part_upload_size + settings.s3_min_upload_part_size + 1);
+        runSimpleScenario(counters, settings.s3_max_single_part_upload_size + 2*settings.s3_min_upload_part_size - 1);
+        runSimpleScenario(counters, settings.s3_max_single_part_upload_size + 2*settings.s3_min_upload_part_size);
+    }
+
+
+    {
+        // but not in that case, when s3_min_upload_part_size > s3_max_single_part_upload_size
+        getSettings().s3_max_single_part_upload_size = 10;
+        getSettings().s3_min_upload_part_size = 1000;
+
+        auto counters = MockS3::EventCounts{.headObject = 2, .multiUploadCreate = 1, .multiUploadComplete = 1, .uploadParts = 2};
+        runSimpleScenario(counters, settings.s3_max_single_part_upload_size + settings.s3_min_upload_part_size);
+        runSimpleScenario(counters, settings.s3_max_single_part_upload_size + settings.s3_min_upload_part_size + 1);
+        runSimpleScenario(counters, 2*settings.s3_min_upload_part_size-1);
+        runSimpleScenario(counters, 2*settings.s3_min_upload_part_size);
+
+        counters.uploadParts = 3;
+        runSimpleScenario(counters, 2*settings.s3_min_upload_part_size+1);
+    }
+}
+
+TEST_P(SyncAsync, increase_upload_buffer) {
+    getSettings().s3_check_objects_after_upload = true;
+
+    {
+        getSettings().s3_max_single_part_upload_size = 10;
+        getSettings().s3_min_upload_part_size = 10;
+        getSettings().s3_upload_part_size_multiply_parts_count_threshold = 1;
+        // parts: 10 20 40 80  160
+        // size:  10 30 70 150 310
+
+        auto counters = MockS3::EventCounts{.headObject = 2, .multiUploadCreate = 1, .multiUploadComplete = 1, .uploadParts = 6};
+        runSimpleScenario(counters, 350);
+
+        auto actual_parts_sizes = MockS3::BucketMemStore::GetPartSizes(getCompletedPartUploads().back().second);
+        ASSERT_THAT(actual_parts_sizes, testing::ElementsAre(10, 20, 40, 80, 160, 40));
+    }
+
+    {
+        getSettings().s3_max_single_part_upload_size = 10;
+        getSettings().s3_min_upload_part_size = 10;
+        getSettings().s3_upload_part_size_multiply_parts_count_threshold = 2;
+        getSettings().s3_upload_part_size_multiply_factor = 3;
+        // parts: 10 10 30 30 90
+        // size:  10 20 50 80 170
+
+        auto counters = MockS3::EventCounts{.headObject = 2, .multiUploadCreate = 1, .multiUploadComplete = 1, .uploadParts = 6};
+        runSimpleScenario(counters, 190);
+
+        auto actual_parts_sizes = MockS3::BucketMemStore::GetPartSizes(getCompletedPartUploads().back().second);
+        ASSERT_THAT(actual_parts_sizes, testing::ElementsAre(10, 10, 30, 30, 90, 20));
+    }
+}
+
+TEST_P(SyncAsync, increase_limited) {
+    getSettings().s3_check_objects_after_upload = true;
+
+    {
+        getSettings().s3_max_single_part_upload_size = 10;
+        getSettings().s3_min_upload_part_size = 10;
+        getSettings().s3_upload_part_size_multiply_parts_count_threshold = 1;
+        getSettings().s3_max_upload_part_size = 45;
+        // parts: 10 20 40 45  45  45
+        // size:  10 30 70 115 160 205
+
+        auto counters = MockS3::EventCounts{.headObject = 2, .multiUploadCreate = 1, .multiUploadComplete = 1, .uploadParts = 7};
+        runSimpleScenario(counters, 220);
+
+        auto actual_parts_sizes = MockS3::BucketMemStore::GetPartSizes(getCompletedPartUploads().back().second);
+        ASSERT_THAT(actual_parts_sizes, testing::ElementsAre(10, 20, 40, 45, 45, 45, 15));
+    }
+}
+
+#endif
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 9adfcf7fef7..f61c1dad59f 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -290,6 +290,7 @@ void MergeTreeData::initializeDirectoriesAndFormatVersion(const std::string & re
             {
                 auto buf = disk->writeFile(format_version_path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite, getContext()->getWriteSettings());
                 writeIntText(format_version.toUnderType(), *buf);
+                buf->finalize();
                 if (getContext()->getSettingsRef().fsync_metadata)
                     buf->sync();
             }
diff --git a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp
index b843ce6a078..6c6a6ded5dd 100644
--- a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp
+++ b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp
@@ -160,7 +160,10 @@ void MergeTreeDeduplicationLog::rotate()
     existing_logs.emplace(current_log_number, log_description);
 
     if (current_writer)
+    {
+        current_writer->finalize();
         current_writer->sync();
+    }
 
     current_writer = disk->writeFile(log_description.path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Append);
 }
diff --git a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp
index 2e30a3f3986..feffffb57ea 100644
--- a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp
+++ b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp
@@ -75,6 +75,7 @@ MergeTreeMutationEntry::MergeTreeMutationEntry(MutationCommands commands_, DiskP
             TransactionID::write(tid, *out);
             *out << "\n";
         }
+        out->finalize();
         out->sync();
     }
     catch (...)
diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index 00e72482a17..a4d9dc9f2e3 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -777,7 +777,6 @@ public:
                 key,
                 configuration_.request_settings,
                 std::nullopt,
-                DBMS_DEFAULT_BUFFER_SIZE,
                 threadPoolCallbackRunner<void>(IOThreadPool::get(), "S3ParallelWrite"),
                 context->getWriteSettings()),
             compression_method,
diff --git a/tests/queries/0_stateless/02240_filesystem_query_cache.reference b/tests/queries/0_stateless/02240_filesystem_query_cache.reference
index f4b9f7bb127..16c4cd1c049 100644
--- a/tests/queries/0_stateless/02240_filesystem_query_cache.reference
+++ b/tests/queries/0_stateless/02240_filesystem_query_cache.reference
@@ -6,6 +6,7 @@ SET skip_download_if_exceeds_query_cache=1;
 SET filesystem_cache_max_download_size=128;
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_4', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false;
+SYSTEM DROP FILESYSTEM CACHE;
 INSERT INTO test SELECT number, toString(number) FROM numbers(100);
 SELECT  * FROM test FORMAT Null;
 SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_end, size;
diff --git a/tests/queries/0_stateless/02240_filesystem_query_cache.sql b/tests/queries/0_stateless/02240_filesystem_query_cache.sql
index 94eb4bc5ccd..44856a2188c 100644
--- a/tests/queries/0_stateless/02240_filesystem_query_cache.sql
+++ b/tests/queries/0_stateless/02240_filesystem_query_cache.sql
@@ -9,8 +9,8 @@ SET filesystem_cache_max_download_size=128;
 
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_4', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false;
+SYSTEM DROP FILESYSTEM CACHE;
 INSERT INTO test SELECT number, toString(number) FROM numbers(100);
-
 SELECT  * FROM test FORMAT Null;
 SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_end, size;
 SYSTEM DROP FILESYSTEM CACHE;
diff --git a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference
index f3fac9b32d3..b3b7d12d219 100644
--- a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference
+++ b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference
@@ -1,6 +1,6 @@
 Using storage policy: s3_cache
 0
-0
+0	0
 Row 1:
 ──────
 file_segment_range_begin: 0
@@ -8,11 +8,11 @@ file_segment_range_end:   745
 size:                     746
 state:                    DOWNLOADED
 8
-8
+8	1100
 0
 2
 2
-8
+8	1100
 Row 1:
 ──────
 file_segment_range_begin: 0
@@ -20,17 +20,17 @@ file_segment_range_end:   1659
 size:                     1660
 state:                    DOWNLOADED
 8
-8
-8
-8
-24
-35
-43
+8	2014
+8	2014
+8	2014
+24	84045
+35	168815
+44	252113
 5010500
 18816
 Using storage policy: local_cache
 0
-0
+0	0
 Row 1:
 ──────
 file_segment_range_begin: 0
@@ -38,11 +38,11 @@ file_segment_range_end:   745
 size:                     746
 state:                    DOWNLOADED
 8
-8
+8	1100
 0
 2
 2
-8
+8	1100
 Row 1:
 ──────
 file_segment_range_begin: 0
@@ -50,11 +50,11 @@ file_segment_range_end:   1659
 size:                     1660
 state:                    DOWNLOADED
 8
-8
-8
-8
-24
-35
-43
+8	2014
+8	2014
+8	2014
+24	84045
+35	168815
+44	252113
 5010500
 18816
diff --git a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh
index 048fb792e6e..e65bf9cb35f 100755
--- a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh
+++ b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh
@@ -33,7 +33,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache'; do
     FORMAT Vertical"
 
     $CLICKHOUSE_CLIENT --query "SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path"
-    $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
+    $CLICKHOUSE_CLIENT --query "SELECT count(), sum(size) FROM system.filesystem_cache"
 
     $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100)"
 
@@ -54,7 +54,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache'; do
     FORMAT Vertical"
 
     $CLICKHOUSE_CLIENT --query "SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path"
-    $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
+    $CLICKHOUSE_CLIENT --query "SELECT count(), sum(size) FROM system.filesystem_cache"
 
     $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache WHERE cache_hits > 0"
 
@@ -64,7 +64,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache'; do
     $CLICKHOUSE_CLIENT --query "SELECT * FROM test_02241 FORMAT Null"
     $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache WHERE cache_hits > 0"
 
-    $CLICKHOUSE_CLIENT --query "SELECT count() size FROM system.filesystem_cache"
+    $CLICKHOUSE_CLIENT --query "SELECT count(), sum(size) size FROM system.filesystem_cache"
 
     $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE"
 
@@ -87,24 +87,23 @@ for STORAGE_POLICY in 's3_cache' 'local_cache'; do
     FORMAT Vertical;"
 
     $CLICKHOUSE_CLIENT --query "SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path"
-    $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
+    $CLICKHOUSE_CLIENT --query "SELECT count(), sum(size) FROM system.filesystem_cache"
 
-    $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
+    $CLICKHOUSE_CLIENT --query "SELECT count(), sum(size) FROM system.filesystem_cache"
     $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100) SETTINGS enable_filesystem_cache_on_write_operations=0"
-    $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
+    $CLICKHOUSE_CLIENT --query "SELECT count(), sum(size) FROM system.filesystem_cache"
 
     $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100)"
     $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(300, 10000)"
-    $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
+    $CLICKHOUSE_CLIENT --query "SELECT count(), sum(size) FROM system.filesystem_cache"
 
     $CLICKHOUSE_CLIENT --query "SYSTEM START MERGES test_02241"
 
     $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=1 --query "OPTIMIZE TABLE test_02241 FINAL"
-    $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
+    $CLICKHOUSE_CLIENT --query "SELECT count(), sum(size) FROM system.filesystem_cache"
 
     $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=1 --mutations_sync=2 --query "ALTER TABLE test_02241 UPDATE value = 'kek' WHERE key = 100"
-    $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
-
+    $CLICKHOUSE_CLIENT --query "SELECT count(), sum(size) FROM system.filesystem_cache"
     $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(5000000)"
 
     $CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS"
diff --git a/tests/queries/0_stateless/02382_filesystem_cache_persistent_files.reference b/tests/queries/0_stateless/02382_filesystem_cache_persistent_files.reference
index 083f0f69dc8..e77afc98007 100644
--- a/tests/queries/0_stateless/02382_filesystem_cache_persistent_files.reference
+++ b/tests/queries/0_stateless/02382_filesystem_cache_persistent_files.reference
@@ -8,7 +8,7 @@ SYSTEM STOP MERGES nopers;
 INSERT INTO nopers SELECT number, toString(number) FROM numbers(10);
 SELECT * FROM nopers FORMAT Null;
 SELECT sum(size) FROM system.filesystem_cache;
-194
+195
 SELECT extract(local_path, '.*/([\w.]+)') as file, extract(cache_path, '.*/([\w.]+)') as cache, size
 FROM
 (
@@ -21,17 +21,18 @@ ON data_paths.cache_path = caches.cache_path
 ORDER BY file, cache, size;
 data.bin	0	114
 data.mrk3	0	80
+format_version.txt	0	1
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_small', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false;
 SYSTEM STOP MERGES test;
 INSERT INTO test SELECT number, toString(number) FROM numbers(100);
 SELECT * FROM test FORMAT Null;
 SELECT sum(size) FROM system.filesystem_cache;
-1020
+1021
 SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path;
-4
+5
 SELECT count() FROM system.filesystem_cache;
-4
+5
 SELECT extract(local_path, '.*/([\w.]+)') as file, extract(cache_path, '.*/([\w.]+)') as cache, size
 FROM
 (
@@ -46,17 +47,18 @@ data.bin	0	114
 data.bin	0	746
 data.mrk3	0	80
 data.mrk3	0_persistent	80
+format_version.txt	0	1
 DROP TABLE IF EXISTS test2;
 CREATE TABLE test2 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_small', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false;
 SYSTEM STOP MERGES test2;
 INSERT INTO test2 SELECT number, toString(number) FROM numbers(100000);
 SELECT * FROM test2 FORMAT Null;
 SELECT sum(size) FROM system.filesystem_cache;
-794
+795
 SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path;
-4
+5
 SELECT count() FROM system.filesystem_cache;
-4
+5
 SELECT extract(local_path, '.*/([\w.]+)') as file, extract(cache_path, '.*/([\w.]+)') as cache, size
 FROM
 (
@@ -71,6 +73,7 @@ data.bin	0	114
 data.mrk3	0	80
 data.mrk3	0_persistent	80
 data.mrk3	0_persistent	520
+format_version.txt	0	1
 DROP TABLE test;
 DROP TABLE test2;
 DROP TABLE nopers;
diff --git a/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.reference b/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.reference
index 00e93b1db3d..3f34d5e2c79 100644
--- a/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.reference
+++ b/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.reference
@@ -1,8 +1,8 @@
 INSERT TO S3
  [ 0 ] S3CompleteMultipartUpload: 1 
  [ 0 ] S3CreateMultipartUpload: 1 
- [ 0 ] S3HeadObject: 1 
- [ 0 ] S3ReadRequestsCount: 1 
+ [ 0 ] S3HeadObject: 2 
+ [ 0 ] S3ReadRequestsCount: 2 
  [ 0 ] S3UploadPart: 1 
  [ 0 ] S3WriteRequestsCount: 3 
 CHECK WITH query_log

From cd449cca38ce8bb831a75822921dd9f35c649562 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Fri, 5 May 2023 13:18:42 +0200
Subject: [PATCH 404/689] WriteBufferFromS3BufferAllocationPolicy for FixedSize
 and Exp policy

---
 src/IO/WriteBufferFromS3.cpp                  |  80 +------------
 src/IO/WriteBufferFromS3.h                    |   5 +-
 ...riteBufferFromS3BufferAllocationPolicy.cpp | 108 ++++++++++++++++++
 .../WriteBufferFromS3BufferAllocationPolicy.h |  26 +++++
 src/IO/WriteBufferFromS3MemoryStream.h        |   2 -
 ...02720_s3_strict_upload_part_size.reference |   8 +-
 .../02720_s3_strict_upload_part_size.sh       |   2 +-
 7 files changed, 144 insertions(+), 87 deletions(-)
 create mode 100644 src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp
 create mode 100644 src/IO/WriteBufferFromS3BufferAllocationPolicy.h

diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp
index 5630ed2cb68..73d78cb13be 100644
--- a/src/IO/WriteBufferFromS3.cpp
+++ b/src/IO/WriteBufferFromS3.cpp
@@ -74,80 +74,6 @@ struct WriteBufferFromS3::PartData
     }
 };
 
-struct WriteBufferFromS3::BufferAllocationPolicy
-{
-    size_t first_size = 0;
-    size_t second_size = 0;
-
-    size_t multiply_factor = 0;
-    size_t multiply_threshold = 0;
-    size_t max_size = 0;
-
-    size_t current_size = 0;
-    size_t buffer_number = 0;
-
-    explicit BufferAllocationPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings_)
-        : first_size(std::max(settings_.max_single_part_upload_size, settings_.min_upload_part_size))
-        , second_size(settings_.min_upload_part_size)
-        , multiply_factor(settings_.upload_part_size_multiply_factor)
-        , multiply_threshold(settings_.upload_part_size_multiply_parts_count_threshold)
-        , max_size(settings_.max_upload_part_size)
-    {
-        if (settings_.strict_upload_part_size > 0)
-        {
-            first_size = settings_.strict_upload_part_size;
-            second_size = settings_.strict_upload_part_size;
-            multiply_factor = 1;
-            multiply_threshold = 10000;
-            max_size = settings_.max_upload_part_size;
-        }
-        else
-        {
-            first_size = std::max(settings_.max_single_part_upload_size, settings_.min_upload_part_size);
-            second_size = settings_.min_upload_part_size;
-            multiply_factor = settings_.upload_part_size_multiply_factor;
-            multiply_threshold = settings_.upload_part_size_multiply_parts_count_threshold;
-            max_size = settings_.max_upload_part_size;
-        }
-
-        chassert(first_size > 0);
-        chassert(second_size > 0);
-        chassert(multiply_factor >= 1);
-        chassert(multiply_threshold > 0);
-        chassert(max_size > 0);
-    }
-
-    size_t getNumber() const
-    {
-        return buffer_number;
-    }
-
-    size_t getSize() const
-    {
-        chassert(buffer_number > 0);
-        return current_size;
-    }
-
-    void next()
-    {
-        ++buffer_number;
-
-        if (1 == buffer_number)
-        {
-            current_size = first_size;
-            return;
-        }
-
-        if (2 == buffer_number)
-            current_size = second_size;
-
-        if (0 == ((buffer_number-1) % multiply_threshold))
-        {
-            current_size *= multiply_factor;
-            current_size = std::min(current_size, max_size);
-        }
-    }
-};
 
 WriteBufferFromS3::WriteBufferFromS3(
     std::shared_ptr<const S3::Client> client_ptr_,
@@ -164,7 +90,7 @@ WriteBufferFromS3::WriteBufferFromS3(
     , write_settings(write_settings_)
     , client_ptr(std::move(client_ptr_))
     , object_metadata(std::move(object_metadata_))
-    , buffer_allocation_policy(std::make_unique<BufferAllocationPolicy>(request_settings_.getUploadSettings()))
+    , buffer_allocation_policy(ChooseBufferPolicy(request_settings_.getUploadSettings()))
     , task_tracker(std::make_unique<WriteBufferFromS3::TaskTracker>(std::move(schedule_)))
 {
     LOG_TRACE(log, "Create WriteBufferFromS3, {}", getLogDetails());
@@ -488,7 +414,7 @@ void WriteBufferFromS3::abortMultipartUpload()
 S3::UploadPartRequest WriteBufferFromS3::getUploadRequest(size_t part_number, PartData & data)
 {
     ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Bytes, data.data_size);
-    LOG_TRACE(log, "fillUploadRequest, size {}, key: {}", data.data_size, key);
+    LOG_TRACE(log, "getUploadRequest, size {}, key: {}", data.data_size, key);
 
     S3::UploadPartRequest req;
 
@@ -515,7 +441,7 @@ void WriteBufferFromS3::writePart(WriteBufferFromS3::PartData && data)
 
     multipart_tags.push_back({});
     size_t part_number = multipart_tags.size();
-    LOG_TRACE(log, "WritePart. {}, part size: {}, part number: {}", getLogDetails(), data.data_size, part_number);
+    LOG_TRACE(log, "writePart {}, part size: {}, part number: {}", getLogDetails(), data.data_size, part_number);
 
     if (multipart_upload_id.empty())
         throw Exception(
diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h
index 13ed151ad57..b0d8d329589 100644
--- a/src/IO/WriteBufferFromS3.h
+++ b/src/IO/WriteBufferFromS3.h
@@ -8,6 +8,7 @@
 #include <IO/BufferWithOwnMemory.h>
 #include <IO/WriteBuffer.h>
 #include <IO/WriteSettings.h>
+#include <IO/WriteBufferFromS3BufferAllocationPolicy.h>
 #include <Storages/StorageS3Settings.h>
 #include <Interpreters/threadPoolCallbackRunner.h>
 
@@ -22,7 +23,6 @@ class Client;
 
 namespace DB
 {
-
 /**
  * Buffer to write a data to a S3 object with specified bucket and key.
  * If data size written to the buffer is less than 'max_single_part_upload_size' write is performed using singlepart upload.
@@ -80,8 +80,7 @@ private:
     const std::optional<std::map<String, String>> object_metadata;
     Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3");
 
-    struct BufferAllocationPolicy;
-    std::unique_ptr<BufferAllocationPolicy> buffer_allocation_policy;
+    IBufferAllocationPolicyPtr buffer_allocation_policy;
 
     /// Upload in S3 is made in parts.
     /// We initiate upload, then upload each part and get ETag as a response, and then finalizeImpl() upload with listing all our parts.
diff --git a/src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp b/src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp
new file mode 100644
index 00000000000..1e9b209087c
--- /dev/null
+++ b/src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp
@@ -0,0 +1,108 @@
+#include "config.h"
+
+#if USE_AWS_S3
+
+#include <IO/WriteBufferFromS3BufferAllocationPolicy.h>
+
+namespace
+{
+
+struct FixedSizeBufferAllocationPolicy : DB::IBufferAllocationPolicy
+{
+    const size_t size = 0;
+    size_t buffer_number = 0;
+
+    explicit FixedSizeBufferAllocationPolicy(const DB::S3Settings::RequestSettings::PartUploadSettings & settings_)
+        : size(settings_.strict_upload_part_size)
+    {
+        chassert(size > 0);
+    }
+
+    size_t getNumber() const override { return buffer_number; }
+
+    size_t getSize() const override
+    {
+        chassert(buffer_number > 0);
+        return size;
+    }
+
+    void next() override
+    {
+        ++buffer_number;
+    }
+};
+
+
+struct ExpBufferAllocationPolicy : DB::IBufferAllocationPolicy
+{
+    const size_t first_size = 0;
+    const size_t second_size = 0;
+
+    const size_t multiply_factor = 0;
+    const size_t multiply_threshold = 0;
+    const size_t max_size = 0;
+
+    size_t current_size = 0;
+    size_t buffer_number = 0;
+
+    explicit ExpBufferAllocationPolicy(const DB::S3Settings::RequestSettings::PartUploadSettings & settings_)
+        : first_size(std::max(settings_.max_single_part_upload_size, settings_.min_upload_part_size))
+        , second_size(settings_.min_upload_part_size)
+        , multiply_factor(settings_.upload_part_size_multiply_factor)
+        , multiply_threshold(settings_.upload_part_size_multiply_parts_count_threshold)
+        , max_size(settings_.max_upload_part_size)
+    {
+        chassert(first_size > 0);
+        chassert(second_size > 0);
+        chassert(multiply_factor >= 1);
+        chassert(multiply_threshold > 0);
+        chassert(max_size > 0);
+    }
+
+    size_t getNumber() const override { return buffer_number; }
+
+    size_t getSize() const override
+    {
+        chassert(buffer_number > 0);
+        return current_size;
+    }
+
+    void next() override
+    {
+        ++buffer_number;
+
+        if (1 == buffer_number)
+        {
+            current_size = first_size;
+            return;
+        }
+
+        if (2 == buffer_number)
+            current_size = second_size;
+
+        if (0 == ((buffer_number - 1) % multiply_threshold))
+        {
+            current_size *= multiply_factor;
+            current_size = std::min(current_size, max_size);
+        }
+    }
+};
+
+}
+
+namespace DB
+{
+
+IBufferAllocationPolicy::~IBufferAllocationPolicy() { }
+
+IBufferAllocationPolicyPtr ChooseBufferPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings_)
+{
+    if (settings_.strict_upload_part_size > 0)
+        return std::make_unique<FixedSizeBufferAllocationPolicy>(settings_);
+    else
+        return std::make_unique<ExpBufferAllocationPolicy>(settings_);
+}
+
+}
+
+#endif
diff --git a/src/IO/WriteBufferFromS3BufferAllocationPolicy.h b/src/IO/WriteBufferFromS3BufferAllocationPolicy.h
new file mode 100644
index 00000000000..1ee7c982ed2
--- /dev/null
+++ b/src/IO/WriteBufferFromS3BufferAllocationPolicy.h
@@ -0,0 +1,26 @@
+#pragma once
+
+#include "config.h"
+
+#if USE_AWS_S3
+
+#include <Storages/StorageS3Settings.h>
+
+namespace DB
+{
+
+struct IBufferAllocationPolicy
+{
+    virtual size_t getNumber() const = 0;
+    virtual size_t getSize() const = 0;
+    virtual void next() = 0;
+    virtual ~IBufferAllocationPolicy() = 0;
+};
+
+using IBufferAllocationPolicyPtr = std::unique_ptr<IBufferAllocationPolicy>;
+
+IBufferAllocationPolicyPtr ChooseBufferPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings_);
+
+}
+
+#endif
diff --git a/src/IO/WriteBufferFromS3MemoryStream.h b/src/IO/WriteBufferFromS3MemoryStream.h
index 5a7cc17705d..e9606798910 100644
--- a/src/IO/WriteBufferFromS3MemoryStream.h
+++ b/src/IO/WriteBufferFromS3MemoryStream.h
@@ -4,8 +4,6 @@
 
 #if USE_AWS_S3
 
-#include "WriteBufferFromS3.h"
-
 #include <iostream>
 
 namespace DB
diff --git a/tests/queries/0_stateless/02720_s3_strict_upload_part_size.reference b/tests/queries/0_stateless/02720_s3_strict_upload_part_size.reference
index 360b484bf28..f7c4ece5f1f 100644
--- a/tests/queries/0_stateless/02720_s3_strict_upload_part_size.reference
+++ b/tests/queries/0_stateless/02720_s3_strict_upload_part_size.reference
@@ -1,4 +1,4 @@
-Size: 6000001
-Size: 6000001
-Size: 6000001
-Size: 2971517
+part size: 6000001, part number: 1
+part size: 6000001, part number: 2
+part size: 6000001, part number: 3
+part size: 2971517, part number: 4
diff --git a/tests/queries/0_stateless/02720_s3_strict_upload_part_size.sh b/tests/queries/0_stateless/02720_s3_strict_upload_part_size.sh
index 69e2f734914..9799ef0478a 100755
--- a/tests/queries/0_stateless/02720_s3_strict_upload_part_size.sh
+++ b/tests/queries/0_stateless/02720_s3_strict_upload_part_size.sh
@@ -19,7 +19,7 @@ $CLICKHOUSE_LOCAL -q "SELECT randomPrintableASCII(1023) FROM numbers(20*1024) FO
 
 $CLICKHOUSE_CLIENT --send_logs_level=trace --server_logs_file="$log" -q "INSERT INTO FUNCTION s3(s3_conn, filename='$CLICKHOUSE_TEST_UNIQUE_NAME', format='LineAsString', structure='line String') FORMAT LineAsString" --s3_strict_upload_part_size=6000001 < "$in"
 grep -F '<Fatal>' "$log" || :
-grep -o 'WriteBufferFromS3: Writing part.*Size: .*' "$log" | grep -o 'Size: .*'
+grep -o 'WriteBufferFromS3: writePart.*, part size: .*' "$log" | grep -o 'part size: .*'
 $CLICKHOUSE_CLIENT -q "SELECT * FROM s3(s3_conn, filename='$CLICKHOUSE_TEST_UNIQUE_NAME', format='LineAsString', structure='line String') FORMAT LineAsString" > "$out"
 
 diff -q "$in" "$out"

From c8028bfd7fc83ec64aeb65f75fa7f85b05225fb7 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Fri, 5 May 2023 15:46:52 +0000
Subject: [PATCH 405/689] ajust 02240_system_filesystem_cache_table

---
 ...40_system_filesystem_cache_table.reference | 24 ++++++++++++-------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference b/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference
index cf2bf5fb521..f960b4eb21c 100644
--- a/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference
+++ b/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference
@@ -1,13 +1,15 @@
 Using storage policy: s3_cache
 0
 Expect cache
+DOWNLOADED	0	0	1
 DOWNLOADED	0	79	80
 DOWNLOADED	0	745	746
-2
+3
 Expect cache
+DOWNLOADED	0	0	1
 DOWNLOADED	0	79	80
 DOWNLOADED	0	745	746
-2
+3
 Expect no cache
 Expect cache
 DOWNLOADED	0	79	80
@@ -15,13 +17,15 @@ DOWNLOADED	0	745	746
 2
 Expect no cache
 Expect cache
+DOWNLOADED	0	0	1
 DOWNLOADED	0	79	80
 DOWNLOADED	0	745	746
-2
+3
 Expect cache
+DOWNLOADED	0	0	1
 DOWNLOADED	0	79	80
 DOWNLOADED	0	745	746
-2
+3
 Expect no cache
 Expect cache
 DOWNLOADED	0	79	80
@@ -31,13 +35,15 @@ Expect no cache
 Using storage policy: local_cache
 0
 Expect cache
+DOWNLOADED	0	0	1
 DOWNLOADED	0	79	80
 DOWNLOADED	0	745	746
-2
+3
 Expect cache
+DOWNLOADED	0	0	1
 DOWNLOADED	0	79	80
 DOWNLOADED	0	745	746
-2
+3
 Expect no cache
 Expect cache
 DOWNLOADED	0	79	80
@@ -45,13 +51,15 @@ DOWNLOADED	0	745	746
 2
 Expect no cache
 Expect cache
+DOWNLOADED	0	0	1
 DOWNLOADED	0	79	80
 DOWNLOADED	0	745	746
-2
+3
 Expect cache
+DOWNLOADED	0	0	1
 DOWNLOADED	0	79	80
 DOWNLOADED	0	745	746
-2
+3
 Expect no cache
 Expect cache
 DOWNLOADED	0	79	80

From 60bf45f863a1e4184cd2159d5435f32e228a0008 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <104093494+CheSema@users.noreply.github.com>
Date: Sat, 6 May 2023 12:11:16 +0200
Subject: [PATCH 406/689] Update WriteBufferFromS3BufferAllocationPolicy.cpp

---
 src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp b/src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp
index 1e9b209087c..0eec6b0d034 100644
--- a/src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp
+++ b/src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp
@@ -93,7 +93,7 @@ struct ExpBufferAllocationPolicy : DB::IBufferAllocationPolicy
 namespace DB
 {
 
-IBufferAllocationPolicy::~IBufferAllocationPolicy() { }
+IBufferAllocationPolicy::~IBufferAllocationPolicy() = default;
 
 IBufferAllocationPolicyPtr ChooseBufferPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings_)
 {

From 8bc9a32d19b5ecd02bc08787800cd475564069fa Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Wed, 10 May 2023 18:45:59 +0000
Subject: [PATCH 407/689] fix special build

---
 src/IO/S3Common.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h
index 370c5911482..71d52c727c7 100644
--- a/src/IO/S3Common.h
+++ b/src/IO/S3Common.h
@@ -20,8 +20,6 @@
 #include <aws/core/Aws.h>
 #include <aws/s3/S3Errors.h>
 
-namespace Aws::S3 { class Client; }
-
 namespace DB
 {
 

From 23da90d111d91468b1b5cc6f15c42f392239128f Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 10 May 2023 21:53:54 +0200
Subject: [PATCH 408/689] fix flaky test 02504_regexp_dictionary_ua_parser

---
 ...2504_regexp_dictionary_ua_parser.reference | 1586 ++++++++---------
 .../02504_regexp_dictionary_ua_parser.sh      |    6 +-
 2 files changed, 796 insertions(+), 796 deletions(-)

diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.reference b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.reference
index b161b099eef..872ee1a0f33 100644
--- a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.reference
+++ b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.reference
@@ -1,793 +1,793 @@
-AppleTV	Other 0.0	ATV OS X 0.0.0
-LG-M150	Firefox Mobile 68.0	Android 7.0.0
-Generic Smartphone	Firefox Mobile 68.0	Android 8.0.0
-Generic Tablet	Firefox Mobile 68.0	Android 8.1.0
-Generic Smartphone	Firefox Mobile 68.0	Android 9.0.0
-PH-1	Chrome Mobile 77.0	Android 10.0.0
-Pixel 2 XL	Chrome Mobile 77.0	Android 10.0.0
-Pixel 2	Chrome Mobile 77.0	Android 10.0.0
-Pixel 3	Facebook 240.0	Android 10.0.0
-Pixel XL	Chrome Mobile WebView 77.0	Android 10.0.0
-Pixel XL	Chrome Mobile 77.0	Android 10.0.0
-HTC Sensation 4G	Chrome Mobile 42.0	Android 4.0.3
-Kindle	Amazon Silk 73.7	Android 4.0.3
-Samsung GT-I9152 	Chrome Mobile 42.0	Android 4.2.2
-Samsung GT-N5110	Chrome 76.0	Android 4.4.2
-RCT6773W22	Chrome 77.0	Android 4.4.2
-Samsung SM-T217S	Chrome 77.0	Android 4.4.2
-Samsung SM-T530NU	Chrome 77.0	Android 4.4.2
-TegraNote-P1640	Chrome 69.0	Android 4.4.2
-Kindle	Amazon Silk 76.3	Android 4.4.3
-Samsung SM-A500H	Chrome Mobile 73.0	Android 5.0.2
-Samsung SM-T357T	Chrome 77.0	Android 5.0.2
-Samsung SM-T530NU	Chrome 76.0	Android 5.0.2
-Samsung SM-T530NU	Chrome 77.0	Android 5.0.2
-RCT6213W87DK	Yandex Browser 19.4	Android 5.0.0
-Samsung SM-N900T	Facebook 229.0	Android 5.0.0
-Generic Smartphone	Chrome Mobile WebView 70.0	Android 5.1.1
-Kindle	Amazon Silk 76.3	Android 5.1.1
-AFTT	Chrome Mobile WebView 70.0	Android 5.1.1
-Kindle	Amazon Silk 76.3	Android 5.1.1
-Kindle	Amazon Silk 76.3	Android 5.1.1
-Kindle	Amazon Silk 71.2	Android 5.1.1
-Kindle	Amazon Silk 76.3	Android 5.1.1
-Kindle	Amazon Silk 76.3	Android 5.1.1
-Kindle	Amazon Silk 76.3	Android 5.1.1
-Kindle	Amazon Silk 76.3	Android 5.1.1
-Kindle	Amazon Silk 77.1	Android 5.1.1
-LG-AS330	Chrome Mobile 77.0	Android 5.1.1
-LGL43AL	Chrome Mobile 77.0	Android 5.1.1
-Samsung SM-G530R7	Samsung Internet 9.2	Android 5.1.1
-Samsung SM-T377P	Samsung Internet 10.1	Android 5.1.1
-Samsung SM-T900	Samsung Internet 10.1	Android 5.1.1
-Samsung SM-T337A	Chrome 69.0	Android 5.1.1
-Samsung SM-G360T1	Chrome Mobile 67.0	Android 5.1.1
-Samsung SM-J320FN	Chrome Mobile 74.0	Android 5.1.1
-SM-T280	Chrome 74.0	Android 5.1.1
-Samsung SM-T330NU	Chrome 71.0	Android 5.1.1
-SM-T670	Chrome 76.0	Android 5.1.1
-SM-T670	Chrome 77.0	Android 5.1.1
-Vodafone Smart ultra 6	Chrome Mobile WebView 74.0	Android 5.1.1
-BLU Advance 5.0	Chrome Mobile 66.0	Android 5.1.0
-HTC Desire 626s	Chrome Mobile 77.0	Android 5.1.0
-HUAWEI LUA-L22	Chrome Mobile 50.0	Android 5.1.0
-NX16A11264	Chrome 77.0	Android 5.1.0
-XT1526	Chrome Mobile 73.0	Android 5.1.0
-Oppo CPH1613	Chrome Mobile 77.0	Android 6.0.1
-LG-M153 	Chrome Mobile WebView 55.0	Android 6.0.1
-LG-M153	Chrome Mobile 77.0	Android 6.0.1
-LGLS676	Chrome Mobile 77.0	Android 6.0.1
-N9136	Chrome Mobile 74.0	Android 6.0.1
-Asus Nexus 7	Chrome 44.0	Android 6.0.1
-Samsung SM-G900I	Samsung Internet 10.1	Android 6.0.1
-Samsung SM-G900P	Samsung Internet 7.2	Android 6.0.1
-Samsung SM-J700M	Samsung Internet 10.1	Android 6.0.1
-Samsung SM-S327VL	Samsung Internet 10.1	Android 6.0.1
-Samsung SM-T377A	Chrome 77.0	Android 6.0.1
-Samsung SM-G532M	Chrome Mobile 55.0	Android 6.0.1
-Samsung SM-G532M	Facebook 240.0	Android 6.0.1
-Samsung SM-G532M	Chrome Mobile 77.0	Android 6.0.1
-Samsung SM-G550T	Chrome Mobile 76.0	Android 6.0.1
-Samsung SM-G550T	Chrome Mobile 77.0	Android 6.0.1
-Samsung SM-G550T1	Chrome Mobile 76.0	Android 6.0.1
-Samsung SM-G900V	Chrome Mobile 73.0	Android 6.0.1
-Samsung SM-G920A	Chrome Mobile 77.0	Android 6.0.1
-Samsung SM-J327P	Chrome Mobile 77.0	Android 6.0.1
-Samsung SM-N910S	Chrome Mobile 75.0	Android 6.0.1
-Samsung SM-N920V	Chrome Mobile 76.0	Android 6.0.1
-Samsung SM-T350	Chrome 59.0	Android 6.0.1
-Samsung SM-T560NU	Chrome 77.0	Android 6.0.1
-SM-T800	Chrome 77.0	Android 6.0.1
-XT1254	Chrome Mobile 77.0	Android 6.0.1
-Z798BL	Chrome Mobile 67.0	Android 6.0.1
-Z799VL	Chrome Mobile WebView 45.0	Android 6.0.1
-5010X	Chrome Mobile 76.0	Android 6.0.0
-Huawei CAM-L21	Chrome Mobile 77.0	Android 6.0.0
-F3313	Chrome Mobile 77.0	Android 6.0.0
-RCT6603W47M7	Chrome 77.0	Android 6.0.0
-5049Z	Chrome Mobile 56.0	Android 7.0.0
-Asus A002A	Chrome Mobile 77.0	Android 7.0.0
-Alcatel_5044C	Chrome Mobile 77.0	Android 7.0.0
-Astra Young Pro	Chrome Mobile WebView 59.0	Android 7.0.0
-Infinix X571	Chrome Mobile 77.0	Android 7.0.0
-LG-H872 	Chrome Mobile 64.0	Android 7.0.0
-LG-K425 	Chrome Mobile 55.0	Android 7.0.0
-LG-LS777	Chrome Mobile 77.0	Android 7.0.0
-LG-M210	Chrome Mobile 77.0	Android 7.0.0
-LG-M430	Chrome Mobile 77.0	Android 7.0.0
-LG-TP260 	Chrome Mobile WebView 64.0	Android 7.0.0
-LG-TP260	Chrome Mobile 77.0	Android 7.0.0
-LG-TP450 	Chrome Mobile 64.0	Android 7.0.0
-LG-V521	Chrome 75.0	Android 7.0.0
-LG-V521	Chrome 77.0	Android 7.0.0
-LGMP260	Chrome Mobile 58.0	Android 7.0.0
-LGMS210	Chrome Mobile 55.0	Android 7.0.0
-LGMS210	Chrome Mobile 77.0	Android 7.0.0
-P00I	Chrome 77.0	Android 7.0.0
-RS988	Chrome Mobile 77.0	Android 7.0.0
-Samsung SM-J701F	Samsung Internet 10.1	Android 7.0.0
-Samsung SM-J710F	Samsung Internet 10.1	Android 7.0.0
-Samsung SM-N920T	Samsung Internet 9.2	Android 7.0.0
-Samsung SM-G920A	Chrome Mobile 77.0	Android 7.0.0
-Samsung SM-G920P	Flipboard 4.2	Android 7.0.0
-Samsung SM-G920V	Chrome Mobile 76.0	Android 7.0.0
-Samsung SM-G928V	Chrome Mobile 77.0	Android 7.0.0
-Samsung SM-G950U	Chrome Mobile 77.0	Android 7.0.0
-Samsung SM-G955U	Chrome Mobile 77.0	Android 7.0.0
-Samsung SM-J327T	Chrome Mobile 74.0	Android 7.0.0
-Samsung SM-J327T	Chrome Mobile 77.0	Android 7.0.0
-Samsung SM-J327T1	Chrome Mobile 64.0	Android 7.0.0
-Samsung SM-J327T1	Chrome Mobile 75.0	Android 7.0.0
-Samsung SM-J327T1	Chrome Mobile 77.0	Android 7.0.0
-Samsung SM-N9208	Chrome Mobile 73.0	Android 7.0.0
-Samsung SM-N920P	Chrome Mobile 74.0	Android 7.0.0
-Samsung SM-N920T	Chrome Mobile 77.0	Android 7.0.0
-SM-T585	Chrome 77.0	Android 7.0.0
-SM-T810	Chrome 75.0	Android 7.0.0
-SM-T810	Chrome 76.0	Android 7.0.0
-SM-T810	Chrome 77.0	Android 7.0.0
-SM-T813	Chrome 76.0	Android 7.0.0
-SM-T813	Chrome 76.0	Android 7.0.0
-Trekstor ST1009X	Chrome 75.0	Android 7.0.0
-XT1663	Chrome Mobile 77.0	Android 7.0.0
-Generic Smartphone	Chrome Mobile 58.0	Android 7.0.0
-A574BL	Chrome Mobile WebView 77.0	Android 7.1.1
-A574BL	Chrome Mobile 77.0	Android 7.1.1
-Oppo CPH1729	Facebook 240.0	Android 7.1.1
-3632A	Chrome Mobile 74.0	Android 7.1.1
-General Mobile 4G Dual	Chrome Mobile 77.0	Android 7.1.1
-Moto E (4) Plus	Chrome Mobile WebView 76.0	Android 7.1.1
-Moto E (4)	Chrome Mobile 70.0	Android 7.1.1
-Moto E (4)	Chrome Mobile 76.0	Android 7.1.1
-Moto E (4)	Chrome Mobile 77.0	Android 7.1.1
-Moto E (4)	Chrome Mobile 77.0	Android 7.1.1
-NX591J	Chrome Mobile 77.0	Android 7.1.1
-REVVLPLUS C3701A	Chrome Mobile 64.0	Android 7.1.1
-Samsung SM-J320A	Samsung Internet 10.1	Android 7.1.1
-Samsung SM-T550	Samsung Internet 10.1	Android 7.1.1
-Samsung SM-T377A	Chrome 64.0	Android 7.1.1
-Samsung SM-J250F	Chrome Mobile 76.0	Android 7.1.1
-Samsung SM-J700T	Chrome Mobile 77.0	Android 7.1.1
-SM-T350	Chrome 77.0	Android 7.1.1
-Samsung SM-T377T	Chrome 77.0	Android 7.1.1
-Samsung SM-T550	Chrome 69.0	Android 7.1.1
-SM-T550	Chrome 77.0	Android 7.1.1
-Samsung SM-T560NU	Chrome 77.0	Android 7.1.1
-X20	Chrome Mobile WebView 52.0	Android 7.1.1
-Z851M	Chrome Mobile 58.0	Android 7.1.1
-Z899VL	Chrome Mobile WebView 74.0	Android 7.1.1
-Z982	Chrome Mobile WebView 75.0	Android 7.1.1
-Z982	Chrome Mobile 77.0	Android 7.1.1
-Generic Smartphone	Chrome Mobile WebView 70.0	Android 7.1.2
-AFTKMST12	Chrome Mobile WebView 70.0	Android 7.1.2
-Kindle	Amazon Silk 76.3	Android 7.1.2
-AFTMM	Chrome Mobile WebView 70.0	Android 7.1.2
-AFTN	Chrome Mobile WebView 70.0	Android 7.1.2
-KFKAWI	Chrome Mobile WebView 59.0	Android 7.1.2
-Kindle	Amazon Silk 76.3	Android 7.1.2
-Kindle	Amazon Silk 76.3	Android 7.1.2
-LG-SP200	Chrome Mobile 75.0	Android 7.1.2
-LG-SP200	Chrome Mobile 76.0	Android 7.1.2
-LM-X210(G)	Chrome Mobile 76.0	Android 7.1.2
-LM-X210	Chrome Mobile 76.0	Android 7.1.2
-RCT6973W43R	Chrome 77.0	Android 7.1.2
-XiaoMi Redmi 4	Chrome Mobile 77.0	Android 7.1.2
-Generic Smartphone	Chrome Mobile WebView 76.0	Android 8.0.0
-Asus Z01FD	Chrome Mobile 77.0	Android 8.0.0
-Huawei AUM-L29	Chrome Mobile 77.0	Android 8.0.0
-BRAVIA 4K GB	Chrome Mobile WebView 77.0	Android 8.0.0
-CMR-W09	Chrome 77.0	Android 8.0.0
-EVA-AL00	Chrome Mobile 77.0	Android 8.0.0
-G3223	Chrome Mobile 77.0	Android 8.0.0
-LG-H910	Chrome Mobile 77.0	Android 8.0.0
-LG-H931	Chrome Mobile 76.0	Android 8.0.0
-LG-H932	Chrome Mobile 77.0	Android 8.0.0
-Samsung SM-A520F	Samsung Internet 10.1	Android 8.0.0
-Samsung SM-G891A	Samsung Internet 8.2	Android 8.0.0
-Samsung SM-G935T	Samsung Internet 10.1	Android 8.0.0
-Samsung SM-G955U	Samsung Internet 10.1	Android 8.0.0
-Samsung SM-J337T	Samsung Internet 9.2	Android 8.0.0
-Samsung SM-J737P	Samsung Internet 10.1	Android 8.0.0
-Samsung SM-N950F	Samsung Internet 10.1	Android 8.0.0
-Samsung SM-G891A	Chrome Mobile 72.0	Android 8.0.0
-Samsung SM-G935A	Chrome Mobile 76.0	Android 8.0.0
-Samsung SM-A720F	Chrome Mobile 77.0	Android 8.0.0
-Samsung SM-G570F	Facebook 231.0	Android 8.0.0
-Samsung SM-G570Y	Chrome Mobile 77.0	Android 8.0.0
-Samsung SM-G930T	Chrome Mobile WebView 77.0	Android 8.0.0
-Samsung SM-G930V	Chrome Mobile 64.0	Android 8.0.0
-Samsung SM-G930VL	Chrome Mobile 74.0	Android 8.0.0
-Samsung SM-G935F	Chrome Mobile 75.0	Android 8.0.0
-Samsung SM-G935P	Chrome Mobile 77.0	Android 8.0.0
-Samsung SM-G935T	Facebook 240.0	Android 8.0.0
-Samsung SM-G935T	Chrome Mobile 77.0	Android 8.0.0
-Samsung SM-G950U	Chrome Mobile 77.0	Android 8.0.0
-Samsung SM-G955U	Chrome Mobile 74.0	Android 8.0.0
-Samsung SM-G955U	Chrome Mobile 77.0	Android 8.0.0
-Samsung SM-J330G	Chrome Mobile 77.0	Android 8.0.0
-Samsung SM-J337T	Chrome Mobile 77.0	Android 8.0.0
-Samsung SM-J737A	Chrome Mobile 77.0	Android 8.0.0
-Samsung SM-J737T1	Chrome Mobile 66.0	Android 8.0.0
-Samsung SM-J737T1	Chrome Mobile 77.0	Android 8.0.0
-Samsung SM-N950F	Chrome Mobile 66.0	Android 8.0.0
-Samsung SM-N950U	Chrome Mobile 76.0	Android 8.0.0
-Samsung SM-N950U	Chrome Mobile 77.0	Android 8.0.0
-Samsung SM-N950U1	Chrome Mobile 77.0	Android 8.0.0
-Samsung SM-S367VL	Chrome Mobile 77.0	Android 8.0.0
-VS995	Chrome Mobile 77.0	Android 8.0.0
-XT1635-02	Chrome Mobile 77.0	Android 8.0.0
-moto e5 play	Chrome Mobile 76.0	Android 8.0.0
-moto e5 play	Chrome Mobile 77.0	Android 8.0.0
-moto e5 supra	Chrome Mobile 76.0	Android 8.0.0
-moto g(6)	Chrome Mobile 77.0	Android 8.0.0
-5041C	Chrome Mobile 77.0	Android 8.1.0
-6062W	Chrome Mobile 77.0	Android 8.1.0
-A502DL	Chrome Mobile 67.0	Android 8.1.0
-A502DL	Chrome Mobile 76.0	Android 8.1.0
-Huawei BKK-LX2	Chrome Mobile 76.0	Android 8.1.0
-C4	Chrome Mobile 70.0	Android 8.1.0
-3310A	Chrome Mobile 77.0	Android 8.1.0
-Infinix X604	Chrome Mobile 64.0	Android 8.1.0
-Joy 1	Chrome Mobile 77.0	Android 8.1.0
-LAVA LE9820	Chrome Mobile 77.0	Android 8.1.0
-LG-Q710AL	Chrome Mobile 77.0	Android 8.1.0
-LM-Q610(FGN)	Chrome Mobile 77.0	Android 8.1.0
-LM-Q710(FGN)	Facebook 235.0	Android 8.1.0
-LM-Q710(FGN)	Chrome Mobile 70.0	Android 8.1.0
-LM-Q710(FGN)	Chrome Mobile 76.0	Android 8.1.0
-LM-Q710(FGN)	Chrome Mobile 76.0	Android 8.1.0
-LM-Q710(FGN)	Chrome Mobile 77.0	Android 8.1.0
-LM-V405	Chrome Mobile 77.0	Android 8.1.0
-LM-X210(G)	UC Browser 11.6	Android 8.1.0
-LM-X210(G)	Chrome Mobile 70.0	Android 8.1.0
-LM-X210(G)	Chrome Mobile 72.0	Android 8.1.0
-LM-X210(G)	Chrome Mobile 77.0	Android 8.1.0
-LM-X212(G)	Chrome Mobile 77.0	Android 8.1.0
-LM-X220	Chrome Mobile 70.0	Android 8.1.0
-LM-X220	Chrome Mobile 76.0	Android 8.1.0
-LM-X220PM	Chrome Mobile WebView 77.0	Android 8.1.0
-LM-X410(FG)	Chrome Mobile 70.0	Android 8.1.0
-LM-X410(FG)	Chrome Mobile 76.0	Android 8.1.0
-LM-X410(FG)	Chrome Mobile 77.0	Android 8.1.0
-LM-X410.FGN	Chrome Mobile 68.0	Android 8.1.0
-LML414DL	Chrome Mobile 76.0	Android 8.1.0
-LML713DL	Chrome Mobile 77.0	Android 8.1.0
-Moto G (5S) Plus	Chrome Mobile 77.0	Android 8.1.0
-HTC One	Chrome Mobile WebView 70.0	Android 8.1.0
-RCT6873W42BMF8KC	Chrome Mobile 77.0	Android 8.1.0
-REVVL 2	Chrome Mobile 67.0	Android 8.1.0
-REVVL 2	Chrome Mobile 76.0	Android 8.1.0
-Samsung SM-J727T	Samsung Internet 10.1	Android 8.1.0
-Samsung SM-J727T1	Samsung Internet 9.4	Android 8.1.0
-Samsung SM-J727T1	Samsung Internet 10.1	Android 8.1.0
-Samsung SM-T580	Samsung Internet 9.4	Android 8.1.0
-Samsung SM-J727A	Facebook 240.0	Android 8.1.0
-Samsung SM-G610F	Chrome Mobile 77.0	Android 8.1.0
-Samsung SM-J260T1	Chrome Mobile 76.0	Android 8.1.0
-Samsung SM-J260T1	Chrome Mobile 76.0	Android 8.1.0
-Samsung SM-J260T1	Chrome Mobile 77.0	Android 8.1.0
-Samsung SM-J410F	Chrome Mobile 77.0	Android 8.1.0
-Samsung SM-J727P	Chrome Mobile 68.0	Android 8.1.0
-Samsung SM-J727T	Chrome Mobile 66.0	Android 8.1.0
-Samsung SM-J727T1	Chrome Mobile 76.0	Android 8.1.0
-Samsung SM-J727T1	Chrome Mobile 77.0	Android 8.1.0
-Samsung SM-J727T1	Chrome Mobile 77.0	Android 8.1.0
-Samsung SM-J727V	Chrome Mobile 70.0	Android 8.1.0
-Samsung SM-J727V	Chrome Mobile 77.0	Android 8.1.0
-SM-P580	Chrome 77.0	Android 8.1.0
-SM-T380	Chrome 75.0	Android 8.1.0
-SM-T580	Edge Mobile 42.0	Android 8.1.0
-SM-T580	Chrome 76.0	Android 8.1.0
-SM-T580	Chrome 76.0	Android 8.1.0
-SM-T580	Chrome 77.0	Android 8.1.0
-Samsung SM-T837T	Chrome 77.0	Android 8.1.0
-TECNO CF8	Facebook 239.0	Android 8.1.0
-V1818CA	Chrome Mobile 75.0	Android 8.1.0
-meizu C9	Chrome Mobile 68.0	Android 8.1.0
-vivo 1724	Chrome Mobile 76.0	Android 8.1.0
-vivo 1814	Chrome Mobile 77.0	Android 8.1.0
-Generic Smartphone	DuckDuckGo Mobile 5.0	Android 9.0.0
-1825	Chrome Mobile 70.0	Android 9.0.0
-ANE-LX2	Facebook 236.0	Android 9.0.0
-BLA-A09	Chrome Mobile 77.0	Android 9.0.0
-Huawei CLT-L04	Chrome Mobile 77.0	Android 9.0.0
-Oppo CPH1911	Facebook 239.0	Android 9.0.0
-Oppo CPH1923	Chrome Mobile WebView 76.0	Android 9.0.0
-Huawei ELE-L29	Chrome Mobile 77.0	Android 9.0.0
-G8142	Chrome Mobile 77.0	Android 9.0.0
-GM1911	Chrome Mobile 76.0	Android 9.0.0
-GM1917	Chrome Mobile 77.0	Android 9.0.0
-Huawei INE-LX2	Chrome Mobile 76.0	Android 9.0.0
-LM-G710	Chrome Mobile WebView 77.0	Android 9.0.0
-LM-Q720	Chrome Mobile 77.0	Android 9.0.0
-LM-V405	Chrome Mobile WebView 77.0	Android 9.0.0
-LM-V405	Chrome Mobile 76.0	Android 9.0.0
-LM-V500N	Chrome Mobile 77.0	Android 9.0.0
-LM-X420	Chrome Mobile 72.0	Android 9.0.0
-LM-X420	Chrome Mobile 77.0	Android 9.0.0
-MAR-LX1A	Chrome Mobile 77.0	Android 9.0.0
-XiaoMi MI 9	Chrome Mobile 77.0	Android 9.0.0
-XiaoMi Mi A2	Chrome Mobile 77.0	Android 9.0.0
-Moto Z (2)	Chrome Mobile 77.0	Android 9.0.0
-Nokia 6	Chrome Mobile 77.0	Android 9.0.0
-OnePlus ONEPLUS A6000	Chrome Mobile 77.0	Android 9.0.0
-OnePlus ONEPLUS A6003	Chrome Mobile 77.0	Android 9.0.0
-OnePlus ONEPLUS A6013	Chrome Mobile WebView 77.0	Android 9.0.0
-OnePlus ONEPLUS A6013	Chrome Mobile 74.0	Android 9.0.0
-OnePlus ONEPLUS A6013	Chrome Mobile 77.0	Android 9.0.0
-PAR-AL00	Facebook 235.0	Android 9.0.0
-Pixel 2 XL	Chrome Mobile 77.0	Android 9.0.0
-Pixel 3	Chrome Mobile WebView 77.0	Android 9.0.0
-Pixel 3	Chrome Mobile 76.0	Android 9.0.0
-Pixel 3	Chrome Mobile 77.0	Android 9.0.0
-Pixel 3a XL	Chrome Mobile 77.0	Android 9.0.0
-REVVLRY 	Chrome Mobile 73.0	Android 9.0.0
-Oppo RMX1801	Chrome Mobile 75.0	Android 9.0.0
-XiaoMi Redmi 7	Chrome Mobile 77.0	Android 9.0.0
-XiaoMi Redmi Note 7	Chrome Mobile 76.0	Android 9.0.0
-Samsung SM-A102U	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-A505FN	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-A505GN	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-G892U	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-G950U	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-G955F	Samsung Internet 9.4	Android 9.0.0
-Samsung SM-G955U	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-G9600	Samsung Internet 9.4	Android 9.0.0
-Samsung SM-G960U	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-G965U	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-G970F	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-G970U	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-G973U	Samsung Internet 9.4	Android 9.0.0
-Samsung SM-G973U	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-G975U	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-J415F	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-J730F	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-J737P	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-J737T	Samsung Internet 9.0	Android 9.0.0
-Samsung SM-N950U	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-N960F	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-N960U	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-N960U1	Samsung Internet 9.2	Android 9.0.0
-Samsung SM-N970U	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-N975U	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-N975U1	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-T510	Samsung Internet 10.1	Android 9.0.0
-Samsung SM-T720	Samsung Internet 10.1	Android 9.0.0
-SHIELD Android TV	Chrome Mobile WebView 77.0	Android 9.0.0
-Samsung SM-A102U	Chrome Mobile 72.0	Android 9.0.0
-Samsung SM-A102U	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-A105M	Facebook 237.0	Android 9.0.0
-Samsung SM-A205G	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-A205U	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-A505F	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-A530F	Facebook 240.0	Android 9.0.0
-Samsung SM-A530N	Chrome Mobile WebView 77.0	Android 9.0.0
-Samsung SM-A600T	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-A605F	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-A920F	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G892A	Chrome Mobile 74.0	Android 9.0.0
-Samsung SM-G950F	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G950U	Chrome Mobile WebView 77.0	Android 9.0.0
-Samsung SM-G950U	Chrome Mobile 71.0	Android 9.0.0
-Samsung SM-G950U	Chrome Mobile 76.0	Android 9.0.0
-Samsung SM-G950U	Chrome Mobile 76.0	Android 9.0.0
-Samsung SM-G950U	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G950U1	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G955F	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G955U	Facebook 240.0	Android 9.0.0
-Samsung SM-G955U	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G9600	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G960U	Facebook 233.0	Android 9.0.0
-Samsung SM-G960U	Chrome Mobile WebView 77.0	Android 9.0.0
-Samsung SM-G960U	Chrome Mobile 71.0	Android 9.0.0
-Samsung SM-G960U	Chrome Mobile 74.0	Android 9.0.0
-Samsung SM-G960U	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G960U1	Facebook 240.0	Android 9.0.0
-Samsung SM-G960U1	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G965F	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G965U	Chrome Mobile 74.0	Android 9.0.0
-Samsung SM-G965U	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G965U	Chrome Mobile 79.0	Android 9.0.0
-Samsung SM-G965U1	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G970U	Facebook 240.0	Android 9.0.0
-Samsung SM-G970U	Chrome Mobile 75.0	Android 9.0.0
-Samsung SM-G970U	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G970U1	Chrome Mobile WebView 77.0	Android 9.0.0
-Samsung SM-G973U	Chrome Mobile 74.0	Android 9.0.0
-Samsung SM-G973U	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G973U1	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G975U	Chrome Mobile 75.0	Android 9.0.0
-Samsung SM-G975U	Chrome Mobile 76.0	Android 9.0.0
-Samsung SM-G975U	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-G975U1	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-J260A	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-J337P	Chrome Mobile 76.0	Android 9.0.0
-Samsung SM-J600FN	Chrome Mobile 75.0	Android 9.0.0
-Samsung SM-J600G	Facebook 238.0	Android 9.0.0
-Samsung SM-J730F	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-J737A	Chrome Mobile WebView 77.0	Android 9.0.0
-Samsung SM-J737A	Chrome Mobile 74.0	Android 9.0.0
-Samsung SM-J737V	Pinterest 0.0	Android 9.0.0
-Samsung SM-J737V	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-J810M	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-N950U	Facebook 240.0	Android 9.0.0
-Samsung SM-N950U	Chrome Mobile 72.0	Android 9.0.0
-Samsung SM-N950U	Chrome Mobile 75.0	Android 9.0.0
-Samsung SM-N950U	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-N950U	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-N960F	Chrome Mobile 76.0	Android 9.0.0
-Samsung SM-N960U	Facebook 240.0	Android 9.0.0
-Samsung SM-N960U	Chrome Mobile WebView 77.0	Android 9.0.0
-Samsung SM-N960U	Chrome Mobile 74.0	Android 9.0.0
-Samsung SM-N960U	Chrome Mobile 75.0	Android 9.0.0
-Samsung SM-N960U	Chrome Mobile 76.0	Android 9.0.0
-Samsung SM-N960U	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-N960U1	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-N975U	Chrome Mobile WebView 77.0	Android 9.0.0
-Samsung SM-N975U	Chrome Mobile WebView 77.0	Android 9.0.0
-Samsung SM-N975U	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-N976V	Facebook 240.0	Android 9.0.0
-Samsung SM-S367VL	Chrome Mobile 77.0	Android 9.0.0
-Samsung SM-S767VL	Chrome Mobile 76.0	Android 9.0.0
-Samsung SM-T597P	Chrome 77.0	Android 9.0.0
-SM-T720	Chrome 77.0	Android 9.0.0
-TECNO KC8	Chrome Mobile 77.0	Android 9.0.0
-Huawei VOG-L29	Chrome Mobile 77.0	Android 9.0.0
-cp3705A	Chrome Mobile 74.0	Android 9.0.0
-moto g(6)	Chrome Mobile WebView 77.0	Android 9.0.0
-moto g(6) play	Chrome Mobile 77.0	Android 9.0.0
-moto g(7) play	Facebook 235.0	Android 9.0.0
-moto g(7) play	Chrome Mobile 70.0	Android 9.0.0
-moto g(7) power	Chrome Mobile 75.0	Android 9.0.0
-moto g(7) power	Chrome Mobile 77.0	Android 9.0.0
-moto z4	Chrome Mobile 73.0	Android 9.0.0
-moto z4	Chrome Mobile 77.0	Android 9.0.0
-Samsung GT-P3113 	Android 4.1	Android 4.1.1
-Samsung GT-I8160 	Android 4.1	Android 4.1.2
-Asus Nexus 7	Android 4.2	Android 4.2.2
-Samsung SM-E500H	Android 4.4	Android 4.4.0
-LGMS550	Chrome Mobile WebView 43.0	Android 6.0.1
-Samsung SM-J737T1	Chrome Mobile WebView 43.0	Android 6.0.1
-TECNO CA6	Opera Mobile 5.3	Android 7.0.0
-XiaoMi Redmi 5A	MiuiBrowser 9.5	Android 7.1.2
-Oppo CPH1911	Chrome Mobile WebView 70.0	Android 9.0.0
-vivo 1904	Opera Mobile 44.1	Android 9.0.0
-Mac	Firefox 68.0	Mac OS X 10.11.0
-Mac	Firefox 69.0	Mac OS X 10.13.0
-Mac	Firefox 67.0	Mac OS X 10.14.0
-Mac	Firefox 68.0	Mac OS X 10.14.0
-Mac	Firefox 69.0	Mac OS X 10.14.0
-Mac	Firefox 70.0	Mac OS X 10.14.0
-Mac	Chrome 76.0	Mac OS X 10.10.5
-Mac	Chrome 77.0	Mac OS X 10.10.5
-Mac	Safari 10.1	Mac OS X 10.10.5
-Mac	Chrome 76.0	Mac OS X 10.11.4
-Mac	Chrome 72.0	Mac OS X 10.11.6
-Mac	Chrome 76.0	Mac OS X 10.11.6
-Mac	Chrome 76.0	Mac OS X 10.11.6
-Mac	Chrome 77.0	Mac OS X 10.11.6
-Mac	Safari 9.1	Mac OS X 10.11.6
-Mac	Safari 10.0	Mac OS X 10.11.6
-Mac	Safari 11.1	Mac OS X 10.11.6
-Mac	Chrome 77.0	Mac OS X 10.12.1
-Mac	Safari 10.0	Mac OS X 10.12.3
-Mac	Chrome 75.0	Mac OS X 10.12.6
-Mac	Chrome 76.0	Mac OS X 10.12.6
-Mac	Chrome 76.0	Mac OS X 10.12.6
-Mac	Chrome 77.0	Mac OS X 10.12.6
-Mac	Safari 12.1	Mac OS X 10.12.6
-Mac	Safari 11.0	Mac OS X 10.13.0
-Mac	Chrome 77.0	Mac OS X 10.13.1
-Mac	Chrome 77.0	Mac OS X 10.13.2
-Mac	Chrome 76.0	Mac OS X 10.13.4
-Mac	Chrome 76.0	Mac OS X 10.13.4
-Mac	Chrome 76.0	Mac OS X 10.13.5
-Mac	Chrome 75.0	Mac OS X 10.13.6
-Mac	Chrome 76.0	Mac OS X 10.13.6
-Mac	Chrome 77.0	Mac OS X 10.13.6
-Mac	Safari 12.0	Mac OS X 10.13.6
-Mac	Safari 12.1	Mac OS X 10.13.6
-Mac	Safari 12.1	Mac OS X 10.13.6
-Mac	Safari 13.0	Mac OS X 10.13.6
-Mac	Safari 13.0	Mac OS X 10.13.6
-Mac	Chrome 75.0	Mac OS X 10.14.0
-Mac	Chrome 76.0	Mac OS X 10.14.0
-Mac	Chrome 77.0	Mac OS X 10.14.0
-Mac	Chrome 77.0	Mac OS X 10.14.1
-Mac	Chrome 76.0	Mac OS X 10.14.2
-Mac	Chrome 69.0	Mac OS X 10.14.3
-Mac	Safari 12.0	Mac OS X 10.14.3
-Mac	Chrome 75.0	Mac OS X 10.14.4
-Mac	Chrome 77.0	Mac OS X 10.14.4
-Mac	Safari 12.1	Mac OS X 10.14.4
-Mac	Chrome 76.0	Mac OS X 10.14.5
-Mac	Chrome 77.0	Mac OS X 10.14.5
-Mac	Safari 12.1	Mac OS X 10.14.5
-Mac	Chrome 75.0	Mac OS X 10.14.6
-Mac	Chrome 76.0	Mac OS X 10.14.6
-Mac	Chrome 76.0	Mac OS X 10.14.6
-Mac	Chrome 77.0	Mac OS X 10.14.6
-Mac	Chrome 77.0	Mac OS X 10.14.6
-Mac	Safari 12.1	Mac OS X 10.14.6
-Mac	Safari 13.0	Mac OS X 10.14.6
-Mac	Chrome 65.0	Mac OS X 10.9.5
-Mac	Chrome 66.0	Mac OS X 10.9.5
-Mac	Chrome 67.0	Mac OS X 10.9.5
-PlayStation 4	Apple Mail 605.1	Other 0.0.0
-Samsung SMART-TV	Safari 3.0	Tizen 3.0.0
-Samsung SMART-TV	Samsung Internet 2.0	Tizen 3.0.0
-Samsung SMART-TV	Samsung Internet 2.1	Tizen 4.0.0
-Samsung SMART-TV	Samsung Internet 2.2	Tizen 5.0.0
-Other	Edge 17.17134	Windows 10.0.0
-Other	Edge 18.17763	Windows 10.0.0
-Other	Chrome 77.0	Windows 10.0.0
-Other	Maxthon 5.2	Windows 10.0.0
-Other	Chrome 73.1	Windows 10.0.0
-Other	Chrome 76.0	Windows 10.0.0
-Other	Opera 63.0	Windows 10.0.0
-Other	Chrome 77.0	Windows 10.0.0
-Other	Chrome 77.0	Windows 10.0.0
-Other	Coc Coc 82.0	Windows 10.0.0
-Other	IE 11.0	Windows 10.0.0
-Other	Firefox 59.0	Windows 10.0.0
-Other	Firefox 60.0	Windows 10.0.0
-Other	Edge 15.15063	Windows 10.0.0
-Other	Edge 16.16299	Windows 10.0.0
-Other	Edge 17.17134	Windows 10.0.0
-Other	Edge 18.17763	Windows 10.0.0
-Other	Chrome 65.0	Windows 10.0.0
-Other	Chrome 70.0	Windows 10.0.0
-Other	Edge 18.18362	Windows 10.0.0
-Other	Edge 18.18995	Windows 10.0.0
-Other	Edge 18.19493	Windows 10.0.0
-Other	Chrome 70.0	Windows 10.0.0
-Other	Chrome 71.0	Windows 10.0.0
-Other	Chrome 73.0	Windows 10.0.0
-Other	Chrome 74.0	Windows 10.0.0
-Other	Chrome 75.0	Windows 10.0.0
-Other	Chrome 76.0	Windows 10.0.0
-Other	Vivaldi 2.7	Windows 10.0.0
-Other	Chrome 76.0	Windows 10.0.0
-Other	Opera 63.0	Windows 10.0.0
-Other	Chrome 77.0	Windows 10.0.0
-Other	Chrome 77.0	Windows 10.0.0
-Other	Edge 79.0	Windows 10.0.0
-Other	Edge 18.18362	Windows 10.0.0
-Other	Edge 18.18363	Windows 10.0.0
-Other	Edge 18.18362	Windows 10.0.0
-Other	Firefox 61.0	Windows 10.0.0
-Other	Firefox 63.0	Windows 10.0.0
-Other	Firefox 67.0	Windows 10.0.0
-Other	Firefox 68.0	Windows 10.0.0
-Other	Firefox 69.0	Windows 10.0.0
-Other	Firefox 69.0	Windows 10.0.0
-Other	Chrome 49.0	Windows XP.0.0
-Other	Chrome 49.0	Windows Vista.0.0
-Other	Chrome 49.0	Windows Vista.0.0
-Other	Chrome 76.0	Windows 7.0.0
-Other	Chrome 77.0	Windows 7.0.0
-Other	Chrome 77.0	Windows 7.0.0
-Other	Coc Coc 80.0	Windows 7.0.0
-Other	Coc Coc 82.0	Windows 7.0.0
-Other	IE 11.0	Windows 7.0.0
-Other	Chrome 67.0	Windows 7.0.0
-Other	Chrome 70.0	Windows 7.0.0
-Other	Chrome 72.0	Windows 7.0.0
-Other	Chrome 74.0	Windows 7.0.0
-Other	Chrome 75.0	Windows 7.0.0
-Other	Chrome 76.0	Windows 7.0.0
-Other	Chrome 76.0	Windows 7.0.0
-Other	Chrome 77.0	Windows 7.0.0
-Other	Waterfox 56.2	Windows 7.0.0
-Other	Firefox 60.0	Windows 7.0.0
-Other	Firefox 63.0	Windows 7.0.0
-Other	Firefox 68.0	Windows 7.0.0
-Other	Firefox 69.0	Windows 7.0.0
-Other	Firefox 69.0	Windows 7.0.0
-Other	Chrome 77.0	Windows 8.0.0
-Other	Firefox 69.0	Windows 8.0.0
-Other	Chrome 77.0	Windows 8.1.0
-Other	IE 11.0	Windows RT 8.1.0
-Other	IE 11.0	Windows 8.1.0
-Other	IE 11.0	Windows 8.1.0
-Other	Chrome 63.0	Windows 8.1.0
-Other	Chrome 64.0	Windows 8.1.0
-Other	Chrome 76.0	Windows 8.1.0
-Other	Chrome 76.0	Windows 8.1.0
-Other	Chrome 77.0	Windows 8.1.0
-Other	Firefox 69.0	Windows 8.1.0
-Other	Firefox 69.0	Windows 8.1.0
-Other	Chrome 72.0	Windows 10.0.0
-Other	Chrome 77.0	Chrome OS 12371.75.0
-Other	Chrome 76.0	Chrome OS 12239.92.0
-Other	Chrome 69.0	Chrome OS 10895.78.0
-Other	Chrome 70.0	Chrome OS 11021.81.0
-Other	Chrome 74.0	Chrome OS 11895.118.0
-Other	Chrome 76.0	Chrome OS 12239.92.0
-Other	Chrome 76.0	Chrome OS 12239.92.1
-Other	Chrome 76.0	Chrome OS 12239.92.4
-Other	Chrome 77.0	Chrome OS 12371.46.0
-Other	Chrome 77.0	Chrome OS 12371.65.0
-Other	Chrome 75.0	Linux 0.0.0
-Other	Chrome 77.0	Linux 0.0.0
-Other	Samsung Internet 10.1	Linux 0.0.0
-Other	Chrome 66.0	Linux 0.0.0
-Other	Chrome 66.0	Linux 0.0.0
-Other	Chrome 66.0	Linux 0.0.0
-Other	Chrome 66.0	Linux 0.0.0
-Other	Chrome 66.0	Linux 0.0.0
-Other	Firefox 65.0	Ubuntu 0.0.0
-Other	Firefox 66.0	Ubuntu 0.0.0
-Other	Firefox 67.0	Ubuntu 0.0.0
-iPad	Google 22.0	iOS 10.3.3
-iPad	Chrome Mobile iOS 71.0	iOS 10.3.3
-iPad	Firefox iOS 14.0	iOS 10.3.3
-iPad	Mobile Safari UI/WKWebView 0.0	iOS 10.3.3
-iPad	Facebook 240.0	iOS 10.3.3
-iPad	Mobile Safari 10.0	iOS 10.3.3
-iPad	Mobile Safari 10.0	iOS 10.3.4
-iPad	Chrome Mobile iOS 76.0	iOS 11.1.0
-iPad	Chrome Mobile iOS 76.0	iOS 11.1.2
-iPad	Mobile Safari 11.0	iOS 11.2.1
-iPad	Mobile Safari 11.0	iOS 11.2.2
-iPad	Mobile Safari 11.0	iOS 11.2.6
-iPad	Mobile Safari 11.0	iOS 11.3.0
-iPad	Mobile Safari 11.0	iOS 11.4.0
-iPad	Mobile Safari UI/WKWebView 0.0	iOS 11.4.1
-iPad	Mobile Safari 11.0	iOS 11.4.1
-iPad	Google 83.0	iOS 12.0.0
-iPad	Mobile Safari 12.0	iOS 12.0.0
-iPad	Chrome Mobile iOS 75.0	iOS 12.1.0
-iPad	Chrome Mobile iOS 76.0	iOS 12.1.0
-iPad	Mobile Safari UI/WKWebView 0.0	iOS 12.1.0
-iPad	Mobile Safari 12.0	iOS 12.1.0
-iPad	Mobile Safari 12.0	iOS 12.1.1
-iPad	Google 48.0	iOS 12.1.4
-iPad	Mobile Safari UI/WKWebView 0.0	iOS 12.1.4
-iPad	Mobile Safari 12.0	iOS 12.1.4
-iPad	Chrome Mobile iOS 76.0	iOS 12.2.0
-iPad	Mobile Safari UI/WKWebView 0.0	iOS 12.2.0
-iPad	Mobile Safari 12.1	iOS 12.2.0
-iPad	Chrome Mobile iOS 77.0	iOS 12.3.0
-iPad	Google 83.0	iOS 12.3.0
-iPad	Mobile Safari 12.1	iOS 12.3.0
-iPad	Mobile Safari UI/WKWebView 0.0	iOS 12.3.1
-iPad	Mobile Safari 12.1	iOS 12.3.1
-iPad	Chrome Mobile iOS 76.0	iOS 12.4.0
-iPad	Chrome Mobile iOS 76.0	iOS 12.4.0
-iPad	Chrome Mobile iOS 77.0	iOS 12.4.0
-iPad	Chrome Mobile iOS 77.0	iOS 12.4.0
-iPad	Chrome Mobile iOS 77.0	iOS 12.4.0
-iPad	Google 74.0	iOS 12.4.0
-iPad	Google 83.0	iOS 12.4.0
-iPad	Mobile Safari UI/WKWebView 0.0	iOS 12.4.0
-iPad	Mobile Safari 12.1	iOS 12.4.0
-iPad	Chrome Mobile iOS 67.0	iOS 12.4.1
-iPad	Firefox iOS 19.0	iOS 12.4.1
-iPad	Mobile Safari UI/WKWebView 0.0	iOS 12.4.1
-iPad	Facebook 0.0	iOS 12.4.1
-iPad	Facebook 0.0	iOS 12.4.1
-iPad	Facebook 0.0	iOS 12.4.1
-iPad	Facebook 0.0	iOS 12.4.1
-iPad	Mobile Safari 12.1	iOS 12.4.1
-iPad	Mobile Safari 6.0	iOS 6.1.3
-iPad	Mobile Safari 8.0	iOS 8.0.0
-iPad	Mobile Safari 8.0	iOS 8.2.0
-iPad	Google 23.1	iOS 8.4.0
-iPad	Mobile Safari 9.0	iOS 9.3.2
-iPad	Mobile Safari 9.0	iOS 9.3.5
-iPhone	Mobile Safari 10.0	iOS 10.2.0
-iPhone	Facebook 0.0	iOS 10.3.3
-iPhone	Google 68.0	iOS 10.3.4
-iPhone	Mobile Safari 10.0	iOS 10.3.4
-iPhone	Mobile Safari 11.0	iOS 11.0.3
-iPhone	Mobile Safari 11.0	iOS 11.1.1
-iPhone	Mobile Safari 11.0	iOS 11.1.2
-iPhone	Mobile Safari 11.0	iOS 11.2.1
-iPhone	Facebook 207.0	iOS 11.2.6
-iPhone	Chrome Mobile iOS 76.0	iOS 11.3.0
-iPhone	Facebook 0.0	iOS 11.3.0
-iPhone	Mobile Safari 11.0	iOS 11.3.0
-iPhone	Google 83.0	iOS 11.4.0
-iPhone	Mobile Safari 11.0	iOS 11.4.0
-iPhone	Google 74.1	iOS 11.4.1
-iPhone	Mobile Safari 11.0	iOS 11.4.1
-iPhone	Mobile Safari 12.0	iOS 12.0.0
-iPhone	Mobile Safari 12.0	iOS 12.1.0
-iPhone	Mobile Safari 12.0	iOS 12.1.1
-iPhone	Google 74.1	iOS 12.1.2
-iPhone	Facebook 0.0	iOS 12.1.2
-iPhone	Mobile Safari 12.0	iOS 12.1.2
-iPhone	Mobile Safari 12.0	iOS 12.1.3
-iPhone	Google 74.1	iOS 12.1.4
-iPhone	Mobile Safari 12.0	iOS 12.1.4
-iPhone	Chrome Mobile iOS 72.0	iOS 12.2.0
-iPhone	Chrome Mobile iOS 76.0	iOS 12.2.0
-iPhone	Chrome Mobile iOS 77.0	iOS 12.2.0
-iPhone	Facebook 0.0	iOS 12.2.0
-iPhone	Facebook 0.0	iOS 12.2.0
-iPhone	Mobile Safari 12.1	iOS 12.2.0
-iPhone	Chrome Mobile iOS 77.0	iOS 12.3.0
-iPhone	Google 83.0	iOS 12.3.0
-iPhone	Mobile Safari 12.1	iOS 12.3.0
-iPhone	Google 79.0	iOS 12.3.1
-iPhone	Mobile Safari UI/WKWebView 0.0	iOS 12.3.1
-iPhone	DuckDuckGo Mobile 7.0	iOS 12.3.1
-iPhone	Facebook 0.0	iOS 12.3.1
-iPhone	Facebook 0.0	iOS 12.3.1
-iPhone	Facebook 0.0	iOS 12.3.1
-iPhone	Mobile Safari 12.1	iOS 12.3.1
-iPhone	Mobile Safari 12.1	iOS 12.3.2
-iPhone	Chrome Mobile iOS 69.0	iOS 12.4.0
-iPhone	Chrome Mobile iOS 73.0	iOS 12.4.0
-iPhone	Chrome Mobile iOS 75.0	iOS 12.4.0
-iPhone	Chrome Mobile iOS 76.0	iOS 12.4.0
-iPhone	Chrome Mobile iOS 77.0	iOS 12.4.0
-iPhone	Chrome Mobile iOS 77.0	iOS 12.4.0
-iPhone	Google 81.0	iOS 12.4.0
-iPhone	Google 82.1	iOS 12.4.0
-iPhone	Google 83.0	iOS 12.4.0
-iPhone	Facebook 0.0	iOS 12.4.0
-iPhone	Facebook 0.0	iOS 12.4.0
-iPhone	Facebook 0.0	iOS 12.4.0
-iPhone	Facebook 0.0	iOS 12.4.0
-iPhone	Mobile Safari 12.1	iOS 12.4.0
-iPhone	Google 74.1	iOS 12.4.1
-iPhone	Mobile Safari UI/WKWebView 0.0	iOS 12.4.1
-iPhone	Instagram 89.0	iOS 12.4.1
-iPhone	Facebook 240.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Facebook 0.0	iOS 12.4.1
-iPhone	Mobile Safari 12.1	iOS 12.4.1
-iPhone	Mobile Safari 12.4	iOS 12.4.1
-iPhone	Mobile Safari UI/WKWebView 0.0	iOS 12.4.2
-iPhone	Mobile Safari 12.1	iOS 12.4.2
-iPhone	Chrome Mobile iOS 77.0	iOS 13.0.0
-iPhone	Facebook 0.0	iOS 13.0.0
-iPhone	Facebook 0.0	iOS 13.0.0
-iPhone	Facebook 0.0	iOS 13.0.0
-iPhone	Facebook 0.0	iOS 13.0.0
-iPhone	Facebook 0.0	iOS 13.0.0
-iPhone	Facebook 0.0	iOS 13.0.0
-iPhone	Mobile Safari 13.0	iOS 13.0.0
-iPhone	Chrome Mobile iOS 76.0	iOS 13.1.0
-iPhone	Chrome Mobile iOS 77.0	iOS 13.1.0
-iPhone	Chrome Mobile iOS 77.0	iOS 13.1.0
-iPhone	Firefox iOS 8.1	iOS 13.1.0
-iPhone	Google 83.0	iOS 13.1.0
-iPhone	Mobile Safari UI/WKWebView 0.0	iOS 13.1.0
-iPhone	DuckDuckGo Mobile 7.0	iOS 13.1.0
-iPhone	Facebook 0.0	iOS 13.1.0
-iPhone	Facebook 0.0	iOS 13.1.0
-iPhone	Facebook 0.0	iOS 13.1.0
-iPhone	Facebook 0.0	iOS 13.1.0
-iPhone	Facebook 0.0	iOS 13.1.0
-iPhone	Mobile Safari 13.0	iOS 13.1.0
-iPhone	Mobile Safari UI/WKWebView 0.0	iOS 13.1.1
-iPhone	Facebook 0.0	iOS 13.1.1
-iPhone	Facebook 0.0	iOS 13.1.1
-iPhone	Facebook 0.0	iOS 13.1.1
-iPhone	Facebook 0.0	iOS 13.1.1
-iPhone	Mobile Safari 13.0	iOS 13.1.1
-iPhone	Mobile Safari UI/WKWebView 0.0	iOS 13.1.2
-iPhone	Facebook 0.0	iOS 13.1.2
-iPhone	Mobile Safari 13.0	iOS 13.1.2
+AppleCoreMedia/1.0.0.12B466 (Apple TV; U; CPU OS 8_1_3 like Mac OS X; en_us)	AppleTV	Other 0.0	ATV OS X 0.0.0
+Mozilla/5.0 (Android 7.0; Mobile; LG-M150; rv:68.0) Gecko/68.0 Firefox/68.0	LG-M150	Firefox Mobile 68.0	Android 7.0.0
+Mozilla/5.0 (Android 8.0.0; Mobile; rv:68.0) Gecko/68.0 Firefox/68.0	Generic Smartphone	Firefox Mobile 68.0	Android 8.0.0
+Mozilla/5.0 (Android 8.1.0; Tablet; rv:68.0) Gecko/68.0 Firefox/68.0	Generic Tablet	Firefox Mobile 68.0	Android 8.1.0
+Mozilla/5.0 (Android 9; Mobile; rv:68.0) Gecko/68.0 Firefox/68.0	Generic Smartphone	Firefox Mobile 68.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 10; PH-1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	PH-1	Chrome Mobile 77.0	Android 10.0.0
+Mozilla/5.0 (Linux; Android 10; Pixel 2 XL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Pixel 2 XL	Chrome Mobile 77.0	Android 10.0.0
+Mozilla/5.0 (Linux; Android 10; Pixel 2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Pixel 2	Chrome Mobile 77.0	Android 10.0.0
+Mozilla/5.0 (Linux; Android 10; Pixel 3 Build/QP1A.190711.020.C3; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/240.0.0.38.121;]	Pixel 3	Facebook 240.0	Android 10.0.0
+Mozilla/5.0 (Linux; Android 10; Pixel XL Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36	Pixel XL	Chrome Mobile WebView 77.0	Android 10.0.0
+Mozilla/5.0 (Linux; Android 10; Pixel XL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Pixel XL	Chrome Mobile 77.0	Android 10.0.0
+Mozilla/5.0 (Linux; Android 4.0.3; HTC Sensation 4G Build/IML74K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36	HTC Sensation 4G	Chrome Mobile 42.0	Android 4.0.3
+Mozilla/5.0 (Linux; Android 4.0.3; KFTT) AppleWebKit/537.36 (KHTML, like Gecko) Silk/73.7.5 like Chrome/73.0.3683.90 Safari/537.36	Kindle	Amazon Silk 73.7	Android 4.0.3
+Mozilla/5.0 (Linux; Android 4.2.2; GT-I9152 Build/JDQ39) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36	Samsung GT-I9152 	Chrome Mobile 42.0	Android 4.2.2
+Mozilla/5.0 (Linux; Android 4.4.2; GT-N5110) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Samsung GT-N5110	Chrome 76.0	Android 4.4.2
+Mozilla/5.0 (Linux; Android 4.4.2; RCT6773W22) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	RCT6773W22	Chrome 77.0	Android 4.4.2
+Mozilla/5.0 (Linux; Android 4.4.2; SM-T217S) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	Samsung SM-T217S	Chrome 77.0	Android 4.4.2
+Mozilla/5.0 (Linux; Android 4.4.2; SM-T530NU) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	Samsung SM-T530NU	Chrome 77.0	Android 4.4.2
+Mozilla/5.0 (Linux; Android 4.4.2; TegraNote-P1640 Build/KOT49H; en-us) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36	TegraNote-P1640	Chrome 69.0	Android 4.4.2
+Mozilla/5.0 (Linux; Android 4.4.3; KFTHWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/76.3.6 like Chrome/76.0.3809.132 Safari/537.36	Kindle	Amazon Silk 76.3	Android 4.4.3
+Mozilla/5.0 (Linux; Android 5.0.2; SM-A500H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.90 Mobile Safari/537.36	Samsung SM-A500H	Chrome Mobile 73.0	Android 5.0.2
+Mozilla/5.0 (Linux; Android 5.0.2; SM-T357T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	Samsung SM-T357T	Chrome 77.0	Android 5.0.2
+Mozilla/5.0 (Linux; Android 5.0.2; SM-T530NU) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Samsung SM-T530NU	Chrome 76.0	Android 5.0.2
+Mozilla/5.0 (Linux; Android 5.0.2; SM-T530NU) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	Samsung SM-T530NU	Chrome 77.0	Android 5.0.2
+Mozilla/5.0 (Linux; Android 5.0; RCT6213W87DK) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 YaBrowser/19.4.1.454.01 Safari/537.36	RCT6213W87DK	Yandex Browser 19.4	Android 5.0.0
+Mozilla/5.0 (Linux; Android 5.0; SM-N900T Build/LRX21V; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/229.0.0.35.117;]	Samsung SM-N900T	Facebook 229.0	Android 5.0.0
+Mozilla/5.0 (Linux; Android 5.1.1) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Focus/4.4.1 Chrome/70.0.3538.110 Mobile Safari/537.36	Generic Smartphone	Chrome Mobile WebView 70.0	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; AFTB) AppleWebKit/537.36 (KHTML, like Gecko) Silk/76.3.16 like Chrome/76.0.3809.132 Safari/537.36	Kindle	Amazon Silk 76.3	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; AFTT Build/LVY48F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.110 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.26	AFTT	Chrome Mobile WebView 70.0	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; AFTT) AppleWebKit/537.36 (KHTML, like Gecko) Silk/76.3.16 like Chrome/76.0.3809.132 Safari/537.36	Kindle	Amazon Silk 76.3	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; KFAUWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/76.3.6 like Chrome/76.0.3809.132 Safari/537.36	Kindle	Amazon Silk 76.3	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; KFDOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/71.2.4 like Chrome/71.0.3578.98 Safari/537.36	Kindle	Amazon Silk 71.2	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; KFDOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/76.3.6 like Chrome/76.0.3809.132 Safari/537.36	Kindle	Amazon Silk 76.3	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/76.3.6 like Chrome/76.0.3809.132 Safari/537.36	Kindle	Amazon Silk 76.3	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; KFGIWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/76.3.6 like Chrome/76.0.3809.132 Safari/537.36	Kindle	Amazon Silk 76.3	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; KFSUWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/76.3.6 like Chrome/76.0.3809.132 Safari/537.36	Kindle	Amazon Silk 76.3	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; KFSUWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/77.1.127 like Chrome/77.0.3865.92 Safari/537.36	Kindle	Amazon Silk 77.1	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; LG-AS330) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LG-AS330	Chrome Mobile 77.0	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; LGL43AL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LGL43AL	Chrome Mobile 77.0	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G530R7 Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.2 Chrome/67.0.3396.87 Mobile Safari/537.36	Samsung SM-G530R7	Samsung Internet 9.2	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-T377P) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Safari/537.36	Samsung SM-T377P	Samsung Internet 10.1	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-T900) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Safari/537.36	Samsung SM-T900	Samsung Internet 10.1	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG-SM-T337A Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36	Samsung SM-T337A	Chrome 69.0	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; SM-G360T1 Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.81 Mobile Safari/537.36	Samsung SM-G360T1	Chrome Mobile 67.0	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; SM-J320FN) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Mobile Safari/537.36	Samsung SM-J320FN	Chrome Mobile 74.0	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; SM-T280) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36	SM-T280	Chrome 74.0	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; SM-T330NU) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36	Samsung SM-T330NU	Chrome 71.0	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; SM-T670) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	SM-T670	Chrome 76.0	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; SM-T670) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	SM-T670	Chrome 77.0	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1.1; Vodafone Smart ultra 6 Build/LMY47V; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/74.0.3729.136 Mobile Safari/537.36	Vodafone Smart ultra 6	Chrome Mobile WebView 74.0	Android 5.1.1
+Mozilla/5.0 (Linux; Android 5.1; BLU Advance 5.0 Build/LMY47I) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.158 Mobile Safari/537.36	BLU Advance 5.0	Chrome Mobile 66.0	Android 5.1.0
+Mozilla/5.0 (Linux; Android 5.1; HTC Desire 626s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	HTC Desire 626s	Chrome Mobile 77.0	Android 5.1.0
+Mozilla/5.0 (Linux; Android 5.1; HUAWEI LUA-L22 Build/HUAWEILUA-L22) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.89 Mobile Safari/537.36	HUAWEI LUA-L22	Chrome Mobile 50.0	Android 5.1.0
+Mozilla/5.0 (Linux; Android 5.1; NX16A11264) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	NX16A11264	Chrome 77.0	Android 5.1.0
+Mozilla/5.0 (Linux; Android 5.1; XT1526) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.90 Mobile Safari/537.36	XT1526	Chrome Mobile 73.0	Android 5.1.0
+Mozilla/5.0 (Linux; Android 6.0.1; CPH1613) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Oppo CPH1613	Chrome Mobile 77.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; LG-M153 Build/MXB48T; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Mobile Safari/537.36	LG-M153 	Chrome Mobile WebView 55.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; LG-M153) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LG-M153	Chrome Mobile 77.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; LGLS676) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LGLS676	Chrome Mobile 77.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; N9136) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Mobile Safari/537.36	N9136	Chrome Mobile 74.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; Nexus 7 Build/MOB30X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.133 Safari/537.36	Asus Nexus 7	Chrome 44.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SAMSUNG SM-G900I) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G900I	Samsung Internet 10.1	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SAMSUNG SM-G900P Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/7.2 Chrome/59.0.3071.125 Mobile Safari/537.36	Samsung SM-G900P	Samsung Internet 7.2	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SAMSUNG SM-J700M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-J700M	Samsung Internet 10.1	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SAMSUNG SM-S327VL) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-S327VL	Samsung Internet 10.1	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SAMSUNG-SM-T377A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	Samsung SM-T377A	Chrome 77.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-G532M Build/MMB29T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.91 Mobile Safari/537.36	Samsung SM-G532M	Chrome Mobile 55.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-G532M Build/MMB29T; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/75.0.3770.101 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/240.0.0.38.121;]	Samsung SM-G532M	Facebook 240.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-G532M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G532M	Chrome Mobile 77.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-G550T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Samsung SM-G550T	Chrome Mobile 76.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-G550T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G550T	Chrome Mobile 77.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-G550T1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Samsung SM-G550T1	Chrome Mobile 76.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-G900V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Mobile Safari/537.36	Samsung SM-G900V	Chrome Mobile 73.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-G920A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G920A	Chrome Mobile 77.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-J327P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J327P	Chrome Mobile 77.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-N910S) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.143 Mobile Safari/537.36	Samsung SM-N910S	Chrome Mobile 75.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-N920V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.89 Mobile Safari/537.36	Samsung SM-N920V	Chrome Mobile 76.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-T350 Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Safari/537.36	Samsung SM-T350	Chrome 59.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-T560NU) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	Samsung SM-T560NU	Chrome 77.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; SM-T800) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	SM-T800	Chrome 77.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; XT1254) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	XT1254	Chrome Mobile 77.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; Z798BL Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36	Z798BL	Chrome Mobile 67.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0.1; Z799VL Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/45.0.2454.95 Mobile Safari/537.36	Z799VL	Chrome Mobile WebView 45.0	Android 6.0.1
+Mozilla/5.0 (Linux; Android 6.0; 5010X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.89 Mobile Safari/537.36	5010X	Chrome Mobile 76.0	Android 6.0.0
+Mozilla/5.0 (Linux; Android 6.0; CAM-L21) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Huawei CAM-L21	Chrome Mobile 77.0	Android 6.0.0
+Mozilla/5.0 (Linux; Android 6.0; F3313) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	F3313	Chrome Mobile 77.0	Android 6.0.0
+Mozilla/5.0 (Linux; Android 6.0; RCT6603W47M7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	RCT6603W47M7	Chrome 77.0	Android 6.0.0
+Mozilla/5.0 (Linux; Android 7.0; 5049Z Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Mobile Safari/537.36	5049Z	Chrome Mobile 56.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; ASUS_A002A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Asus A002A	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; Alcatel_5044C) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Alcatel_5044C	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; Astra Young Pro Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/59.0.3071.125 Mobile Safari/537.36	Astra Young Pro	Chrome Mobile WebView 59.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; Infinix X571) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Infinix X571	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; LG-H872 Build/NRD90U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.123 Mobile Safari/537.36	LG-H872 	Chrome Mobile 64.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; LG-K425 Build/NRD90U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.91 Mobile Safari/537.36	LG-K425 	Chrome Mobile 55.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; LG-LS777) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LG-LS777	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; LG-M210) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LG-M210	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; LG-M430) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LG-M430	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; LG-TP260 Build/NRD90U; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/64.0.3282.137 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	LG-TP260 	Chrome Mobile WebView 64.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; LG-TP260) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LG-TP260	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; LG-TP450 Build/NRD90U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.137 Mobile Safari/537.36	LG-TP450 	Chrome Mobile 64.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; LG-V521) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.143 Safari/537.36	LG-V521	Chrome 75.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; LG-V521) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	LG-V521	Chrome 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; LGMP260 Build/NRD90U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36	LGMP260	Chrome Mobile 58.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; LGMS210 Build/NRD90U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.91 Mobile Safari/537.36	LGMS210	Chrome Mobile 55.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; LGMS210) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LGMS210	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; P00I) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	P00I	Chrome 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; RS988) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	RS988	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-J701F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-J701F	Samsung Internet 10.1	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-J710F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-J710F	Samsung Internet 10.1	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-N920T Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.2 Chrome/67.0.3396.87 Mobile Safari/537.36	Samsung SM-N920T	Samsung Internet 9.2	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SAMSUNG-SM-G920A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G920A	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-G920P Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 Flipboard/4.2.23/4722,4.2.23.4722	Samsung SM-G920P	Flipboard 4.2	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-G920V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Samsung SM-G920V	Chrome Mobile 76.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-G928V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G928V	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-G950U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G950U	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-G955U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G955U	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-J327T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Mobile Safari/537.36	Samsung SM-J327T	Chrome Mobile 74.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-J327T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J327T	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-J327T1 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.137 Mobile Safari/537.36	Samsung SM-J327T1	Chrome Mobile 64.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-J327T1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.101 Mobile Safari/537.36	Samsung SM-J327T1	Chrome Mobile 75.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-J327T1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J327T1	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-N9208) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.90 Mobile Safari/537.36	Samsung SM-N9208	Chrome Mobile 73.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-N920P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Mobile Safari/537.36	Samsung SM-N920P	Chrome Mobile 74.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-N920T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-N920T	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-T585) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	SM-T585	Chrome 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-T810) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.143 Safari/537.36	SM-T810	Chrome 75.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-T810) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Safari/537.36	SM-T810	Chrome 76.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-T810) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	SM-T810	Chrome 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-T813) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Safari/537.36	SM-T813	Chrome 76.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; SM-T813) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	SM-T813	Chrome 76.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; ST1009X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.143 Safari/537.36	Trekstor ST1009X	Chrome 75.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0; XT1663) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	XT1663	Chrome Mobile 77.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.96 Mobile Safari/537.36	Generic Smartphone	Chrome Mobile 58.0	Android 7.0.0
+Mozilla/5.0 (Linux; Android 7.1.1; A574BL Build/NMF26F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	A574BL	Chrome Mobile WebView 77.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; A574BL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	A574BL	Chrome Mobile 77.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; CPH1729 Build/N6F26Q; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/240.0.0.38.121;]	Oppo CPH1729	Facebook 240.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; Coolpad 3632A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Mobile Safari/537.36	3632A	Chrome Mobile 74.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; General Mobile 4G Dual) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	General Mobile 4G Dual	Chrome Mobile 77.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; Moto E (4) Plus Build/NCRS26.58-44-20; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/76.0.3809.111 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	Moto E (4) Plus	Chrome Mobile WebView 76.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; Moto E (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.80 Mobile Safari/537.36	Moto E (4)	Chrome Mobile 70.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; Moto E (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Moto E (4)	Chrome Mobile 76.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; Moto E (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.73 Mobile Safari/537.36	Moto E (4)	Chrome Mobile 77.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; Moto E (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Moto E (4)	Chrome Mobile 77.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; NX591J) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	NX591J	Chrome Mobile 77.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; REVVLPLUS C3701A Build/143.54.190611.3701A-TMO) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.137 Mobile Safari/537.36	REVVLPLUS C3701A	Chrome Mobile 64.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; SAMSUNG SM-J320A) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-J320A	Samsung Internet 10.1	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; SAMSUNG SM-T550) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Safari/537.36	Samsung SM-T550	Samsung Internet 10.1	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; SAMSUNG-SM-T377A Build/NMF26X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.137 Safari/537.36	Samsung SM-T377A	Chrome 64.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; SM-J250F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Samsung SM-J250F	Chrome Mobile 76.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; SM-J700T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J700T	Chrome Mobile 77.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; SM-T350) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	SM-T350	Chrome 77.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; SM-T377T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.73 Safari/537.36	Samsung SM-T377T	Chrome 77.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; SM-T550 Build/NMF26X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36	Samsung SM-T550	Chrome 69.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; SM-T550) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	SM-T550	Chrome 77.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; SM-T560NU) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	Samsung SM-T560NU	Chrome 77.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; X20 Build/N6F26Q; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.100 Mobile Safari/537.36	X20	Chrome Mobile WebView 52.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; Z851M Build/NMF26V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36	Z851M	Chrome Mobile 58.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; Z899VL Build/NMF26V; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/74.0.3729.136 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	Z899VL	Chrome Mobile WebView 74.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; Z982 Build/NMF26V; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/75.0.3770.143 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	Z982	Chrome Mobile WebView 75.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.1; Z982) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Z982	Chrome Mobile 77.0	Android 7.1.1
+Mozilla/5.0 (Linux; Android 7.1.2) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Focus/4.4.1 Chrome/70.0.3538.110 Mobile Safari/537.36	Generic Smartphone	Chrome Mobile WebView 70.0	Android 7.1.2
+Mozilla/5.0 (Linux; Android 7.1.2; AFTKMST12 Build/NS6265; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.110 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.26	AFTKMST12	Chrome Mobile WebView 70.0	Android 7.1.2
+Mozilla/5.0 (Linux; Android 7.1.2; AFTKMST12) AppleWebKit/537.36 (KHTML, like Gecko) Silk/76.3.16 like Chrome/76.0.3809.132 Safari/537.36	Kindle	Amazon Silk 76.3	Android 7.1.2
+Mozilla/5.0 (Linux; Android 7.1.2; AFTMM Build/NS6265; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.110 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.26	AFTMM	Chrome Mobile WebView 70.0	Android 7.1.2
+Mozilla/5.0 (Linux; Android 7.1.2; AFTN Build/NS6265; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.110 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.26	AFTN	Chrome Mobile WebView 70.0	Android 7.1.2
+Mozilla/5.0 (Linux; Android 7.1.2; KFKAWI Build/NS6301; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/59.0.3071.125 Safari/537.36	KFKAWI	Chrome Mobile WebView 59.0	Android 7.1.2
+Mozilla/5.0 (Linux; Android 7.1.2; KFKAWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/76.3.6 like Chrome/76.0.3809.132 Safari/537.36	Kindle	Amazon Silk 76.3	Android 7.1.2
+Mozilla/5.0 (Linux; Android 7.1.2; KFMUWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/76.3.6 like Chrome/76.0.3809.132 Safari/537.36	Kindle	Amazon Silk 76.3	Android 7.1.2
+Mozilla/5.0 (Linux; Android 7.1.2; LG-SP200) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.143 Mobile Safari/537.36	LG-SP200	Chrome Mobile 75.0	Android 7.1.2
+Mozilla/5.0 (Linux; Android 7.1.2; LG-SP200) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	LG-SP200	Chrome Mobile 76.0	Android 7.1.2
+Mozilla/5.0 (Linux; Android 7.1.2; LM-X210(G)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Mobile Safari/537.36	LM-X210(G)	Chrome Mobile 76.0	Android 7.1.2
+Mozilla/5.0 (Linux; Android 7.1.2; LM-X210) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	LM-X210	Chrome Mobile 76.0	Android 7.1.2
+Mozilla/5.0 (Linux; Android 7.1.2; RCT6973W43R) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	RCT6973W43R	Chrome 77.0	Android 7.1.2
+Mozilla/5.0 (Linux; Android 7.1.2; Redmi 4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	XiaoMi Redmi 4	Chrome Mobile 77.0	Android 7.1.2
+Mozilla/5.0 (Linux; Android 8.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/76.0.3809.132 Mobile Safari/537.36	Generic Smartphone	Chrome Mobile WebView 76.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; ASUS_Z01FD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Asus Z01FD	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; AUM-L29) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Huawei AUM-L29	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; BRAVIA 4K GB Build/OPR2.170623.027.S25; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	BRAVIA 4K GB	Chrome Mobile WebView 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; CMR-W09) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	CMR-W09	Chrome 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; EVA-AL00) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.73 Mobile Safari/537.36	EVA-AL00	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; G3223) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	G3223	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; LG-H910) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LG-H910	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; LG-H931) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	LG-H931	Chrome Mobile 76.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; LG-H932) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LG-H932	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SAMSUNG SM-A520F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-A520F	Samsung Internet 10.1	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SAMSUNG SM-G891A Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/8.2 Chrome/63.0.3239.111 Mobile Safari/537.36	Samsung SM-G891A	Samsung Internet 8.2	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SAMSUNG SM-G935T) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G935T	Samsung Internet 10.1	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SAMSUNG SM-G955U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G955U	Samsung Internet 10.1	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SAMSUNG SM-J337T Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.2 Chrome/67.0.3396.87 Mobile Safari/537.36	Samsung SM-J337T	Samsung Internet 9.2	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SAMSUNG SM-J737P) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-J737P	Samsung Internet 10.1	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SAMSUNG SM-N950F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-N950F	Samsung Internet 10.1	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SAMSUNG-SM-G891A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36	Samsung SM-G891A	Chrome Mobile 72.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SAMSUNG-SM-G935A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Samsung SM-G935A	Chrome Mobile 76.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-A720F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-A720F	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-G570F Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/73.0.3683.90 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/231.0.0.39.113;]	Samsung SM-G570F	Facebook 231.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-G570Y) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G570Y	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-G930T Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	Samsung SM-G930T	Chrome Mobile WebView 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-G930V Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.137 Mobile Safari/537.36	Samsung SM-G930V	Chrome Mobile 64.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-G930VL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Mobile Safari/537.36	Samsung SM-G930VL	Chrome Mobile 74.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-G935F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.143 Mobile Safari/537.36	Samsung SM-G935F	Chrome Mobile 75.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-G935P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G935P	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-G935T Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/240.0.0.38.121;]	Samsung SM-G935T	Facebook 240.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-G935T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G935T	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-G950U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G950U	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-G955U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.112 Mobile Safari/537.36	Samsung SM-G955U	Chrome Mobile 74.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-G955U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G955U	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-J330G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J330G	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-J337T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J337T	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-J737A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J737A	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-J737T1 Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.126 Mobile Safari/537.36	Samsung SM-J737T1	Chrome Mobile 66.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-J737T1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J737T1	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-N950F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.126 Mobile Safari/537.36	Samsung SM-N950F	Chrome Mobile 66.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-N950U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Samsung SM-N950U	Chrome Mobile 76.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-N950U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-N950U	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-N950U1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-N950U1	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; SM-S367VL Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36 OPT/1.22.80	Samsung SM-S367VL	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; VS995) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	VS995	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; XT1635-02) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	XT1635-02	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; moto e5 play) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Mobile Safari/537.36	moto e5 play	Chrome Mobile 76.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; moto e5 play) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	moto e5 play	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; moto e5 supra) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	moto e5 supra	Chrome Mobile 76.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.0.0; moto g(6)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	moto g(6)	Chrome Mobile 77.0	Android 8.0.0
+Mozilla/5.0 (Linux; Android 8.1.0; 5041C) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	5041C	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; 6062W) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	6062W	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; A502DL Build/OPM1.171019.011) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36	A502DL	Chrome Mobile 67.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; A502DL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	A502DL	Chrome Mobile 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; BKK-LX2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Huawei BKK-LX2	Chrome Mobile 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; C4 Build/OPM2.171019.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Mobile Safari/537.36	C4	Chrome Mobile 70.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; Coolpad 3310A Build/3310A.SPRINT.190213.0S) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	3310A	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; Infinix X604 Build/O11019) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.137 Mobile Safari/537.36	Infinix X604	Chrome Mobile 64.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; Joy 1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Joy 1	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LAVA LE9820) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LAVA LE9820	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LG-Q710AL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LG-Q710AL	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-Q610(FGN)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LM-Q610(FGN)	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-Q710(FGN) Build/OPM1.171019.019; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/235.0.0.38.118;]	LM-Q710(FGN)	Facebook 235.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-Q710(FGN)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Mobile Safari/537.36	LM-Q710(FGN)	Chrome Mobile 70.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-Q710(FGN)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Mobile Safari/537.36	LM-Q710(FGN)	Chrome Mobile 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-Q710(FGN)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	LM-Q710(FGN)	Chrome Mobile 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-Q710(FGN)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LM-Q710(FGN)	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-V405) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LM-V405	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-X210(G) Build/OPM1.171019.026; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 agentweb/4.0.2  UCBrowser/11.6.4.950	LM-X210(G)	UC Browser 11.6	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-X210(G)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Mobile Safari/537.36	LM-X210(G)	Chrome Mobile 70.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-X210(G)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.105 Mobile Safari/537.36	LM-X210(G)	Chrome Mobile 72.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-X210(G)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LM-X210(G)	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-X212(G)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LM-X212(G)	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-X220) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Mobile Safari/537.36	LM-X220	Chrome Mobile 70.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-X220) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	LM-X220	Chrome Mobile 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-X220PM Build/O11019; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	LM-X220PM	Chrome Mobile WebView 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-X410(FG)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Mobile Safari/537.36	LM-X410(FG)	Chrome Mobile 70.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-X410(FG)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Mobile Safari/537.36	LM-X410(FG)	Chrome Mobile 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-X410(FG)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LM-X410(FG)	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LM-X410.FGN Build/OPM1.171019.019) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.91 Mobile Safari/537.36	LM-X410.FGN	Chrome Mobile 68.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LML414DL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	LML414DL	Chrome Mobile 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; LML713DL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LML713DL	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; Moto G (5S) Plus) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Moto G (5S) Plus	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; One) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.110 Mobile Safari/537.36/TansoDL	HTC One	Chrome Mobile WebView 70.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; RCT6873W42BMF8KC Build/O11019) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	RCT6873W42BMF8KC	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; REVVL 2 Build/OPM1.171019.011) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36	REVVL 2	Chrome Mobile 67.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; REVVL 2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	REVVL 2	Chrome Mobile 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SAMSUNG SM-J727T) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-J727T	Samsung Internet 10.1	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SAMSUNG SM-J727T1 Build/M1AJQ) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.4 Chrome/67.0.3396.87 Mobile Safari/537.36	Samsung SM-J727T1	Samsung Internet 9.4	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SAMSUNG SM-J727T1) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-J727T1	Samsung Internet 10.1	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SAMSUNG SM-T580 Build/M1AJQ) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.4 Chrome/67.0.3396.87 Safari/537.36	Samsung SM-T580	Samsung Internet 9.4	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SAMSUNG-SM-J727A Build/M1AJQ; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/240.0.0.38.121;]	Samsung SM-J727A	Facebook 240.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-G610F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G610F	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-J260T1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Mobile Safari/537.36	Samsung SM-J260T1	Chrome Mobile 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-J260T1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Samsung SM-J260T1	Chrome Mobile 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-J260T1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J260T1	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-J410F Build/M1AJB) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J410F	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-J727P Build/M1AJQ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.91 Mobile Safari/537.36	Samsung SM-J727P	Chrome Mobile 68.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-J727T Build/M1AJQ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.126 Mobile Safari/537.36	Samsung SM-J727T	Chrome Mobile 66.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-J727T1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Mobile Safari/537.36	Samsung SM-J727T1	Chrome Mobile 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-J727T1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.73 Mobile Safari/537.36	Samsung SM-J727T1	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-J727T1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J727T1	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-J727V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Mobile Safari/537.36	Samsung SM-J727V	Chrome Mobile 70.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-J727V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J727V	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-P580) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	SM-P580	Chrome 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-T380) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.143 Safari/537.36	SM-T380	Chrome 75.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-T580) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.90 Safari/537.36 EdgA/42.0.2.3928	SM-T580	Edge Mobile 42.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-T580) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	SM-T580	Chrome 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-T580) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.89 Safari/537.36	SM-T580	Chrome 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-T580) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	SM-T580	Chrome 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; SM-T837T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	Samsung SM-T837T	Chrome 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; TECNO CF8 Build/O11019; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/239.0.0.41.152;]	TECNO CF8	Facebook 239.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; V1818CA) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.101 Mobile Safari/537.36	V1818CA	Chrome Mobile 75.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; meizu C9 Build/OPM2.171019.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.91 Mobile Safari/537.36	meizu C9	Chrome Mobile 68.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; vivo 1724) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	vivo 1724	Chrome Mobile 76.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 8.1.0; vivo 1814) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	vivo 1814	Chrome Mobile 77.0	Android 8.1.0
+Mozilla/5.0 (Linux; Android 9) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 DuckDuckGo/5	Generic Smartphone	DuckDuckGo Mobile 5.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; 1825) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Mobile Safari/537.36	1825	Chrome Mobile 70.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; ANE-LX2 Build/HUAWEIANE-L22; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/76.0.3809.132 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/236.0.0.40.117;]	ANE-LX2	Facebook 236.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; BLA-A09) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	BLA-A09	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; CLT-L04) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Huawei CLT-L04	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; CPH1911 Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/239.0.0.41.152;]	Oppo CPH1911	Facebook 239.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; CPH1923 Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/76.0.3809.111 Mobile Safari/537.36	Oppo CPH1923	Chrome Mobile WebView 76.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; ELE-L29) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Huawei ELE-L29	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; G8142) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	G8142	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; GM1911) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Mobile Safari/537.36	GM1911	Chrome Mobile 76.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; GM1917) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	GM1917	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; INE-LX2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Mobile Safari/537.36	Huawei INE-LX2	Chrome Mobile 76.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; LM-G710 Build/PKQ1.181105.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36	LM-G710	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; LM-Q720) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LM-Q720	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; LM-V405 Build/PKQ1.190202.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.15	LM-V405	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; LM-V405) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Mobile Safari/537.36	LM-V405	Chrome Mobile 76.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; LM-V500N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LM-V500N	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; LM-X420) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36	LM-X420	Chrome Mobile 72.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; LM-X420) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	LM-X420	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; MAR-LX1A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	MAR-LX1A	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; MI 9) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	XiaoMi MI 9	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; Mi A2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	XiaoMi Mi A2	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; Moto Z (2)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Moto Z (2)	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; Nokia 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Nokia 6	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; ONEPLUS A6000) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	OnePlus ONEPLUS A6000	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; ONEPLUS A6003) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	OnePlus ONEPLUS A6003	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; ONEPLUS A6013 Build/PKQ1.180716.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36	OnePlus ONEPLUS A6013	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; ONEPLUS A6013) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Mobile Safari/537.36	OnePlus ONEPLUS A6013	Chrome Mobile 74.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; ONEPLUS A6013) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	OnePlus ONEPLUS A6013	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; PAR-AL00 Build/HUAWEIPAR-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/235.0.0.38.118;]	PAR-AL00	Facebook 235.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; Pixel 2 XL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Pixel 2 XL	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; Pixel 3 Build/PQ1A.190105.004; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36	Pixel 3	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; Pixel 3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Mobile Safari/537.36	Pixel 3	Chrome Mobile 76.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; Pixel 3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Pixel 3	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; Pixel 3a XL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Pixel 3a XL	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; REVVLRY ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.90 Mobile Safari/537.36	REVVLRY 	Chrome Mobile 73.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; RMX1801) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.101 Mobile Safari/537.36	Oppo RMX1801	Chrome Mobile 75.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; Redmi 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	XiaoMi Redmi 7	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; Redmi Note 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.89 Mobile Safari/537.36	XiaoMi Redmi Note 7	Chrome Mobile 76.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-A102U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-A102U	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-A505FN) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-A505FN	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-A505GN) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-A505GN	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-G892U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G892U	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-G950U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G950U	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-G955F Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.4 Chrome/67.0.3396.87 Mobile Safari/537.36	Samsung SM-G955F	Samsung Internet 9.4	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-G955U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G955U	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-G9600 Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.4 Chrome/67.0.3396.87 Mobile Safari/537.36	Samsung SM-G9600	Samsung Internet 9.4	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-G960U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G960U	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-G965U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G965U	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-G970F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G970F	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-G970U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G970U	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-G973U Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.4 Chrome/67.0.3396.87 Mobile Safari/537.36	Samsung SM-G973U	Samsung Internet 9.4	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-G973U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G973U	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-G975U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G975U	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-J415F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-J415F	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-J730F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-J730F	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-J737P) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-J737P	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-J737T Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.0 Chrome/67.0.3396.87 Mobile Safari/537.36	Samsung SM-J737T	Samsung Internet 9.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-N950U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-N950U	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-N960F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-N960F	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-N960U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-N960U	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-N960U1 Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.2 Chrome/67.0.3396.87 Mobile Safari/537.36	Samsung SM-N960U1	Samsung Internet 9.2	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-N970U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-N970U	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-N975U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-N975U	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-N975U1) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-N975U1	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-T510) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Safari/537.36	Samsung SM-T510	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-T720) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Safari/537.36	Samsung SM-T720	Samsung Internet 10.1	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SHIELD Android TV Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	SHIELD Android TV	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-A102U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36	Samsung SM-A102U	Chrome Mobile 72.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-A102U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-A102U	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-A105M Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/237.0.0.44.120;]	Samsung SM-A105M	Facebook 237.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-A205G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-A205G	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-A205U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-A205U	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-A505F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-A505F	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-A530F Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/240.0.0.38.121;]	Samsung SM-A530F	Facebook 240.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-A530N Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36;KAKAOTALK 1908560	Samsung SM-A530N	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-A600T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-A600T	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-A605F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-A605F	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-A920F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-A920F	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G892A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.136 Mobile Safari/537.36	Samsung SM-G892A	Chrome Mobile 74.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G950F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G950F	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G950U Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	Samsung SM-G950U	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G950U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G950U	Chrome Mobile 71.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G950U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Mobile Safari/537.36	Samsung SM-G950U	Chrome Mobile 76.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G950U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Samsung SM-G950U	Chrome Mobile 76.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G950U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G950U	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G950U1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G950U1	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G955F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G955F	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G955U Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.73 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/240.0.0.38.121;]	Samsung SM-G955U	Facebook 240.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G955U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G955U	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G9600) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G9600	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G960U Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/73.0.3683.90 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/233.0.0.36.117;]	Samsung SM-G960U	Facebook 233.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G960U Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	Samsung SM-G960U	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.99 Mobile Safari/537.36	Samsung SM-G960U	Chrome Mobile 71.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.136 Mobile Safari/537.36	Samsung SM-G960U	Chrome Mobile 74.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G960U	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G960U1 Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/240.0.0.38.121;]	Samsung SM-G960U1	Facebook 240.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G960U1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G960U1	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G965F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G965F	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G965U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.136 Mobile Safari/537.36	Samsung SM-G965U	Chrome Mobile 74.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G965U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G965U	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G965U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3921.2 Mobile Safari/537.36	Samsung SM-G965U	Chrome Mobile 79.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G965U1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G965U1	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G970U Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/240.0.0.38.121;]	Samsung SM-G970U	Facebook 240.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G970U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.101 Mobile Safari/537.36	Samsung SM-G970U	Chrome Mobile 75.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G970U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G970U	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G970U1 Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G970U1	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G973U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.136 Mobile Safari/537.36	Samsung SM-G973U	Chrome Mobile 74.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G973U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G973U	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G973U1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G973U1	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G975U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.101 Mobile Safari/537.36	Samsung SM-G975U	Chrome Mobile 75.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G975U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Samsung SM-G975U	Chrome Mobile 76.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G975U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G975U	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-G975U1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-G975U1	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-J260A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J260A	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-J337P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Samsung SM-J337P	Chrome Mobile 76.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-J600FN) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.89 Mobile Safari/537.36	Samsung SM-J600FN	Chrome Mobile 75.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-J600G Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.73 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/238.0.0.41.116;]	Samsung SM-J600G	Facebook 238.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-J730F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J730F	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-J737A Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	Samsung SM-J737A	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-J737A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.136 Mobile Safari/537.36	Samsung SM-J737A	Chrome Mobile 74.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-J737V Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/75.0.3770.101 Mobile Safari/537.36 [Pinterest/Android]	Samsung SM-J737V	Pinterest 0.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-J737V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J737V	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-J810M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-J810M	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N950U Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/240.0.0.38.121;]	Samsung SM-N950U	Facebook 240.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N950U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36	Samsung SM-N950U	Chrome Mobile 72.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N950U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.101 Mobile Safari/537.36	Samsung SM-N950U	Chrome Mobile 75.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N950U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.73 Mobile Safari/537.36	Samsung SM-N950U	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N950U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-N950U	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N960F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Samsung SM-N960F	Chrome Mobile 76.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N960U Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/74.0.3729.136 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/240.0.0.38.121;]	Samsung SM-N960U	Facebook 240.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N960U Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	Samsung SM-N960U	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Mobile Safari/537.36	Samsung SM-N960U	Chrome Mobile 74.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.143 Mobile Safari/537.36	Samsung SM-N960U	Chrome Mobile 75.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Samsung SM-N960U	Chrome Mobile 76.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-N960U	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N960U1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-N960U1	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N975U Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.73 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	Samsung SM-N975U	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N975U Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	Samsung SM-N975U	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N975U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-N975U	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-N976V Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/240.0.0.38.121;]	Samsung SM-N976V	Facebook 240.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-S367VL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Samsung SM-S367VL	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-S767VL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36	Samsung SM-S767VL	Chrome Mobile 76.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-T597P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	Samsung SM-T597P	Chrome 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; SM-T720) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36	SM-T720	Chrome 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; TECNO KC8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	TECNO KC8	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; VOG-L29) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	Huawei VOG-L29	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; cp3705A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.136 Mobile Safari/537.36	cp3705A	Chrome Mobile 74.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; moto g(6) Build/PPS29.118-15-11; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.92 Mobile Safari/537.36;dailymotion-player-sdk-android 0.1.31	moto g(6)	Chrome Mobile WebView 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; moto g(6) play) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	moto g(6) play	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; moto g(7) play Build/PCYS29.105-134-1; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/76.0.3809.132 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/235.0.0.38.118;]	moto g(7) play	Facebook 235.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; moto g(7) play) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.80 Mobile Safari/537.36	moto g(7) play	Chrome Mobile 70.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; moto g(7) power) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.101 Mobile Safari/537.36	moto g(7) power	Chrome Mobile 75.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; moto g(7) power) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Mobile Safari/537.36	moto g(7) power	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; moto z4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.90 Mobile Safari/537.36	moto z4	Chrome Mobile 73.0	Android 9.0.0
+Mozilla/5.0 (Linux; Android 9; moto z4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.73 Mobile Safari/537.36	moto z4	Chrome Mobile 77.0	Android 9.0.0
+Mozilla/5.0 (Linux; U; Android 4.1.1; en-us; GT-P3113 Build/JRO03C) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30	Samsung GT-P3113 	Android 4.1	Android 4.1.1
+Mozilla/5.0 (Linux; U; Android 4.1.2; ar-ae; GT-I8160 Build/JZO54K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30	Samsung GT-I8160 	Android 4.1	Android 4.1.2
+Mozilla/5.0 (Linux; U; Android 4.2.2; en-us; Nexus 7 Build/JDQ39) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30; DailymotionEmbedSDK 1.0	Asus Nexus 7	Android 4.2	Android 4.2.2
+Mozilla/5.0 (Linux; U; Android 4.4; en-us; SM-E500H Build/JOP24G) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30	Samsung SM-E500H	Android 4.4	Android 4.4.0
+Mozilla/5.0 (Linux; U; Android 6.0.1; en-us; LGMS550 Build/JOP24G) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Chrome/43.0.2357.65 Mobile Safari/534.30	LGMS550	Chrome Mobile WebView 43.0	Android 6.0.1
+Mozilla/5.0 (Linux; U; Android 6.0.1; en-us; SM-J737T1 Build/JOP24G) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Chrome/43.0.2357.65 Mobile Safari/534.30	Samsung SM-J737T1	Chrome Mobile WebView 43.0	Android 6.0.1
+Mozilla/5.0 (Linux; U; Android 7.0; TECNO CA6 Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/64.0.3282.137 Mobile Safari/537.36 OPR/5.3.2254.135058	TECNO CA6	Opera Mobile 5.3	Android 7.0.0
+Mozilla/5.0 (Linux; U; Android 7.1.2; id-id; Redmi 5A Build/N2G47H) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/53.0.2785.146 Mobile Safari/537.36 XiaoMi/MiuiBrowser/9.5.6	XiaoMi Redmi 5A	MiuiBrowser 9.5	Android 7.1.2
+Mozilla/5.0 (Linux; U; Android 9; in-id; CPH1911 Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.80 Mobile Safari/537.36 OppoBrowser/25.6.0.0.5beta	Oppo CPH1911	Chrome Mobile WebView 70.0	Android 9.0.0
+Mozilla/5.0 (Linux; U; Android 9; vivo 1904 Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/74.0.3729.136 Mobile Safari/537.36 OPR/44.1.2254.143214	vivo 1904	Opera Mobile 44.1	Android 9.0.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:68.0) Gecko/20100101 Firefox/68.0	Mac	Firefox 68.0	Mac OS X 10.11.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:69.0) Gecko/20100101 Firefox/69.0	Mac	Firefox 69.0	Mac OS X 10.13.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:67.0) Gecko/20100101 Firefox/67.0	Mac	Firefox 67.0	Mac OS X 10.14.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:68.0) Gecko/20100101 Firefox/68.0	Mac	Firefox 68.0	Mac OS X 10.14.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:69.0) Gecko/20100101 Firefox/69.0	Mac	Firefox 69.0	Mac OS X 10.14.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:70.0) Gecko/20100101 Firefox/70.0	Mac	Firefox 70.0	Mac OS X 10.14.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.10.5
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Mac	Chrome 77.0	Mac OS X 10.10.5
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8	Mac	Safari 10.1	Mac OS X 10.10.5
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.11.4
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36	Mac	Chrome 72.0	Mac OS X 10.11.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.11.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.11.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Mac	Chrome 77.0	Mac OS X 10.11.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/601.7.7 (KHTML, like Gecko) Version/9.1.2 Safari/601.7.7	Mac	Safari 9.1	Mac OS X 10.11.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14	Mac	Safari 10.0	Mac OS X 10.11.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15	Mac	Safari 11.1	Mac OS X 10.11.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Mac	Chrome 77.0	Mac OS X 10.12.1
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/602.4.8 (KHTML, like Gecko) Version/10.0.3 Safari/602.4.8	Mac	Safari 10.0	Mac OS X 10.12.3
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36	Mac	Chrome 75.0	Mac OS X 10.12.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.12.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.12.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Mac	Chrome 77.0	Mac OS X 10.12.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15	Mac	Safari 12.1	Mac OS X 10.12.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Safari/604.1.38	Mac	Safari 11.0	Mac OS X 10.13.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Mac	Chrome 77.0	Mac OS X 10.13.1
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Mac	Chrome 77.0	Mac OS X 10.13.2
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.13.4
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.13.4
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.13.5
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36	Mac	Chrome 75.0	Mac OS X 10.13.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.13.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Mac	Chrome 77.0	Mac OS X 10.13.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0.3 Safari/605.1.15	Mac	Safari 12.0	Mac OS X 10.13.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15	Mac	Safari 12.1	Mac OS X 10.13.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15	Mac	Safari 12.1	Mac OS X 10.13.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Safari/605.1.15	Mac	Safari 13.0	Mac OS X 10.13.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.1 Safari/605.1.15	Mac	Safari 13.0	Mac OS X 10.13.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36	Mac	Chrome 75.0	Mac OS X 10.14.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.14.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Mac	Chrome 77.0	Mac OS X 10.14.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Mac	Chrome 77.0	Mac OS X 10.14.1
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.14.2
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36	Mac	Chrome 69.0	Mac OS X 10.14.3
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0.3 Safari/605.1.15	Mac	Safari 12.0	Mac OS X 10.14.3
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36	Mac	Chrome 75.0	Mac OS X 10.14.4
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Mac	Chrome 77.0	Mac OS X 10.14.4
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Safari/605.1.15	Mac	Safari 12.1	Mac OS X 10.14.4
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.14.5
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Mac	Chrome 77.0	Mac OS X 10.14.5
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15	Mac	Safari 12.1	Mac OS X 10.14.5
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36	Mac	Chrome 75.0	Mac OS X 10.14.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.14.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Mac	Chrome 76.0	Mac OS X 10.14.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36	Mac	Chrome 77.0	Mac OS X 10.14.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Mac	Chrome 77.0	Mac OS X 10.14.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15	Mac	Safari 12.1	Mac OS X 10.14.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.1 Safari/605.1.15	Mac	Safari 13.0	Mac OS X 10.14.6
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36	Mac	Chrome 65.0	Mac OS X 10.9.5
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36	Mac	Chrome 66.0	Mac OS X 10.9.5
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36	Mac	Chrome 67.0	Mac OS X 10.9.5
+Mozilla/5.0 (PlayStation 4 6.72) AppleWebKit/605.1.15 (KHTML, like Gecko)	PlayStation 4	Apple Mail 605.1	Other 0.0.0
+Mozilla/5.0 (SMART-TV; LINUX; Tizen 3.0) AppleWebKit/538.1 (KHTML, like Gecko) Version/3.0 TV Safari/538.1	Samsung SMART-TV	Safari 3.0	Tizen 3.0.0
+Mozilla/5.0 (SMART-TV; Linux; Tizen 3.0) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/2.0 Chrome/47.0.2526.69 TV safari/537.36	Samsung SMART-TV	Samsung Internet 2.0	Tizen 3.0.0
+Mozilla/5.0 (SMART-TV; Linux; Tizen 4.0) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/2.1 Chrome/56.0.2924.0 TV Safari/537.36	Samsung SMART-TV	Samsung Internet 2.1	Tizen 4.0.0
+Mozilla/5.0 (SMART-TV; Linux; Tizen 5.0) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/2.2 Chrome/63.0.3239.84 TV Safari/537.36	Samsung SMART-TV	Samsung Internet 2.2	Tizen 5.0.0
+Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134	Other	Edge 17.17134	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/18.17763	Other	Edge 18.17763	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36	Other	Chrome 77.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.7.5000	Other	Maxthon 5.2	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.1.3683.41 Safari/537.36	Other	Chrome 73.1	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36	Other	Chrome 76.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36 OPR/63.0.3368.94	Other	Opera 63.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36	Other	Chrome 77.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Other	Chrome 77.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) coc_coc_browser/82.0.144 Chrome/76.0.3809.144 Safari/537.36	Other	Coc Coc 82.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko	Other	IE 11.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; WOW64; rv:59.0) Gecko/20100101 Firefox/59.0	Other	Firefox 59.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; WOW64; rv:60.0) Gecko/20100101 Firefox/60.0	Other	Firefox 60.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063	Other	Edge 15.15063	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299	Other	Edge 16.16299	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134	Other	Edge 17.17134	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/18.17763	Other	Edge 18.17763	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36	Other	Chrome 65.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36	Other	Chrome 70.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18362	Other	Edge 18.18362	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18995	Other	Edge 18.18995	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19493	Other	Edge 18.19493	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36	Other	Chrome 70.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36	Other	Chrome 71.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36	Other	Chrome 73.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36	Other	Chrome 74.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36	Other	Chrome 75.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36	Other	Chrome 76.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.110 Safari/537.36 Vivaldi/2.7.1628.30	Other	Vivaldi 2.7	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Other	Chrome 76.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36 OPR/63.0.3368.94	Other	Opera 63.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36	Other	Chrome 77.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Other	Chrome 77.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3907.0 Safari/537.36 Edg/79.0.279.0	Other	Edge 79.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; Xbox; Xbox One) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18362	Other	Edge 18.18362	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; Xbox; Xbox One) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363	Other	Edge 18.18363	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; Xbox; Xbox One; WebView/3.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18362	Other	Edge 18.18362	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0	Other	Firefox 61.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0	Other	Firefox 63.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0	Other	Firefox 67.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:68.0) Gecko/20100101 Firefox/68.0	Other	Firefox 68.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0	Other	Firefox 69.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 10.0; rv:69.0) Gecko/20100101 Firefox/69.0	Other	Firefox 69.0	Windows 10.0.0
+Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36	Other	Chrome 49.0	Windows XP.0.0
+Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36	Other	Chrome 49.0	Windows Vista.0.0
+Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36	Other	Chrome 49.0	Windows Vista.0.0
+Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Other	Chrome 76.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Other	Chrome 77.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Other	Chrome 77.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) coc_coc_browser/80.0.180 Chrome/74.0.3729.180 Safari/537.36	Other	Coc Coc 80.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) coc_coc_browser/82.0.144 Chrome/76.0.3809.144 Safari/537.36	Other	Coc Coc 82.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko	Other	IE 11.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36	Other	Chrome 67.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36	Other	Chrome 70.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36	Other	Chrome 72.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36	Other	Chrome 74.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36	Other	Chrome 75.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36	Other	Chrome 76.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Other	Chrome 76.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Other	Chrome 77.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0 Waterfox/56.2.14	Other	Waterfox 56.2	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0	Other	Firefox 60.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0	Other	Firefox 63.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:68.0) Gecko/20100101 Firefox/68.0	Other	Firefox 68.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0	Other	Firefox 69.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.1; rv:69.0) Gecko/20100101 Firefox/69.0	Other	Firefox 69.0	Windows 7.0.0
+Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Other	Chrome 77.0	Windows 8.0.0
+Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0	Other	Firefox 69.0	Windows 8.0.0
+Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Other	Chrome 77.0	Windows 8.1.0
+Mozilla/5.0 (Windows NT 6.3; ARM; Trident/7.0; Touch; rv:11.0) like Gecko	Other	IE 11.0	Windows RT 8.1.0
+Mozilla/5.0 (Windows NT 6.3; Trident/7.0; Touch; rv:11.0) like Gecko	Other	IE 11.0	Windows 8.1.0
+Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko	Other	IE 11.0	Windows 8.1.0
+Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36	Other	Chrome 63.0	Windows 8.1.0
+Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36	Other	Chrome 64.0	Windows 8.1.0
+Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36	Other	Chrome 76.0	Windows 8.1.0
+Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36	Other	Chrome 76.0	Windows 8.1.0
+Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36	Other	Chrome 77.0	Windows 8.1.0
+Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0	Other	Firefox 69.0	Windows 8.1.0
+Mozilla/5.0 (Windows NT 6.3; rv:69.0) Gecko/20100101 Firefox/69.0	Other	Firefox 69.0	Windows 8.1.0
+Mozilla/5.0 (Windows; U; Windows NT 10.0; en-US; Valve Steam GameOverlay/1568860339; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36	Other	Chrome 72.0	Windows 10.0.0
+Mozilla/5.0 (X11; CrOS aarch64 12371.75.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.105 Safari/537.36	Other	Chrome 77.0	Chrome OS 12371.75.0
+Mozilla/5.0 (X11; CrOS armv7l 12239.92.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.136 Safari/537.36	Other	Chrome 76.0	Chrome OS 12239.92.0
+Mozilla/5.0 (X11; CrOS x86_64 10895.78.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.120 Safari/537.36	Other	Chrome 69.0	Chrome OS 10895.78.0
+Mozilla/5.0 (X11; CrOS x86_64 11021.81.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36	Other	Chrome 70.0	Chrome OS 11021.81.0
+Mozilla/5.0 (X11; CrOS x86_64 11895.118.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.159 Safari/537.36	Other	Chrome 74.0	Chrome OS 11895.118.0
+Mozilla/5.0 (X11; CrOS x86_64 12239.92.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.136 Safari/537.36	Other	Chrome 76.0	Chrome OS 12239.92.0
+Mozilla/5.0 (X11; CrOS x86_64 12239.92.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.136 Safari/537.36	Other	Chrome 76.0	Chrome OS 12239.92.1
+Mozilla/5.0 (X11; CrOS x86_64 12239.92.4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.136 Safari/537.36	Other	Chrome 76.0	Chrome OS 12239.92.4
+Mozilla/5.0 (X11; CrOS x86_64 12371.46.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.63 Safari/537.36	Other	Chrome 77.0	Chrome OS 12371.46.0
+Mozilla/5.0 (X11; CrOS x86_64 12371.65.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.93 Safari/537.36	Other	Chrome 77.0	Chrome OS 12371.65.0
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36	Other	Chrome 75.0	Linux 0.0.0
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36	Other	Chrome 77.0	Linux 0.0.0
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/10.1 Chrome/71.0.3578.99 Safari/537.36	Other	Samsung Internet 10.1	Linux 0.0.0
+Mozilla/5.0 (X11; U; U; Linux x86_64; in-id) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36	Other	Chrome 66.0	Linux 0.0.0
+Mozilla/5.0 (X11; U; U; Linux x86_64; pt-pt) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36	Other	Chrome 66.0	Linux 0.0.0
+Mozilla/5.0 (X11; U; U; Linux x86_64; th-th) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36	Other	Chrome 66.0	Linux 0.0.0
+Mozilla/5.0 (X11; U; U; Linux x86_64; vi-vn) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36	Other	Chrome 66.0	Linux 0.0.0
+Mozilla/5.0 (X11; U; U; Linux x86_64; zh-cn) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36	Other	Chrome 66.0	Linux 0.0.0
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0	Other	Firefox 65.0	Ubuntu 0.0.0
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0	Other	Firefox 66.0	Ubuntu 0.0.0
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0	Other	Firefox 67.0	Ubuntu 0.0.0
+Mozilla/5.0 (iPad; CPU OS 10_3_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) GSA/22.0.141836113 Mobile/14G60 Safari/600.1.4	iPad	Google 22.0	iOS 10.3.3
+Mozilla/5.0 (iPad; CPU OS 10_3_3 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) CriOS/71.0.3578.89 Mobile/14G60 Safari/602.1	iPad	Chrome Mobile iOS 71.0	iOS 10.3.3
+Mozilla/5.0 (iPad; CPU OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) FxiOS/14.0b12646 Mobile/14G60 Safari/603.3.8	iPad	Firefox iOS 14.0	iOS 10.3.3
+Mozilla/5.0 (iPad; CPU OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Mobile/14G60	iPad	Mobile Safari UI/WKWebView 0.0	iOS 10.3.3
+Mozilla/5.0 (iPad; CPU OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Mobile/14G60 [FBAN/FBIOS;FBAV/240.0.0.55.117;FBBV/174195427;FBDV/iPad5,3;FBMD/iPad;FBSN/iOS;FBSV/10.3.3;FBSS/2;FBID/tablet;FBLC/zh_TW;FBOP/5;FBRV/175353135;FBCR/]	iPad	Facebook 240.0	iOS 10.3.3
+Mozilla/5.0 (iPad; CPU OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.0 Mobile/14G60 Safari/602.1	iPad	Mobile Safari 10.0	iOS 10.3.3
+Mozilla/5.0 (iPad; CPU OS 10_3_4 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.0 Mobile/14G61 Safari/602.1	iPad	Mobile Safari 10.0	iOS 10.3.4
+Mozilla/5.0 (iPad; CPU OS 11_1 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) CriOS/76.0.3809.123 Mobile/15B101 Safari/604.1	iPad	Chrome Mobile iOS 76.0	iOS 11.1.0
+Mozilla/5.0 (iPad; CPU OS 11_1_2 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) CriOS/76.0.3809.123 Mobile/15B202 Safari/604.1	iPad	Chrome Mobile iOS 76.0	iOS 11.1.2
+Mozilla/5.0 (iPad; CPU OS 11_2_1 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Version/11.0 Mobile/15C153 Safari/604.1	iPad	Mobile Safari 11.0	iOS 11.2.1
+Mozilla/5.0 (iPad; CPU OS 11_2_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Version/11.0 Mobile/15C202 Safari/604.1	iPad	Mobile Safari 11.0	iOS 11.2.2
+Mozilla/5.0 (iPad; CPU OS 11_2_6 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0 Mobile/15D100 Safari/604.1	iPad	Mobile Safari 11.0	iOS 11.2.6
+Mozilla/5.0 (iPad; CPU OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.0 Mobile/15E148 Safari/604.1	iPad	Mobile Safari 11.0	iOS 11.3.0
+Mozilla/5.0 (iPad; CPU OS 11_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.0 Mobile/15E148 Safari/604.1	iPad	Mobile Safari 11.0	iOS 11.4.0
+Mozilla/5.0 (iPad; CPU OS 11_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15G77	iPad	Mobile Safari UI/WKWebView 0.0	iOS 11.4.1
+Mozilla/5.0 (iPad; CPU OS 11_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.0 Mobile/15E148 Safari/604.1	iPad	Mobile Safari 11.0	iOS 11.4.1
+Mozilla/5.0 (iPad; CPU OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/83.0.268992909 Mobile/15E148 Safari/605.1	iPad	Google 83.0	iOS 12.0.0
+Mozilla/5.0 (iPad; CPU OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1	iPad	Mobile Safari 12.0	iOS 12.0.0
+Mozilla/5.0 (iPad; CPU OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/75.0.3770.103 Mobile/15E148 Safari/605.1	iPad	Chrome Mobile iOS 75.0	iOS 12.1.0
+Mozilla/5.0 (iPad; CPU OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/76.0.3809.123 Mobile/15E148 Safari/605.1	iPad	Chrome Mobile iOS 76.0	iOS 12.1.0
+Mozilla/5.0 (iPad; CPU OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/16B92	iPad	Mobile Safari UI/WKWebView 0.0	iOS 12.1.0
+Mozilla/5.0 (iPad; CPU OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1	iPad	Mobile Safari 12.0	iOS 12.1.0
+Mozilla/5.0 (iPad; CPU OS 12_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1	iPad	Mobile Safari 12.0	iOS 12.1.1
+Mozilla/5.0 (iPad; CPU OS 12_1_4 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) GSA/48.0.193557427 Mobile/16D57 Safari/604.1	iPad	Google 48.0	iOS 12.1.4
+Mozilla/5.0 (iPad; CPU OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/16D57	iPad	Mobile Safari UI/WKWebView 0.0	iOS 12.1.4
+Mozilla/5.0 (iPad; CPU OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1	iPad	Mobile Safari 12.0	iOS 12.1.4
+Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/76.0.3809.123 Mobile/15E148 Safari/605.1	iPad	Chrome Mobile iOS 76.0	iOS 12.2.0
+Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148	iPad	Mobile Safari UI/WKWebView 0.0	iOS 12.2.0
+Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Mobile/15E148 Safari/604.1	iPad	Mobile Safari 12.1	iOS 12.2.0
+Mozilla/5.0 (iPad; CPU OS 12_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/77.0.3865.93 Mobile/15E148 Safari/605.1	iPad	Chrome Mobile iOS 77.0	iOS 12.3.0
+Mozilla/5.0 (iPad; CPU OS 12_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/83.0.268992909 Mobile/15E148 Safari/605.1	iPad	Google 83.0	iOS 12.3.0
+Mozilla/5.0 (iPad; CPU OS 12_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1	iPad	Mobile Safari 12.1	iOS 12.3.0
+Mozilla/5.0 (iPad; CPU OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148	iPad	Mobile Safari UI/WKWebView 0.0	iOS 12.3.1
+Mozilla/5.0 (iPad; CPU OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1	iPad	Mobile Safari 12.1	iOS 12.3.1
+Mozilla/5.0 (iPad; CPU OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/76.0.3809.123 Mobile/15E148 Safari/605.1	iPad	Chrome Mobile iOS 76.0	iOS 12.4.0
+Mozilla/5.0 (iPad; CPU OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/76.0.3809.81 Mobile/15E148 Safari/605.1	iPad	Chrome Mobile iOS 76.0	iOS 12.4.0
+Mozilla/5.0 (iPad; CPU OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/77.0.3865.103 Mobile/15E148 Safari/605.1	iPad	Chrome Mobile iOS 77.0	iOS 12.4.0
+Mozilla/5.0 (iPad; CPU OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/77.0.3865.69 Mobile/15E148 Safari/605.1	iPad	Chrome Mobile iOS 77.0	iOS 12.4.0
+Mozilla/5.0 (iPad; CPU OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/77.0.3865.93 Mobile/15E148 Safari/605.1	iPad	Chrome Mobile iOS 77.0	iOS 12.4.0
+Mozilla/5.0 (iPad; CPU OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/74.0.248026584 Mobile/15E148 Safari/605.1	iPad	Google 74.0	iOS 12.4.0
+Mozilla/5.0 (iPad; CPU OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/83.0.268992909 Mobile/15E148 Safari/605.1	iPad	Google 83.0	iOS 12.4.0
+Mozilla/5.0 (iPad; CPU OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148	iPad	Mobile Safari UI/WKWebView 0.0	iOS 12.4.0
+Mozilla/5.0 (iPad; CPU OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Mobile/15E148 Safari/604.1	iPad	Mobile Safari 12.1	iOS 12.4.0
+Mozilla/5.0 (iPad; CPU OS 12_4_1 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) CriOS/67.0.3396.87 Mobile/16G102 Safari/604.1	iPad	Chrome Mobile iOS 67.0	iOS 12.4.1
+Mozilla/5.0 (iPad; CPU OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/19.0b16042 Mobile/15E148 Safari/605.1.15	iPad	Firefox iOS 19.0	iOS 12.4.1
+Mozilla/5.0 (iPad; CPU OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148	iPad	Mobile Safari UI/WKWebView 0.0	iOS 12.4.1
+Mozilla/5.0 (iPad; CPU OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPad4,7;FBMD/iPad;FBSN/iOS;FBSV/12.4.1;FBSS/2;FBID/tablet;FBLC/vi_VN;FBOP/5;FBCR/]	iPad	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPad; CPU OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPad5,1;FBMD/iPad;FBSN/iOS;FBSV/12.4.1;FBSS/2;FBCR/;FBID/tablet;FBLC/en_US;FBOP/5]	iPad	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPad; CPU OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPad6,11;FBMD/iPad;FBSN/iOS;FBSV/12.4.1;FBSS/2;FBID/tablet;FBLC/en_US;FBOP/5;FBCR/]	iPad	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPad; CPU OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPad7,5;FBMD/iPad;FBSN/iOS;FBSV/12.4.1;FBSS/2;FBID/tablet;FBLC/en_US;FBOP/5;FBCR/]	iPad	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPad; CPU OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Mobile/15E148 Safari/604.1	iPad	Mobile Safari 12.1	iOS 12.4.1
+Mozilla/5.0 (iPad; CPU OS 6_1_3 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10B329 Safari/8536.25	iPad	Mobile Safari 6.0	iOS 6.1.3
+Mozilla/5.0 (iPad; CPU OS 8_0 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12A365 Safari/600.1.4	iPad	Mobile Safari 8.0	iOS 8.0.0
+Mozilla/5.0 (iPad; CPU OS 8_2 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12D508 Safari/600.1.4	iPad	Mobile Safari 8.0	iOS 8.2.0
+Mozilla/5.0 (iPad; CPU OS 8_4 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) GSA/23.1.148956103 Mobile/12H143 Safari/600.1.4	iPad	Google 23.1	iOS 8.4.0
+Mozilla/5.0 (iPad; CPU OS 9_3_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13F69 Safari/601.1	iPad	Mobile Safari 9.0	iOS 9.3.2
+Mozilla/5.0 (iPad; CPU OS 9_3_5 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13G36 Safari/601.1	iPad	Mobile Safari 9.0	iOS 9.3.5
+Mozilla/5.0 (iPhone; CPU iPhone OS 10_2 like Mac OS X) AppleWebKit/602.3.12 (KHTML, like Gecko) Version/10.0 Mobile/14C92 Safari/602.1	iPhone	Mobile Safari 10.0	iOS 10.2.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Mobile/14G60 [FBAN/FBIOS;FBDV/iPhone7,1;FBMD/iPhone;FBSN/iOS;FBSV/10.3.3;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/Verizon]	iPhone	Facebook 0.0	iOS 10.3.3
+Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_4 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) GSA/68.0.234683655 Mobile/14G61 Safari/602.1	iPhone	Google 68.0	iOS 10.3.4
+Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_4 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.0 Mobile/14G61 Safari/602.1	iPhone	Mobile Safari 10.0	iOS 10.3.4
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_0_3 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A432 Safari/604.1	iPhone	Mobile Safari 11.0	iOS 11.0.3
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_1_1 like Mac OS X) AppleWebKit/604.3.5 (KHTML, like Gecko) Version/11.0 Mobile/15B150 Safari/604.1	iPhone	Mobile Safari 11.0	iOS 11.1.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_1_2 like Mac OS X) AppleWebKit/604.3.5 (KHTML, like Gecko) Version/11.0 Mobile/15B202 Safari/604.1	iPhone	Mobile Safari 11.0	iOS 11.1.2
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_1 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Version/11.0 Mobile/15C153 Safari/604.1	iPhone	Mobile Safari 11.0	iOS 11.2.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_6 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Mobile/15D100 [FBAN/FBIOS;FBAV/207.0.0.48.100;FBBV/141048683;FBDV/iPhone9,3;FBMD/iPhone;FBSN/iOS;FBSV/11.2.6;FBSS/2;FBCR/SFR;FBID/phone;FBLC/fr_FR;FBOP/5;FBRV/142061404]	iPhone	Facebook 207.0	iOS 11.2.6
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) CriOS/76.0.3809.123 Mobile/15E148 Safari/604.1	iPhone	Chrome Mobile iOS 76.0	iOS 11.3.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E302 [FBAN/FBIOS;FBDV/iPhone7,2;FBMD/iPhone;FBSN/iOS;FBSV/11.3.1;FBSS/2;FBID/phone;FBLC/fr_FR;FBOP/5;FBCR/VINI]	iPhone	Facebook 0.0	iOS 11.3.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.0 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 11.0	iOS 11.3.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_4 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) GSA/83.0.268992909 Mobile/15E148 Safari/604.1	iPhone	Google 83.0	iOS 11.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.0 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 11.0	iOS 11.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_4_1 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) GSA/74.1.250942683 Mobile/15G77 Safari/604.1	iPhone	Google 74.1	iOS 11.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.0 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 11.0	iOS 11.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 12.0	iOS 12.0.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 12.0	iOS 12.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 12.0	iOS 12.1.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/74.1.250942683 Mobile/16C101 Safari/604.1	iPhone	Google 74.1	iOS 12.1.2
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/16C101 [FBAN/FBIOS;FBDV/iPhone9,3;FBMD/iPhone;FBSN/iOS;FBSV/12.1.2;FBSS/2;FBCR/Free;FBID/phone;FBLC/fr_FR;FBOP/5]	iPhone	Facebook 0.0	iOS 12.1.2
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 12.0	iOS 12.1.2
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 12.0	iOS 12.1.3
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/74.1.250942683 Mobile/16D57 Safari/604.1	iPhone	Google 74.1	iOS 12.1.4
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 12.0	iOS 12.1.4
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/72.0.3626.101 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 72.0	iOS 12.2.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/76.0.3809.123 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 76.0	iOS 12.2.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/77.0.3865.69 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 77.0	iOS 12.2.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,2;FBMD/iPhone;FBSN/iOS;FBSV/12.2;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/T-Mobile]	iPhone	Facebook 0.0	iOS 12.2.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,5;FBMD/iPhone;FBSN/iOS;FBSV/12.2;FBSS/3;FBCR/AT&T;FBID/phone;FBLC/en_US;FBOP/5]	iPhone	Facebook 0.0	iOS 12.2.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 12.1	iOS 12.2.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/77.0.3865.69 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 77.0	iOS 12.3.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/83.0.268992909 Mobile/15E148 Safari/605.1	iPhone	Google 83.0	iOS 12.3.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 12.1	iOS 12.3.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/79.0.259819395 Mobile/16F203 Safari/604.1	iPhone	Google 79.0	iOS 12.3.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148	iPhone	Mobile Safari UI/WKWebView 0.0	iOS 12.3.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 DuckDuckGo/7	iPhone	DuckDuckGo Mobile 7.0	iOS 12.3.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,6;FBMD/iPhone;FBSN/iOS;FBSV/12.3.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/AT&T]	iPhone	Facebook 0.0	iOS 12.3.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,6;FBMD/iPhone;FBSN/iOS;FBSV/12.3.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/AT&T]	iPhone	Facebook 0.0	iOS 12.3.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone9,3;FBMD/iPhone;FBSN/iOS;FBSV/12.3.1;FBSS/2;FBID/phone;FBLC/es_LA;FBOP/5;FBCR/AT&T]	iPhone	Facebook 0.0	iOS 12.3.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 12.1	iOS 12.3.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 12.1	iOS 12.3.2
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/69.0.3497.105 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 69.0	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/73.0.3683.68 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 73.0	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/75.0.3770.103 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 75.0	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/76.0.3809.123 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 76.0	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/77.0.3865.103 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 77.0	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/77.0.3865.69 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 77.0	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/81.0.264749124 Mobile/15E148 Safari/605.1	iPhone	Google 81.0	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/82.1.267240167 Mobile/15E148 Safari/605.1	iPhone	Google 82.1	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/83.0.268992909 Mobile/15E148 Safari/605.1	iPhone	Google 83.0	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,1;FBMD/iPhone;FBSN/iOS;FBSV/12.4;FBSS/2;FBID/phone;FBLC/en_US;FBOP/5;FBCR/Verizon]	iPhone	Facebook 0.0	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,5;FBMD/iPhone;FBSN/iOS;FBSV/12.4;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/T-Mobile]	iPhone	Facebook 0.0	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,6;FBMD/iPhone;FBSN/iOS;FBSV/12.4;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/Sprint]	iPhone	Facebook 0.0	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone9,4;FBMD/iPhone;FBSN/iOS;FBSV/12.4;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/T-Mobile]	iPhone	Facebook 0.0	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 12.1	iOS 12.4.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/74.1.250942683 Mobile/16G102 Safari/604.1	iPhone	Google 74.1	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148	iPhone	Mobile Safari UI/WKWebView 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Instagram 89.0.0.14.100 (iPhone11,6; iOS 12_4_1; en_US; en-US; scale=3.00; gamut=normal; 1242x2688; 149781277)	iPhone	Instagram 89.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBAV/240.0.0.55.117;FBBV/174195427;FBDV/iPhone7,2;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/2;FBID/phone;FBLC/es_LA;FBOP/5;FBRV/175040728;FBCR/AT&T]	iPhone	Facebook 240.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,2;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/Sprint]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,3;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/T-Mobile]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,3;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/Verizon]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,4;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/2;FBID/phone;FBLC/es_LA;FBOP/5;FBCR/T-Mobile]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,4;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/2;FBID/phone;FBLC/fr_FR;FBOP/5;FBCR/AT&T]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,5;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBCR/T-Mobile;FBID/phone;FBLC/es_LA;FBOP/5]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,5;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/T-Mobile]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,5;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBID/phone;FBLC/es_LA;FBOP/5;FBCR/T-Mobile]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,6;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/AT&T]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,2;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/Verizon]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,6;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/AT&T]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,6;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/Verizon]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,6;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBID/phone;FBLC/fr_FR;FBOP/5;FBCR/SFR]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,8;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/2;FBID/phone;FBLC/en_US;FBOP/5;FBCR/AT&T]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,8;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/2;FBID/phone;FBLC/en_US;FBOP/5;FBCR/Verizon]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,8;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/2;FBID/phone;FBLC/fr_FR;FBOP/5;FBCR/Carrier]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone7,2;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/2;FBID/phone;FBLC/en_US;FBOP/5;FBCR/T-Mobile]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone8,1;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/2;FBID/phone;FBLC/en_US;FBOP/5;FBCR/MetroPCS]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone9,2;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/cricket]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone9,4;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/AT&T]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone9,4;FBMD/iPhone;FBSN/iOS;FBSV/12.4.1;FBSS/3;FBID/phone;FBLC/fr_FR;FBOP/5;FBCR/T-Mobile]	iPhone	Facebook 0.0	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 12.1	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.4.1 Mobile/15E148 Safari/605.1.15	iPhone	Mobile Safari 12.4	iOS 12.4.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148	iPhone	Mobile Safari UI/WKWebView 0.0	iOS 12.4.2
+Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 12.1	iOS 12.4.2
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/77.0.3865.69 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 77.0	iOS 13.0.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,5;FBMD/iPhone;FBSN/iOS;FBSV/13.0;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/AT&T]	iPhone	Facebook 0.0	iOS 13.0.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,6;FBMD/iPhone;FBSN/iOS;FBSV/13.0;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/T-Mobile]	iPhone	Facebook 0.0	iOS 13.0.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,2;FBMD/iPhone;FBSN/iOS;FBSV/13.0;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/Verizon]	iPhone	Facebook 0.0	iOS 13.0.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,8;FBMD/iPhone;FBSN/iOS;FBSV/13.0;FBSS/2;FBID/phone;FBLC/en_US;FBOP/5;FBCR/T-Mobile]	iPhone	Facebook 0.0	iOS 13.0.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,8;FBMD/iPhone;FBSN/iOS;FBSV/13.0;FBSS/2;FBID/phone;FBLC/en_US;FBOP/5;FBCR/Verizon]	iPhone	Facebook 0.0	iOS 13.0.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone9,4;FBMD/iPhone;FBSN/iOS;FBSV/13.0;FBSS/3;FBID/phone;FBLC/fr_FR;FBOP/5;FBCR/Orange France]	iPhone	Facebook 0.0	iOS 13.0.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 13.0	iOS 13.0.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/76.0.3809.123 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 76.0	iOS 13.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/77.0.3865.69 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 77.0	iOS 13.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/77.0.3865.93 Mobile/15E148 Safari/605.1	iPhone	Chrome Mobile iOS 77.0	iOS 13.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/8.1.3 Mobile/15E148 Safari/605.1.15	iPhone	Firefox iOS 8.1	iOS 13.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/83.0.268992909 Mobile/15E148 Safari/605.1	iPhone	Google 83.0	iOS 13.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148	iPhone	Mobile Safari UI/WKWebView 0.0	iOS 13.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 DuckDuckGo/7	iPhone	DuckDuckGo Mobile 7.0	iOS 13.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,1;FBMD/iPhone;FBSN/iOS;FBSV/13.1;FBSS/2;FBID/phone;FBLC/en_US;FBOP/5;FBCR/AT&T]	iPhone	Facebook 0.0	iOS 13.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,6;FBMD/iPhone;FBSN/iOS;FBSV/13.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/AT&T]	iPhone	Facebook 0.0	iOS 13.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,6;FBMD/iPhone;FBSN/iOS;FBSV/13.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/T-Mobile]	iPhone	Facebook 0.0	iOS 13.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,8;FBMD/iPhone;FBSN/iOS;FBSV/13.1;FBSS/2;FBID/phone;FBLC/en_US;FBOP/5;FBCR/AT&T]	iPhone	Facebook 0.0	iOS 13.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone9,4;FBMD/iPhone;FBSN/iOS;FBSV/13.1;FBSS/3;FBID/phone;FBLC/es_LA;FBOP/5;FBCR/Telcel]	iPhone	Facebook 0.0	iOS 13.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.1 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 13.0	iOS 13.1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148	iPhone	Mobile Safari UI/WKWebView 0.0	iOS 13.1.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,1;FBMD/iPhone;FBSN/iOS;FBSV/13.1.1;FBSS/2;FBID/phone;FBLC/en_US;FBOP/5;FBCR/Union]	iPhone	Facebook 0.0	iOS 13.1.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone10,3;FBMD/iPhone;FBSN/iOS;FBSV/13.1.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/Verizon]	iPhone	Facebook 0.0	iOS 13.1.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,6;FBMD/iPhone;FBSN/iOS;FBSV/13.1.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/T-Mobile]	iPhone	Facebook 0.0	iOS 13.1.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,6;FBMD/iPhone;FBSN/iOS;FBSV/13.1.1;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/Verizon]	iPhone	Facebook 0.0	iOS 13.1.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.1 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 13.0	iOS 13.1.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148	iPhone	Mobile Safari UI/WKWebView 0.0	iOS 13.1.2
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone11,2;FBMD/iPhone;FBSN/iOS;FBSV/13.1.2;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5;FBCR/AT&T]	iPhone	Facebook 0.0	iOS 13.1.2
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_1_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.1 Mobile/15E148 Safari/604.1	iPhone	Mobile Safari 13.0	iOS 13.1.2
diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh
index 8e361a191d6..6e386360d60 100755
--- a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh
+++ b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh
@@ -68,13 +68,13 @@ $CLICKHOUSE_CLIENT -n --query="
 insert into user_agents select ua from input('ua String') FORMAT LineAsString" < $CURDIR/data_ua_parser/useragents.txt
 
 $CLICKHOUSE_CLIENT -n --query="
-select device,
+select ua, device,
 concat(tupleElement(browser, 1), ' ', tupleElement(browser, 2), '.', tupleElement(browser, 3)) as browser ,
 concat(tupleElement(os, 1), ' ', tupleElement(os, 2), '.', tupleElement(os, 3), '.', tupleElement(os, 4)) as os
 from (
-     select dictGet('regexp_os', ('os_replacement', 'os_v1_replacement', 'os_v2_replacement', 'os_v3_replacement'), ua) os,
+     select ua, dictGet('regexp_os', ('os_replacement', 'os_v1_replacement', 'os_v2_replacement', 'os_v3_replacement'), ua) os,
      dictGet('regexp_browser', ('family_replacement', 'v1_replacement', 'v2_replacement'), ua) as browser,
-     dictGet('regexp_device', 'device_replacement', ua) device from user_agents);
+     dictGet('regexp_device', 'device_replacement', ua) device from user_agents) order by ua;
 "
 
 $CLICKHOUSE_CLIENT -n --query="

From 24b6ff47ac851026b6032b7e3c4626ccde6b1596 Mon Sep 17 00:00:00 2001
From: Suzy Wang <suzy.wang@ibm.com>
Date: Wed, 10 May 2023 13:06:30 -0700
Subject: [PATCH 409/689] fix format and some more fix for fixedstring

---
 src/Functions/reinterpretAs.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Functions/reinterpretAs.cpp b/src/Functions/reinterpretAs.cpp
index 93949bf50c3..36c944d16fd 100644
--- a/src/Functions/reinterpretAs.cpp
+++ b/src/Functions/reinterpretAs.cpp
@@ -213,6 +213,7 @@ public:
 
                     size_t offset = 0;
                     size_t copy_size = std::min(step, sizeof(ToFieldType));
+                    size_t index = data_from.size() - copy_size;
 
                     if (sizeof(ToFieldType) <= step)
                         vec_res.resize(size);
@@ -226,9 +227,10 @@ public:
                         else
                         {
                             size_t offset_to = sizeof(ToFieldType) > copy_size ? sizeof(ToFieldType) - copy_size : 0;
-                            memcpy(reinterpret_cast<char*>(&vec_res[i]) + offset_to, &data_from[offset], copy_size);
+                            memcpy(reinterpret_cast<char*>(&vec_res[i]) + offset_to, &data_from[index - offset], copy_size);
                         }
                         offset += step;
+                    }
 
                     result = std::move(col_res);
 

From db52652337e4f6084b9817cd076f5d28118d3497 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Wed, 10 May 2023 21:00:56 +0000
Subject: [PATCH 410/689] fix unit test

---
 .../tests/gtest_thread_pool_schedule_exception.cpp  | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/Common/tests/gtest_thread_pool_schedule_exception.cpp b/src/Common/tests/gtest_thread_pool_schedule_exception.cpp
index fd77a95546b..5dbad00848d 100644
--- a/src/Common/tests/gtest_thread_pool_schedule_exception.cpp
+++ b/src/Common/tests/gtest_thread_pool_schedule_exception.cpp
@@ -56,8 +56,17 @@ static bool check2()
 {
     ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 2);
 
-    pool.scheduleOrThrowOnError([&]{ throw std::runtime_error("Hello, world!"); });
-    pool.scheduleOrThrowOnError([]{});
+    try
+    {
+        pool.scheduleOrThrowOnError([&]{ throw std::runtime_error("Hello, world!"); });
+        pool.scheduleOrThrowOnError([]{});
+    }
+    catch (const std::runtime_error &)
+    {
+        /// Sometimes exception may be thrown from schedule.
+        /// Just retry test in that case.
+        return true;
+    }
 
     try
     {

From ff287c7bba5a94d393e823bdcd8a6dfbba9275ee Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Wed, 10 May 2023 22:46:09 +0000
Subject: [PATCH 411/689] add lock

---
 src/Storages/StorageMergeTree.cpp | 13 +++++++++++--
 src/Storages/StorageMergeTree.h   |  3 +++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 17d1085d6ae..b4dc2830bd6 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -619,8 +619,13 @@ bool comparator(const PartVersionWithName & f, const PartVersionWithName & s)
 std::optional<MergeTreeMutationStatus> StorageMergeTree::getIncompleteMutationsStatus(
     Int64 mutation_version, std::set<String> * mutation_ids, bool from_another_mutation) const
 {
-    std::lock_guard lock(currently_processing_in_background_mutex);
+    std::unique_lock lock(currently_processing_in_background_mutex);
+    return getIncompleteMutationsStatusUnlocked(mutation_version, lock, mutation_ids, from_another_mutation);
+}
 
+std::optional<MergeTreeMutationStatus> StorageMergeTree::getIncompleteMutationsStatusUnlocked(
+    Int64 mutation_version, std::unique_lock<std::mutex> & /*lock*/, std::set<String> * mutation_ids, bool from_another_mutation) const
+{
     auto current_mutation_it = current_mutations_by_version.find(mutation_version);
     /// Killed
     if (current_mutation_it == current_mutations_by_version.end())
@@ -1343,10 +1348,12 @@ bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assign
 
 size_t StorageMergeTree::getNumberOfUnfinishedMutations() const
 {
+    std::unique_lock lock(currently_processing_in_background_mutex);
+
     size_t count = 0;
     for (const auto & [version, _] : current_mutations_by_version | std::views::reverse)
     {
-        auto status = getIncompleteMutationsStatus(version);
+        auto status = getIncompleteMutationsStatusUnlocked(version, lock);
         if (!status)
             continue;
 
@@ -2154,6 +2161,8 @@ void StorageMergeTree::backupData(BackupEntriesCollector & backup_entries_collec
 
 BackupEntries StorageMergeTree::backupMutations(UInt64 version, const String & data_path_in_backup) const
 {
+    std::lock_guard lock(currently_processing_in_background_mutex);
+
     fs::path mutations_path_in_backup = fs::path{data_path_in_backup} / "mutations";
     BackupEntries backup_entries;
     for (auto it = current_mutations_by_version.lower_bound(version); it != current_mutations_by_version.end(); ++it)
diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h
index 884e2f02825..9e75d6a9373 100644
--- a/src/Storages/StorageMergeTree.h
+++ b/src/Storages/StorageMergeTree.h
@@ -257,6 +257,9 @@ private:
     std::optional<MergeTreeMutationStatus> getIncompleteMutationsStatus(Int64 mutation_version, std::set<String> * mutation_ids = nullptr,
                                                                         bool from_another_mutation = false) const;
 
+    std::optional<MergeTreeMutationStatus> getIncompleteMutationsStatusUnlocked(Int64 mutation_version, std::unique_lock<std::mutex> & lock,
+                                                                        std::set<String> * mutation_ids = nullptr, bool from_another_mutation = false) const;
+
     void fillNewPartName(MutableDataPartPtr & part, DataPartsLock & lock);
 
     void startBackgroundMovesIfNeeded() override;

From aed031c3a3f20948978dda9c6a3933d254140969 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 11 May 2023 04:15:54 +0200
Subject: [PATCH 412/689] Fix typo

---
 src/Core/MySQL/Authentication.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Core/MySQL/Authentication.cpp b/src/Core/MySQL/Authentication.cpp
index e6de8c38905..2c10bd88722 100644
--- a/src/Core/MySQL/Authentication.cpp
+++ b/src/Core/MySQL/Authentication.cpp
@@ -31,7 +31,7 @@ namespace Authentication
 static const size_t SCRAMBLE_LENGTH = 20;
 
 /** Generate a random string using ASCII characters but avoid separator character,
-  * produce pseudo random numbers between with about 7 bit worth of entropty between 1-127.
+  * produce pseudo random numbers between with about 7 bit worth of entropy between 1-127.
   * https://github.com/mysql/mysql-server/blob/8.0/mysys/crypt_genhash_impl.cc#L427
   */
 static String generateScramble()

From f82dc03b146f74ceeb16368b5813af5f7ad54486 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 11 May 2023 04:40:31 +0200
Subject: [PATCH 413/689] Fix build of libfiu on clang-16

---
 contrib/libfiu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/libfiu b/contrib/libfiu
index 7ae4328050c..b85edbde4cf 160000
--- a/contrib/libfiu
+++ b/contrib/libfiu
@@ -1 +1 @@
-Subproject commit 7ae4328050ccad8867a05a37af941886f717b6fc
+Subproject commit b85edbde4cf974b1b40d27828a56f0505f4e2ee5

From a07359fbe84ad9bd50802cfaf5fb4631a579af33 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Sat, 6 May 2023 10:37:38 +0800
Subject: [PATCH 414/689] enable used flags's reinit only when the hash talbe
 rehash

---
 src/Interpreters/HashJoin.cpp         | 8 +++++++-
 tests/performance/join_used_flags.xml | 6 ++++++
 2 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100644 tests/performance/join_used_flags.xml

diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp
index 68bde0d3edb..c9843dca825 100644
--- a/src/Interpreters/HashJoin.cpp
+++ b/src/Interpreters/HashJoin.cpp
@@ -79,7 +79,13 @@ namespace JoinStuff
         {
             assert(flags[nullptr].size() <= size);
             need_flags = true;
-            flags[nullptr] = std::vector<std::atomic_bool>(size);
+            // For one disjunct clause case, we don't need to reinit each time we call addJoinedBlock.
+            // and there is no value inserted in this JoinUsedFlags before addJoinedBlock finish.
+            // So we reinit only when the hash table is rehashed to a larger size.
+            if (flags.empty() || flags[nullptr].size() < size) [[unlikely]]
+            {
+                flags[nullptr] = std::vector<std::atomic_bool>(size);
+            }
         }
     }
 
diff --git a/tests/performance/join_used_flags.xml b/tests/performance/join_used_flags.xml
new file mode 100644
index 00000000000..cd2073ee106
--- /dev/null
+++ b/tests/performance/join_used_flags.xml
@@ -0,0 +1,6 @@
+<test>
+    <create_query>CREATE TABLE test_join_used_flags (i64 Int64, i32 Int32) ENGINE = Memory</create_query>
+    <fill_query>INSERT INTO test_join_used_flags SELECT number AS i64, rand32() AS i32 FROM numbers(20000000)</fill_query>
+    <query>SELECT l.i64, r.i64, l.i32, r.i32 FROM test_join_used_flags l RIGHT JOIN test_join_used_flags r USING i64 format Null</query>
+    <drop_query>DROP TABLE IF EXISTS test_join_used_flags</drop_query>
+</test>

From 784e5c2756183ec14ba4cabe914eea9993d0ca51 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 11 May 2023 07:07:19 +0200
Subject: [PATCH 415/689] Update
 02417_opentelemetry_insert_on_distributed_table reference one more time
 (sigh)

This time, checked locally

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 ...17_opentelemetry_insert_on_distributed_table.reference | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference
index fe2cd9d666a..98827438920 100644
--- a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference
+++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference
@@ -8,11 +8,11 @@
 3
 2
 ===3===
-{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
-{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
+{"operation_name":"auto DB::DistributedSink::runWritingJob(JobReplica &, const Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
+{"operation_name":"auto DB::DistributedSink::runWritingJob(JobReplica &, const Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
 1
 ===4===
-{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
-{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
+{"operation_name":"auto DB::DistributedSink::runWritingJob(JobReplica &, const Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"}
+{"operation_name":"auto DB::DistributedSink::runWritingJob(JobReplica &, const Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"}
 3
 2

From 9724688608fadc2c8ccee1d4cb08f598795f3533 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Wed, 10 May 2023 23:27:48 +0200
Subject: [PATCH 416/689] rename field, do not fetch PreActive

---
 src/Storages/MergeTree/DataPartsExchange.cpp  |  6 ++---
 src/Storages/MergeTree/IMergeTreeDataPart.h   | 10 +++++++-
 src/Storages/MergeTree/MergeTask.cpp          |  2 +-
 src/Storages/MergeTree/MergeTreeData.cpp      |  2 +-
 .../MergeTree/MergeTreeDataWriter.cpp         |  2 +-
 src/Storages/MergeTree/MutateTask.cpp         |  4 ++--
 src/Storages/StorageReplicatedMergeTree.cpp   | 24 +++++++++++--------
 7 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp
index 7b6f8036c48..1b6f83cd9bd 100644
--- a/src/Storages/MergeTree/DataPartsExchange.cpp
+++ b/src/Storages/MergeTree/DataPartsExchange.cpp
@@ -355,10 +355,10 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk(
 
 MergeTreeData::DataPartPtr Service::findPart(const String & name)
 {
-    /// It is important to include PreActive and Outdated parts here because remote replicas cannot reliably
+    /// It is important to include Outdated parts here because remote replicas cannot reliably
     /// determine the local state of the part, so queries for the parts in these states are completely normal.
     auto part = data.getPartIfExists(
-        name, {MergeTreeDataPartState::PreActive, MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated});
+        name, {MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated});
     if (part)
         return part;
 
@@ -918,7 +918,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk(
 
         new_data_part->version.setCreationTID(Tx::PrehistoricTID, nullptr);
         new_data_part->is_temp = true;
-        new_data_part->has_exclusive_blobs = false;
+        new_data_part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::PRESERVE_BLOBS;
         new_data_part->modification_time = time(nullptr);
         new_data_part->loadColumnsChecksumsIndexes(true, false);
     }
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h
index 61048d8e257..634e275bd64 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.h
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.h
@@ -218,7 +218,15 @@ public:
     /// FIXME Why do we need this flag? What's difference from Temporary and DeleteOnDestroy state? Can we get rid of this?
     bool is_temp = false;
 
-    std::optional<bool> has_exclusive_blobs = std::nullopt;
+    enum class BlobsRemovalPolicyForTemporaryParts
+    {
+        ASK_KEEPER,
+        REMOVE_BLOBS,
+        PRESERVE_BLOBS,
+    };
+    /// That field is used by replicated merge tree with zero copy replication
+    /// Usually the data has to bo unlocked in keeper unless explicitly otherwise stated
+    BlobsRemovalPolicyForTemporaryParts remove_tmp_policy = BlobsRemovalPolicyForTemporaryParts::ASK_KEEPER;
 
     /// If true it means that there are no ZooKeeper node for this part, so it should be deleted only from filesystem
     bool is_duplicate = false;
diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp
index 2905ab68ef3..1105c467ac8 100644
--- a/src/Storages/MergeTree/MergeTask.cpp
+++ b/src/Storages/MergeTree/MergeTask.cpp
@@ -209,7 +209,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare()
     global_ctx->new_data_part->uuid = global_ctx->future_part->uuid;
     global_ctx->new_data_part->partition.assign(global_ctx->future_part->getPartition());
     global_ctx->new_data_part->is_temp = global_ctx->parent_part == nullptr;
-    global_ctx->new_data_part->has_exclusive_blobs = true;
+    global_ctx->new_data_part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS;
 
     ctx->need_remove_expired_values = false;
     ctx->force_ttl = false;
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index f229a19b45c..6876ece5470 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -8233,7 +8233,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createEmptyPart(
 
     new_data_part->minmax_idx = std::move(minmax_idx);
     new_data_part->is_temp = true;
-    new_data_part->has_exclusive_blobs = true;
+    new_data_part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS;
 
     auto new_data_part_storage = new_data_part->getDataPartStoragePtr();
     new_data_part_storage->beginTransaction();
diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index 28dab4c9068..88acb7df667 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -469,7 +469,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl(
     new_data_part->partition = std::move(partition);
     new_data_part->minmax_idx = std::move(minmax_idx);
     new_data_part->is_temp = true;
-    new_data_part->has_exclusive_blobs = true;
+    new_data_part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS;
 
     SyncGuardPtr sync_guard;
     if (new_data_part->isStoredOnDisk())
diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp
index ac0f0e6d0a9..b2c2b4a96c0 100644
--- a/src/Storages/MergeTree/MutateTask.cpp
+++ b/src/Storages/MergeTree/MutateTask.cpp
@@ -1838,7 +1838,7 @@ bool MutateTask::prepare()
     if (!isWidePart(ctx->source_part) || !isFullPartStorage(ctx->source_part->getDataPartStorage())
         || (ctx->interpreter && ctx->interpreter->isAffectingAllColumns()))
     {
-        ctx->new_data_part->has_exclusive_blobs = true;
+        ctx->new_data_part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS;
 
         task = std::make_unique<MutateAllPartColumnsTask>(ctx);
     }
@@ -1867,7 +1867,7 @@ bool MutateTask::prepare()
             ctx->for_file_renames,
             ctx->mrk_extension);
 
-        ctx->new_data_part->has_exclusive_blobs = std::nullopt; // that means that unlocking in zookeper needed
+        ctx->new_data_part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::ASK_KEEPER;
 
         task = std::make_unique<MutateSomePartColumnsTask>(ctx);
     }
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 1729e0f111f..2bcc0b22f2a 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -8267,22 +8267,26 @@ StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, co
     {
         /// Part {} is in temporary state and has it_temp flag. it means that it is under construction.
         /// That path hasn't been added to active set, no commit procedure has begun.
-        /// The metadata files is about to delete now.
-        /// However remote data might be shared and has to be unlocked in the keper before removing.
-        /// Actually there is some cases when it is clear without keper:
+        /// The metadata files is about to delete now. Clichouse has to make a decision remove or preserve blobs on remote FS.
+        /// In general remote data might be shared and has to be unlocked in the keeper before removing.
+        /// However there are some cases when decision is clear without asking keeper:
         /// When the part has been fetched then remote data has to be preserved, part doesn't own it.
         /// When the part has been merged then remote data can be removed, part owns it.
-        /// In opposition, when the part has been mutated in generally it hardlinks the files.
-        /// Therefore remote data is shared, it has to be unlocked in the keper.
-        /// In order to track all that cases has_exclusive_blobs optional value is used
-        /// If has_exclusive_blobs is determined, then CH tracked the parts origin from the creation
+        /// In opposition, when the part has been mutated in generally it hardlinks the files from source part.
+        /// Therefore remote data could be shared, it has to be unlocked in the keeper.
+        /// In order to track all that cases remove_tmp_policy is used.
+        /// Clickhouse set that field as REMOVE_BLOBS or PRESERVE_BLOBS when it sure about the decision without asking keeper.
 
-        if (part.has_exclusive_blobs.has_value())
+        if (part.remove_tmp_policy == IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS
+            || part.remove_tmp_policy == IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::PRESERVE_BLOBS)
         {
+            bool can_remove_blobs = part.remove_tmp_policy == IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS;
             LOG_INFO(log, "Looks like CH knows the origin of that part. "
                           "Part {} can be deleted without unlocking shared data in zookeeper. "
-                          "Part data is {}", part.name, part.has_exclusive_blobs.value() ? "exclusive" : "shared");
-            return std::make_pair(part.has_exclusive_blobs.value(), NameSet{});
+                          "Part blobs {}.",
+                     part.name,
+                     can_remove_blobs ? "will be removed" : "have to be preserved");
+            return std::make_pair(can_remove_blobs, NameSet{});
         }
     }
 

From c2a4d89b6f4e2102577db64f2e40ed9f30970800 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Thu, 11 May 2023 09:27:43 +0000
Subject: [PATCH 417/689] Fix style

---
 src/Functions/randomStringUTF8.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Functions/randomStringUTF8.cpp b/src/Functions/randomStringUTF8.cpp
index 6aabb81c661..831b19a6216 100644
--- a/src/Functions/randomStringUTF8.cpp
+++ b/src/Functions/randomStringUTF8.cpp
@@ -79,7 +79,8 @@ public:
         size_t max_byte_size = total_codepoints * 4 + input_rows_count;
         data_to.resize(max_byte_size);
 
-        const auto generate_code_point = [](UInt32 rand) {
+        const auto generate_code_point = [](UInt32 rand)
+        {
             /// We want to generate number in [0x0, 0x70000) and shift it if need
 
             /// Generate highest byte in [0, 6]

From 9db78792d0b71e5389a10b1897afb11a1b1a8922 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Thu, 11 May 2023 10:00:32 +0000
Subject: [PATCH 418/689] Fix MsgPackRowInputFormat.cpp build

---
 src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp
index 3600fcd4086..eeca14176cc 100644
--- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp
+++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp
@@ -2,6 +2,15 @@
 
 #if USE_MSGPACK
 
+/// FIXME: there is some issue with clang-15, that incorrectly detect a
+/// "Attempt to free released memory" in msgpack::unpack(), because of delete
+/// operator for zone (from msgpack/v1/detail/cpp11_zone.hpp), hence NOLINT
+///
+/// NOTE: that I was not able to suppress it locally, only with
+/// NOLINTBEGIN/NOLINTEND
+//
+// NOLINTBEGIN(clang-analyzer-cplusplus.NewDelete)
+
 #include <cstdlib>
 #include <Common/assert_cast.h>
 #include <IO/ReadHelpers.h>
@@ -669,6 +678,8 @@ void registerMsgPackSchemaReader(FormatFactory & factory)
 
 }
 
+// NOLINTEND(clang-analyzer-cplusplus.NewDelete)
+
 #else
 
 namespace DB

From 17999a4b8f5fc3cd7046cf645e36a322bc83cf03 Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Thu, 11 May 2023 12:37:49 +0200
Subject: [PATCH 419/689] Fix style

---
 tests/queries/0_stateless/02497_storage_file_reader_selection.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/queries/0_stateless/02497_storage_file_reader_selection.sh b/tests/queries/0_stateless/02497_storage_file_reader_selection.sh
index 76f87633446..20bde68718d 100755
--- a/tests/queries/0_stateless/02497_storage_file_reader_selection.sh
+++ b/tests/queries/0_stateless/02497_storage_file_reader_selection.sh
@@ -7,7 +7,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 DATA_FILE=test_02497_$CLICKHOUSE_TEST_UNIQUE_NAME.tsv
 echo -e 'key\nfoo\nbar' > $DATA_FILE
 
-QUERY_ID=$RANDOM
 $CLICKHOUSE_LOCAL --storage_file_read_method=mmap --print-profile-events -q "SELECT * FROM file($DATA_FILE) FORMAT Null" 2>&1 | grep -F -c "CreatedReadBufferMMap"
 $CLICKHOUSE_LOCAL --storage_file_read_method=mmap --print-profile-events -q "SELECT * FROM file($DATA_FILE) FORMAT Null" 2>&1 | grep -F -c "CreatedReadBufferOrdinary"
 

From 3780a07e4618eb9cdafccfa80bdae13f4937cae3 Mon Sep 17 00:00:00 2001
From: Jordi Villar <jrdi.villar@gmail.com>
Date: Wed, 10 May 2023 18:49:55 +0200
Subject: [PATCH 420/689] Improve local cache documentation

---
 docs/en/operations/storing-data.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md
index ac6ea22ab75..f2fe673526f 100644
--- a/docs/en/operations/storing-data.md
+++ b/docs/en/operations/storing-data.md
@@ -215,7 +215,7 @@ Cache **system tables**:
 
 Cache **commands**:
 
-- `SYSTEM DROP FILESYSTEM CACHE (<path>) (ON CLUSTER)`
+- `SYSTEM DROP FILESYSTEM CACHE (<cache_name>) (ON CLUSTER)` -- `ON CLUSTER` is only supported when no `<cache_name>` is provided
 
 - `SHOW FILESYSTEM CACHES` -- show list of filesystem caches which were configured on the server. (For versions <= `22.8` the command is named `SHOW CACHES`)
 
@@ -231,10 +231,10 @@ Result:
 └───────────┘
 ```
 
-- `DESCRIBE CACHE '<cache_name>'` - show cache configuration and some general statistics for a specific cache. Cache name can be taken from `SHOW CACHES` command. (For versions <= `22.8` the command is named `DESCRIBE CACHE`)
+- `DESCRIBE FILESYSTEM CACHE '<cache_name>'` - show cache configuration and some general statistics for a specific cache. Cache name can be taken from `SHOW FILESYSTEM CACHES` command. (For versions <= `22.8` the command is named `DESCRIBE CACHE`)
 
 ```sql
-DESCRIBE CACHE 's3_cache'
+DESCRIBE FILESYSTEM CACHE 's3_cache'
 ```
 
 ``` text

From 8b4f1b40865d8580ed1f0cea34937c1735745a29 Mon Sep 17 00:00:00 2001
From: Tyler Hannan <tyler@clickhouse.com>
Date: Thu, 11 May 2023 13:47:44 +0200
Subject: [PATCH 421/689] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 7a364795a02..ee464900035 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ curl https://clickhouse.com/ | sh
 
 ## Upcoming Events
 
-* [**v23.4 Release Webinar**](https://clickhouse.com/company/events/v23-4-release-webinar?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-04) - April 26 - 23.4 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
+* [**v23.5 Release Webinar**](https://clickhouse.com/company/events/v23-5-release-webinar?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-05) - May 31 - 23.4 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
 * [**ClickHouse Meetup in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/292892466) - May 16 
 * [**ClickHouse Meetup in Barcelona**](https://www.meetup.com/clickhouse-barcelona-user-group/events/292892669) - May 25 
 * [**ClickHouse Meetup in London**](https://www.meetup.com/clickhouse-london-user-group/events/292892824) - May 25 
@@ -33,5 +33,5 @@ Also, keep an eye out for upcoming meetups in Amsterdam, Boston, NYC, Beijing, a
 
 ## Recent Recordings
 * **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
-* **Recording available**: [**v23.4 Release Webinar**]([https://www.youtube.com/watch?v=ISaGUjvBNao](https://www.youtube.com/watch?v=4rrf6bk_mOg)) UNDROP TABLE, server settings introspection, nested dynamic disks, MySQL compatibility, parseDate Time, Lightweight Deletes, Parallel Replicas, integrations updates, and so much more! Watch it now!
+* **Recording available**: [**v23.4 Release Webinar**](https://www.youtube.com/watch?v=4rrf6bk_mOg) Faster Parquet Reading, Asynchonous Connections to Reoplicas, Trailing Comma before FROM, extractKeyValuePairs, integrations updates, and so much more! Watch it now!
 * **All release webinar recordings**: [YouTube playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3jAlSy1JxyP8zluvXaN3nxU)

From 6b3160f7ceead041824ff82d355e74db086c5f06 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Thu, 11 May 2023 13:49:02 +0200
Subject: [PATCH 422/689] trivial change

---
 src/Interpreters/ExternalDictionariesLoader.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp
index 21613806788..080878c7d86 100644
--- a/src/Interpreters/ExternalDictionariesLoader.cpp
+++ b/src/Interpreters/ExternalDictionariesLoader.cpp
@@ -5,9 +5,7 @@
 #include <Dictionaries/DictionaryStructure.h>
 #include <Databases/IDatabase.h>
 #include <Storages/IStorage.h>
-#include <Common/logger_useful.h>
 
-#include "Storages/ColumnsDescription.h"
 #include "config.h"
 
 #if USE_MYSQL

From 990c22454005b6f45fe4bc09c951a0f0ab0c1873 Mon Sep 17 00:00:00 2001
From: Eridanus <maodingxuan@foxmail.com>
Date: Thu, 11 May 2023 20:04:53 +0800
Subject: [PATCH 423/689] improvement(parser): add support for size suffixes in
 quota creation statement parameters

This commit adds support for creating quotas with size suffixes in the
parameter list of a quota creation statement. The parser now recognizes
size suffixes such as "KB", "MB", and "GB" and correctly converts them
to their corresponding values in bytes. This feature improves the
usability of the quota creation statement and makes it more flexible.
The commit also includes additional tests to ensure that the new syntax
is parsed correctly.
Attention: we did not support string literal with double like
execute_time = '1.5'
---
 src/Parsers/Access/ParserCreateQuotaQuery.cpp | 165 +-----------------
 .../0_stateless/01297_create_quota.reference  |   6 +
 .../0_stateless/01297_create_quota.sql        |  46 +++--
 3 files changed, 44 insertions(+), 173 deletions(-)

diff --git a/src/Parsers/Access/ParserCreateQuotaQuery.cpp b/src/Parsers/Access/ParserCreateQuotaQuery.cpp
index 63bb07a43cd..cb83e6408bb 100644
--- a/src/Parsers/Access/ParserCreateQuotaQuery.cpp
+++ b/src/Parsers/Access/ParserCreateQuotaQuery.cpp
@@ -14,6 +14,7 @@
 #include <boost/algorithm/string/case_conv.hpp>
 #include <boost/algorithm/string/join.hpp>
 #include <boost/algorithm/string/replace.hpp>
+#include <boost/algorithm/string/trim.hpp>
 #include <Common/FieldVisitorConvertToNumber.h>
 
 
@@ -107,151 +108,13 @@ namespace
         });
     }
 
-    template <typename T, typename = std::enable_if_t<std::is_same_v<T, QuotaValue> || std::is_same_v<T, double>>>
-    bool multipleBinarySuffix(T & max_value, int power_of_two)
-    {
-        constexpr T max_t = std::numeric_limits<T>::max();
-        if constexpr (std::is_same_v<T, double>)
-        {
-            const T factor = std::exp2(power_of_two);
-            // Multiplication will cause overflow
-            if (max_t / factor < max_value)
-                return false;
-            max_value *= factor;
-        }
-        else
-        {
-            // Bit shift will cause overflow
-            if (max_t >> power_of_two < max_value)
-                return false;
-            max_value <<= power_of_two;
-        }
-        return true;
-    }
-
-    template <typename T, typename = std::enable_if_t<std::is_same_v<T, QuotaValue> || std::is_same_v<T, double>>>
-    bool multipleDeciMalSuffix(T & max_value, UInt64 base)
-    {
-        constexpr T max_t = std::numeric_limits<T>::max();
-        // Multiplication will cause overflow
-        if (max_t / base < max_value)
-            return false;
-        max_value *= base;
-        return true;
-    }
-
-    template <typename T, typename = std::enable_if_t<std::is_same_v<T, QuotaValue> || std::is_same_v<T, double>>>
-    bool multipleSizeSuffix(T & max_value, ReadBufferFromMemory & suffix)
-    {
-        // Check suffix empty before call this function
-        // Updates max_value depending on the suffix
-        auto finish = [&suffix, &max_value](UInt64 base, int power_of_two)
-        {
-            ++suffix.position();
-            if (*suffix.position() == 'i')
-            {
-                ++suffix.position();
-                return multipleBinarySuffix(max_value, power_of_two); // NOLINT // For binary suffixes, such as ki, Mi, Gi, etc.
-            }
-            else
-            {
-                return multipleDeciMalSuffix(max_value, base); // For decimal suffixes, such as k, M, G etc.
-            }
-        };
-
-        switch (*suffix.position())
-        {
-            case 'k':
-                [[fallthrough]];
-            case 'K':
-                return finish(1e3, 10);
-            case 'M':
-                return finish(1e6, 20);
-            case 'G':
-                return finish(1e9, 30);
-            case 'T':
-                return finish(1e12, 40);
-            default:
-                return false;
-        }
-    }
-
-    bool isSizeSuffix(const char c)
-    {
-        return c == 'k' || c == 'K' || c == 'M' || c == 'G' || c == 'T';
-    }
-
-    template <typename T, typename = std::enable_if_t<std::is_same_v<T, QuotaValue> || std::is_same_v<T, double>>>
-    bool parseWithSizeSuffix(std::string_view s, T & max_value)
-    {
-        ReadBufferFromMemory buf(s.data(), s.size());
-
-        // Consume space
-        while (!buf.eof() && (*buf.position()) == ' ')
-        {
-            ++buf.position();
-        }
-
-        if constexpr (is_integer<T>)
-        {
-            if (!tryReadIntText(max_value, buf))
-            {
-                return false;
-            }
-        }
-        else
-        {
-            if (!tryReadFloatText(max_value, buf))
-            {
-                return false;
-            }
-            // Underflow
-            if (max_value < 0)
-            {
-                return false;
-            }
-        }
-
-        // Has suffix
-        if (!buf.eof() && isSizeSuffix(*buf.position()))
-        {
-            if (!multipleSizeSuffix(max_value, buf))
-            {
-                return false;
-            }
-        }
-
-        // Consume space
-        while (!buf.eof() && isspace(*buf.position()))
-        {
-            ++buf.position();
-        }
-
-        // More after size suffix, illegal syntax
-        if (!buf.eof())
-        {
-            return false;
-        }
-        return true;
-    }
-
-
     template <typename T, typename = std::enable_if_t<std::is_same_v<T, double> || std::is_same_v<T, QuotaValue>>>
-    bool fieldToNumber(const Field & f, T & max_value)
+    T fieldToNumber(const Field & f)
     {
         if (f.getType() == Field::Types::String)
-        {
-            if (!parseWithSizeSuffix<T>(f.get<const String &>(), max_value))
-            {
-                return false;
-            }
-            return true;
-        }
+            return parseWithSizeSuffix<QuotaValue>(boost::algorithm::trim_copy(f.get<std::string>()));
         else
-        {
-            max_value = applyVisitor(FieldVisitorConvertToNumber<T>(), f);
-            return true;
-        }
+            return applyVisitor(FieldVisitorConvertToNumber<T>(), f);
     }
 
     bool parseMaxValue(IParserBase::Pos & pos, Expected & expected, QuotaType quota_type, QuotaValue & max_value)
@@ -263,24 +126,10 @@ namespace
         const Field & max_field = ast->as<ASTLiteral &>().value;
         const auto & type_info = QuotaTypeInfo::get(quota_type);
         if (type_info.output_denominator == 1)
-        {
-            return fieldToNumber<QuotaValue>(max_field, max_value);
-        }
+            max_value = fieldToNumber<QuotaValue>(max_field);
         else
-        {
-            // 2023-04-25 : only for EXECUTION_TIME
-            double tmp_max_value = 0;
-            if (fieldToNumber<double>(max_field, tmp_max_value))
-            {
-                constexpr QuotaValue max_quota = std::numeric_limits<QuotaValue>::max();
-                // Multiplication will cause overflow
-                if (max_quota / type_info.output_denominator < tmp_max_value) // NOLINT
-                    return false;
-                max_value = static_cast<QuotaValue>(tmp_max_value * type_info.output_denominator);
-                return true;
-            }
-            return false;
-        }
+            max_value = static_cast<QuotaValue>(fieldToNumber<double>(max_field) * type_info.output_denominator);
+        return true;
     }
 
     bool parseLimits(IParserBase::Pos & pos, Expected & expected, std::vector<std::pair<QuotaType, QuotaValue>> & limits)
diff --git a/tests/queries/0_stateless/01297_create_quota.reference b/tests/queries/0_stateless/01297_create_quota.reference
index 60ab84ab813..fbd89a8dd54 100644
--- a/tests/queries/0_stateless/01297_create_quota.reference
+++ b/tests/queries/0_stateless/01297_create_quota.reference
@@ -80,9 +80,15 @@ CREATE QUOTA q11_01297 FOR INTERVAL 1 minute MAX execution_time = 12000000
 CREATE QUOTA q12_01297 FOR INTERVAL 1 minute MAX execution_time = 12582912
 CREATE QUOTA q13_01297 FOR INTERVAL 1 minute MAX execution_time = 12000000000
 CREATE QUOTA q14_01297 FOR INTERVAL 1 minute MAX execution_time = 12884901888
+CREATE QUOTA q15_01297 FOR INTERVAL 1 minute MAX query_selects = 1
+CREATE QUOTA q16_01297 FOR INTERVAL 1 minute MAX execution_time = 1.5
 -- overflow test
 CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 18446744073709551615
 CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 18446744073
+CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 0
+CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 18446744073.709553
+CREATE QUOTA q3_01297 FOR INTERVAL 1 minute MAX execution_time = 18446744073.709553
+CREATE QUOTA q4_01297 FOR INTERVAL 1 minute MAX execution_time = 18446744073.709553
 -- zero test
 CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 0
 CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 0
diff --git a/tests/queries/0_stateless/01297_create_quota.sql b/tests/queries/0_stateless/01297_create_quota.sql
index 5a7cecd03ec..c47f68d69ae 100644
--- a/tests/queries/0_stateless/01297_create_quota.sql
+++ b/tests/queries/0_stateless/01297_create_quota.sql
@@ -1,7 +1,7 @@
 -- Tags: no-parallel
 
 DROP QUOTA IF EXISTS q1_01297, q2_01297, q3_01297, q4_01297, q5_01297, q6_01297, q7_01297, q8_01297, q9_01297,
-q10_01297, q11_01297, q12_01297, q13_01297, q14_01297, q15_01297, q16_01297, q17_01297;
+q10_01297, q11_01297, q12_01297, q13_01297, q14_01297, q15_01297, q16_01297, q17_01297,q18_01297;
 
 DROP QUOTA IF EXISTS q2_01297_renamed;
 DROP USER IF EXISTS u1_01297;
@@ -154,6 +154,10 @@ CREATE QUOTA q11_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '12M';
 CREATE QUOTA q12_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '12Mi';
 CREATE QUOTA q13_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '12G';
 CREATE QUOTA q14_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '12Gi';
+CREATE QUOTA q15_01297 FOR INTERVAL 1 MINUTE MAX query_selects = 1.5;
+CREATE QUOTA q16_01297 FOR INTERVAL 1 MINUTE MAX execution_time = 1.5;
+CREATE QUOTA q17_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '1.5'; -- { clientError 27 }
+CREATE QUOTA q18_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1.5'; -- { clientError 27 }
 SHOW CREATE QUOTA q1_01297;
 SHOW CREATE QUOTA q2_01297;
 SHOW CREATE QUOTA q3_01297;
@@ -168,6 +172,8 @@ SHOW CREATE QUOTA q11_01297;
 SHOW CREATE QUOTA q12_01297;
 SHOW CREATE QUOTA q13_01297;
 SHOW CREATE QUOTA q14_01297;
+SHOW CREATE QUOTA q15_01297;
+SHOW CREATE QUOTA q16_01297;
 DROP QUOTA IF EXISTS q1_01297;
 DROP QUOTA IF EXISTS q2_01297;
 DROP QUOTA IF EXISTS q3_01297;
@@ -182,6 +188,8 @@ DROP QUOTA IF EXISTS q11_01297;
 DROP QUOTA IF EXISTS q12_01297;
 DROP QUOTA IF EXISTS q13_01297;
 DROP QUOTA IF EXISTS q14_01297;
+DROP QUOTA IF EXISTS q15_01297;
+DROP QUOTA IF EXISTS q16_01297;
 SELECT '-- overflow test';
 CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '18446744073709551615';
 CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '18446744073';
@@ -189,10 +197,18 @@ SHOW CREATE QUOTA q1_01297;
 SHOW CREATE QUOTA q2_01297;
 DROP QUOTA IF EXISTS q1_01297;
 DROP QUOTA IF EXISTS q2_01297;
-CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '18446744073709551616'; -- { clientError 62 }
-CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1T'; -- { clientError 62 }
-CREATE QUOTA q3_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1Ti'; -- { clientError 62 }
-CREATE QUOTA q4_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '18446744074'; -- { clientError 62 }
+CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '18446744073709551616';
+CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1T';
+CREATE QUOTA q3_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1Ti';
+CREATE QUOTA q4_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '18446744074';
+SHOW CREATE QUOTA q1_01297;
+SHOW CREATE QUOTA q2_01297;
+SHOW CREATE QUOTA q3_01297;
+SHOW CREATE QUOTA q4_01297;
+DROP QUOTA IF EXISTS q1_01297;
+DROP QUOTA IF EXISTS q2_01297;
+DROP QUOTA IF EXISTS q3_01297;
+DROP QUOTA IF EXISTS q4_01297;
 SELECT '-- zero test';
 CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '0';
 CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '0';
@@ -201,8 +217,8 @@ SHOW CREATE QUOTA q2_01297;
 DROP QUOTA IF EXISTS q1_01297;
 DROP QUOTA IF EXISTS q2_01297;
 SELECT '-- underflow test';
-CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '-1'; -- { clientError 62 }
-CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '-1'; -- { clientError 62 }
+CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '-1'; -- { clientError 72 }
+CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '-1'; -- { clientError 72 }
 SELECT '-- syntax test';
 CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = ' 12 ';
 CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = ' 12 ';
@@ -235,11 +251,11 @@ DROP QUOTA IF EXISTS q8_01297;
 DROP QUOTA IF EXISTS q9_01297;
 DROP QUOTA IF EXISTS q10_01297;
 SELECT '-- bad syntax test';
-CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '1 1'; -- { clientError 62 }
-CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1 1'; -- { clientError 62 }
-CREATE QUOTA q3_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '1K 1'; -- { clientError 62 }
-CREATE QUOTA q4_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1K 1'; -- { clientError 62 }
-CREATE QUOTA q5_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '1K1'; -- { clientError 62 }
-CREATE QUOTA q6_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1K1'; -- { clientError 62 }
-CREATE QUOTA q7_01297 FOR INTERVAL 1 MINUTE MAX query_selects = 'foo'; -- { clientError 62 }
-CREATE QUOTA q8_01297 FOR INTERVAL 1 MINUTE MAX execution_time = 'bar'; -- { clientError 62 }
+CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '1 1'; -- { clientError 27 }
+CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1 1'; -- { clientError 27 }
+CREATE QUOTA q3_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '1K 1'; -- { clientError 27 }
+CREATE QUOTA q4_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1K 1'; -- { clientError 27 }
+CREATE QUOTA q5_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '1K1'; -- { clientError 27 }
+CREATE QUOTA q6_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1K1'; -- { clientError 27 }
+CREATE QUOTA q7_01297 FOR INTERVAL 1 MINUTE MAX query_selects = 'foo'; -- { clientError 27 }
+CREATE QUOTA q8_01297 FOR INTERVAL 1 MINUTE MAX execution_time = 'bar'; -- { clientError 27 }

From d890114c5e2224f86152641803abbedcae2d758e Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 11 May 2023 14:21:07 +0200
Subject: [PATCH 424/689] Print debugging information in workflow_jobs_lambda

---
 tests/ci/workflow_jobs_lambda/app.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/ci/workflow_jobs_lambda/app.py b/tests/ci/workflow_jobs_lambda/app.py
index 49d475d11dc..bc8e1212be5 100644
--- a/tests/ci/workflow_jobs_lambda/app.py
+++ b/tests/ci/workflow_jobs_lambda/app.py
@@ -251,14 +251,20 @@ def send_event_workflow_job(workflow_job: WorkflowJob) -> None:
         clickhouse_client.insert_event_into(**kwargs)
 
 
-def handler(event: dict, _: Any) -> dict:
+def handler(event: dict, context: Any) -> dict:
     if event["isBase64Encoded"]:
         event_data = json.loads(b64decode(event["body"]))
     else:
         event_data = json.loads(event["body"])
 
     repo = event_data["repository"]
-    wf_job = event_data["workflow_job"]
+    try:
+        wf_job = event_data["workflow_job"]
+    except KeyError:
+        logging.error("The event does not contain valid workflow_jobs data")
+        logging.error("The event data: %s", event)
+        logging.error("The context data: %s", context)
+
     workflow_job = WorkflowJob(
         wf_job["id"],
         wf_job["run_id"],

From 1340032c8895de12c148cefc8cf579719cb2054f Mon Sep 17 00:00:00 2001
From: Eridanus <maodingxuan@foxmail.com>
Date: Thu, 11 May 2023 20:32:59 +0800
Subject: [PATCH 425/689] improvement(parser): add support for size suffixes in
 quota creation statement parameters

This commit adds support for creating quotas with size suffixes in the
parameter list of a quota creation statement. The parser now recognizes
size suffixes such as "KB", "MB", and "GB" and correctly converts them
to their corresponding values in bytes. This feature improves the
usability of the quota creation statement and makes it more flexible.
The commit also includes additional tests to ensure that the new syntax
is parsed correctly.
Attention: we did not support string literal with double like
execute_time = '1.5'
---
 tests/queries/0_stateless/01297_create_quota.reference | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01297_create_quota.reference b/tests/queries/0_stateless/01297_create_quota.reference
index fbd89a8dd54..76040b5dba5 100644
--- a/tests/queries/0_stateless/01297_create_quota.reference
+++ b/tests/queries/0_stateless/01297_create_quota.reference
@@ -86,9 +86,9 @@ CREATE QUOTA q16_01297 FOR INTERVAL 1 minute MAX execution_time = 1.5
 CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 18446744073709551615
 CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 18446744073
 CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 0
-CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 18446744073.709553
-CREATE QUOTA q3_01297 FOR INTERVAL 1 minute MAX execution_time = 18446744073.709553
-CREATE QUOTA q4_01297 FOR INTERVAL 1 minute MAX execution_time = 18446744073.709553
+CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 9223372036.854776
+CREATE QUOTA q3_01297 FOR INTERVAL 1 minute MAX execution_time = 9223372036.854776
+CREATE QUOTA q4_01297 FOR INTERVAL 1 minute MAX execution_time = 9223372036.854776
 -- zero test
 CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 0
 CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 0

From fd7374fbc5fc357051c5ea2544e2c1c0f73adcac Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 11 May 2023 12:38:46 +0000
Subject: [PATCH 426/689] Analyzer: fix column not found for optimized prewhere
 with sample by

---
 .../QueryPlan/Optimizations/optimizePrewhere.cpp    | 13 +++++++++++--
 tests/broken_tests.txt                              |  1 -
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
index 557b76293ea..ca8a412bf2e 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
@@ -133,8 +133,17 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes & nodes)
     if (!optimize_move_to_prewhere)
         return;
 
-    const auto & storage = read_from_merge_tree->getStorageSnapshot()->storage;
-    const auto & storage_metadata = read_from_merge_tree->getStorageSnapshot()->metadata;
+    const auto & storage_snapshot = read_from_merge_tree->getStorageSnapshot();
+
+    if (table_expression_modifiers && table_expression_modifiers->hasSampleSizeRatio())
+    {
+        const auto & sampling_key = storage_snapshot->getMetadataForQuery()->getSamplingKey();
+        const auto & sampling_columns = sampling_key.sample_block.getColumnsWithTypeAndName();
+        required_columns_after_filter.insert(required_columns_after_filter.end(), sampling_columns.begin(), sampling_columns.end());
+    }
+
+    const auto & storage = storage_snapshot->storage;
+    const auto & storage_metadata = storage_snapshot->metadata;
     auto column_sizes = storage.getColumnSizes();
     if (column_sizes.empty())
         return;
diff --git a/tests/broken_tests.txt b/tests/broken_tests.txt
index f3fd0f395d4..5bb60d9b75d 100644
--- a/tests/broken_tests.txt
+++ b/tests/broken_tests.txt
@@ -112,7 +112,6 @@
 02428_parameterized_view
 02458_use_structure_from_insertion_table
 02479_race_condition_between_insert_and_droppin_mv
-02481_merge_array_join_sample_by
 02493_inconsistent_hex_and_binary_number
 02494_optimize_group_by_function_keys_and_alias_columns
 02511_complex_literals_as_aggregate_function_parameters

From 3160483c17250d67fd661abb63a76c0b2ec9761e Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Thu, 11 May 2023 12:56:59 +0000
Subject: [PATCH 427/689] Add query parameters to clickhouse-local

---
 src/Client/LocalConnection.cpp                         |  6 ++----
 .../0_stateless/02725_local_query_parameters.reference |  1 +
 .../0_stateless/02725_local_query_parameters.sh        | 10 ++++++++++
 3 files changed, 13 insertions(+), 4 deletions(-)
 create mode 100644 tests/queries/0_stateless/02725_local_query_parameters.reference
 create mode 100755 tests/queries/0_stateless/02725_local_query_parameters.sh

diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp
index 4fb80f0ea04..6bb792ac51e 100644
--- a/src/Client/LocalConnection.cpp
+++ b/src/Client/LocalConnection.cpp
@@ -73,9 +73,6 @@ void LocalConnection::sendQuery(
     bool,
     std::function<void(const Progress &)> process_progress_callback)
 {
-    if (!query_parameters.empty())
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "clickhouse local does not support query parameters");
-
     /// Suggestion comes without client_info.
     if (client_info)
         query_context = session.makeQueryContext(*client_info);
@@ -90,6 +87,7 @@ void LocalConnection::sendQuery(
     if (!current_database.empty())
         query_context->setCurrentDatabase(current_database);
 
+    query_context->addQueryParameters(query_parameters);
 
     state.reset();
     state.emplace();
@@ -484,7 +482,7 @@ void LocalConnection::setDefaultDatabase(const String & database)
 
 UInt64 LocalConnection::getServerRevision(const ConnectionTimeouts &)
 {
-    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented");
+    return DBMS_TCP_PROTOCOL_VERSION;
 }
 
 const String & LocalConnection::getServerTimezone(const ConnectionTimeouts &)
diff --git a/tests/queries/0_stateless/02725_local_query_parameters.reference b/tests/queries/0_stateless/02725_local_query_parameters.reference
new file mode 100644
index 00000000000..9972842f982
--- /dev/null
+++ b/tests/queries/0_stateless/02725_local_query_parameters.reference
@@ -0,0 +1 @@
+1	1
diff --git a/tests/queries/0_stateless/02725_local_query_parameters.sh b/tests/queries/0_stateless/02725_local_query_parameters.sh
new file mode 100755
index 00000000000..92d7f645454
--- /dev/null
+++ b/tests/queries/0_stateless/02725_local_query_parameters.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest, no-parallel, no-replicated-database
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+$CLICKHOUSE_LOCAL --multiquery -q "
+    SET param_x=1;
+    SELECT {x:UInt64}, {x:String};"

From 26743b54394de20cf0ff6e3e56b08bdc642c1330 Mon Sep 17 00:00:00 2001
From: "Diego Nieto (lesandie)" <lesandie@gmail.com>
Date: Thu, 11 May 2023 15:36:47 +0200
Subject: [PATCH 428/689] Fix Local Cache documentation explanations

---
 docs/en/operations/storing-data.md | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md
index ac6ea22ab75..495716858ec 100644
--- a/docs/en/operations/storing-data.md
+++ b/docs/en/operations/storing-data.md
@@ -172,17 +172,19 @@ Example of configuration for versions earlier than 22.8:
     </storage_configuration>
 ```
 
-Cache **configuration settings**:
+File Cache **configuration settings**:
+
+These settings should be defined in the disk configuration section.
 
 - `path` - path to the directory with cache. Default: None, this setting is obligatory.
 
 - `max_size` - maximum size of the cache in bytes or in readable format, e.g. `ki, Mi, Gi, etc`, example `10Gi` (such format works starting from `22.10` version). When the limit is reached, cache files are evicted according to the cache eviction policy. Default: None, this setting is obligatory.
 
-- `cache_on_write_operations` - allow to turn on `write-through` cache (caching data on any write operations: `INSERT` queries, background merges). Default: `false`. The `write-through` cache can be disabled per query using setting `enable_filesystem_cache_on_write_operations` (data is cached only if both cache config settings and corresponding query setting are enabled).
+- `cache_on_write_operations` - allow to turn on `write-through` cache (caching data on any write operations: `INSERT` queries, background merges). Default: `false`. 
 
 - `enable_filesystem_query_cache_limit` - allow to limit the size of cache which is downloaded within each query (depends on user setting `max_query_cache_size`). Default: `false`.
 
-- `enable_cache_hits_threshold` - a number, which defines how many times some data needs to be read before it will be cached. Default: `0`, e.g. the data is cached at the first attempt to read it.
+- `enable_cache_hits_threshold` - number which defines how many times some data needs to be read before it will be cached. Default: `0`, e.g. the data is cached at the first attempt to read it.
 
 - `do_not_evict_index_and_mark_files` - do not evict small frequently used files according to cache policy. Default: `false`. This setting was added in version 22.8. If you used filesystem cache before this version, then it will not work on versions starting from 22.8 if this setting is set to `true`. If you want to use this setting, clear old cache created before version 22.8 before upgrading.
 
@@ -190,19 +192,21 @@ Cache **configuration settings**:
 
 - `max_elements` - a limit for a number of cache files. Default: `1048576`.
 
-Cache **query settings**:
+File Cache **query settings**:
 
-- `enable_filesystem_cache` - allows to disable cache per query even if storage policy was configured with `cache` disk type. Default: `true`.
+Some of these settings will disable cache per query features that are enabled by default. For example, setting `cache_on_write_operations` to 1 means that general file and per query cache are enabled but also setting `enable_filesystem_cache_on_write_operations` to 1 means that file cache is enabled but disabled per query cache. The same approach must be used for `enable_filesystem_cache`, if set to 1 cache per query is disabled.
 
-- `read_from_filesystem_cache_if_exists_otherwise_bypass_cache` - allows to use cache in query only if it already exists, otherwise query data will not be written to local cache storage. Default: `false`.
+- `enable_filesystem_cache` - allows to disable cache per query even if storage policy was configured with `cache` disk type. Default: `true`. This setting should be defined in the disk configuration section.
 
-- `enable_filesystem_cache_on_write_operations` - turn on `write-through` cache. This setting works only if setting `cache_on_write_operations` in cache configuration is turned on.
+- `read_from_filesystem_cache_if_exists_otherwise_bypass_cache` - allows to use cache in query only if it already exists, otherwise query data will not be written to local cache storage. Default: `false`. This is a profile level configuration setting.
 
-- `enable_filesystem_cache_log` - turn on logging to `system.filesystem_cache_log` table. Gives a detailed view of cache usage per query. Default: `false`.
+- `enable_filesystem_cache_on_write_operations` - turn on `write-through` cache. This setting works only if setting `cache_on_write_operations` in cache configuration is turned on. This setting should be defined in the disk configuration section.
 
-- `max_query_cache_size` - a limit for the cache size, which can be written to local cache storage. Requires enabled `enable_filesystem_query_cache_limit` in cache configuration. Default: `false`.
+- `enable_filesystem_cache_log` - turn on logging to `system.filesystem_cache_log` table. Gives a detailed view of cache usage per query. Default: `false`. This is a profile level configuration setting.
 
-- `skip_download_if_exceeds_query_cache` - allows to change the behaviour of setting `max_query_cache_size`. Default: `true`. If this setting is turned on and cache download limit during query was reached, no more cache will be downloaded to cache storage. If this setting is turned off and cache download limit during query was reached, cache will still be written by cost of evicting previously downloaded (within current query) data, e.g. second behaviour allows to preserve `last recentltly used` behaviour while keeping query cache limit.
+- `max_query_cache_size` - a limit for the cache size, which can be written to local cache storage. Requires enabled `enable_filesystem_query_cache_limit` in cache configuration. Default: `false`. This setting should be defined in the disk configuration section.
+
+- `skip_download_if_exceeds_query_cache` - allows to change the behaviour of setting `max_query_cache_size`. Default: `true`. If this setting is turned on and cache download limit during query was reached, no more cache will be downloaded to cache storage. If this setting is turned off and cache download limit during query was reached, cache will still be written by cost of evicting previously downloaded (within current query) data, e.g. second behaviour allows to preserve `last recently used` behaviour while keeping query cache limit. This is a profile level configuration setting.
 
 ** Warning **
 Cache configuration settings and cache query settings correspond to the latest ClickHouse version, for earlier versions something might not be supported.

From 10e0c1d832769f3244e4ad7563a02b6cd532883b Mon Sep 17 00:00:00 2001
From: "Diego Nieto (lesandie)" <lesandie@gmail.com>
Date: Thu, 11 May 2023 16:06:14 +0200
Subject: [PATCH 429/689] Reworked documentation using local cache section

---
 docs/en/operations/storing-data.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md
index 495716858ec..78c0fb8a049 100644
--- a/docs/en/operations/storing-data.md
+++ b/docs/en/operations/storing-data.md
@@ -194,7 +194,7 @@ These settings should be defined in the disk configuration section.
 
 File Cache **query settings**:
 
-Some of these settings will disable cache per query features that are enabled by default. For example, setting `cache_on_write_operations` to 1 means that general file and per query cache are enabled but also setting `enable_filesystem_cache_on_write_operations` to 1 means that file cache is enabled but disabled per query cache. The same approach must be used for `enable_filesystem_cache`, if set to 1 cache per query is disabled.
+Some of these settings will disable cache per query features that are enabled by default. For example, setting `cache_on_write_operations` to 1 means that general file and per query cache are enabled but also setting `enable_filesystem_cache_on_write_operations` to 1 means that file cache is enabled but disabled per query cache. The same approach must be used for `enable_filesystem_cache`, if set to 1, cache per query is disabled.
 
 - `enable_filesystem_cache` - allows to disable cache per query even if storage policy was configured with `cache` disk type. Default: `true`. This setting should be defined in the disk configuration section.
 

From 8a664e2cac582669871bf691af1187c645cd5261 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 11 May 2023 16:09:59 +0200
Subject: [PATCH 430/689] Upgrade all lambdas runtime to 3.10, fix requirements

The recent update of urllib3 completely broken the world

- https://github.com/urllib3/urllib3/issues/2168
- https://github.com/boto/botocore/issues/2926
- https://github.com/psf/requests/issues/6443
---
 tests/ci/autoscale_runners_lambda/requirements.txt         | 2 +-
 tests/ci/cancel_and_rerun_workflow_lambda/requirements.txt | 4 ++--
 tests/ci/ci_runners_metrics_lambda/requirements.txt        | 4 ++--
 tests/ci/runner_token_rotation_lambda/requirements.txt     | 4 ++--
 tests/ci/team_keys_lambda/build_and_deploy_archive.sh      | 2 +-
 tests/ci/team_keys_lambda/requirements.txt                 | 2 +-
 tests/ci/terminate_runner_lambda/requirements.txt          | 4 ++--
 tests/ci/workflow_approve_rerun_lambda/requirements.txt    | 4 ++--
 tests/ci/workflow_jobs_lambda/requirements.txt             | 2 +-
 9 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/tests/ci/autoscale_runners_lambda/requirements.txt b/tests/ci/autoscale_runners_lambda/requirements.txt
index f2293605cf1..3bcbe2dfd07 100644
--- a/tests/ci/autoscale_runners_lambda/requirements.txt
+++ b/tests/ci/autoscale_runners_lambda/requirements.txt
@@ -1 +1 @@
-requests
+requests<2.30
diff --git a/tests/ci/cancel_and_rerun_workflow_lambda/requirements.txt b/tests/ci/cancel_and_rerun_workflow_lambda/requirements.txt
index e607f1a9f39..98be09ab232 100644
--- a/tests/ci/cancel_and_rerun_workflow_lambda/requirements.txt
+++ b/tests/ci/cancel_and_rerun_workflow_lambda/requirements.txt
@@ -1,3 +1,3 @@
-requests
+requests<2.30
 PyJWT
-cryptography==37.0.4
+cryptography<38
diff --git a/tests/ci/ci_runners_metrics_lambda/requirements.txt b/tests/ci/ci_runners_metrics_lambda/requirements.txt
index e607f1a9f39..98be09ab232 100644
--- a/tests/ci/ci_runners_metrics_lambda/requirements.txt
+++ b/tests/ci/ci_runners_metrics_lambda/requirements.txt
@@ -1,3 +1,3 @@
-requests
+requests<2.30
 PyJWT
-cryptography==37.0.4
+cryptography<38
diff --git a/tests/ci/runner_token_rotation_lambda/requirements.txt b/tests/ci/runner_token_rotation_lambda/requirements.txt
index e607f1a9f39..98be09ab232 100644
--- a/tests/ci/runner_token_rotation_lambda/requirements.txt
+++ b/tests/ci/runner_token_rotation_lambda/requirements.txt
@@ -1,3 +1,3 @@
-requests
+requests<2.30
 PyJWT
-cryptography==37.0.4
+cryptography<38
diff --git a/tests/ci/team_keys_lambda/build_and_deploy_archive.sh b/tests/ci/team_keys_lambda/build_and_deploy_archive.sh
index 1ea2935c445..4aee85c588a 100644
--- a/tests/ci/team_keys_lambda/build_and_deploy_archive.sh
+++ b/tests/ci/team_keys_lambda/build_and_deploy_archive.sh
@@ -5,7 +5,7 @@ WORKDIR=$(dirname "$0")
 WORKDIR=$(readlink -f "${WORKDIR}")
 cd "$WORKDIR"
 
-PY_VERSION=3.9
+PY_VERSION=3.10
 PY_EXEC="python${PY_VERSION}"
 DOCKER_IMAGE="python:${PY_VERSION}-slim"
 LAMBDA_NAME=$(basename "$WORKDIR")
diff --git a/tests/ci/team_keys_lambda/requirements.txt b/tests/ci/team_keys_lambda/requirements.txt
index f2293605cf1..3bcbe2dfd07 100644
--- a/tests/ci/team_keys_lambda/requirements.txt
+++ b/tests/ci/team_keys_lambda/requirements.txt
@@ -1 +1 @@
-requests
+requests<2.30
diff --git a/tests/ci/terminate_runner_lambda/requirements.txt b/tests/ci/terminate_runner_lambda/requirements.txt
index e607f1a9f39..98be09ab232 100644
--- a/tests/ci/terminate_runner_lambda/requirements.txt
+++ b/tests/ci/terminate_runner_lambda/requirements.txt
@@ -1,3 +1,3 @@
-requests
+requests<2.30
 PyJWT
-cryptography==37.0.4
+cryptography<38
diff --git a/tests/ci/workflow_approve_rerun_lambda/requirements.txt b/tests/ci/workflow_approve_rerun_lambda/requirements.txt
index e607f1a9f39..98be09ab232 100644
--- a/tests/ci/workflow_approve_rerun_lambda/requirements.txt
+++ b/tests/ci/workflow_approve_rerun_lambda/requirements.txt
@@ -1,3 +1,3 @@
-requests
+requests<2.30
 PyJWT
-cryptography==37.0.4
+cryptography<38
diff --git a/tests/ci/workflow_jobs_lambda/requirements.txt b/tests/ci/workflow_jobs_lambda/requirements.txt
index f2293605cf1..3bcbe2dfd07 100644
--- a/tests/ci/workflow_jobs_lambda/requirements.txt
+++ b/tests/ci/workflow_jobs_lambda/requirements.txt
@@ -1 +1 @@
-requests
+requests<2.30

From 00fdfa115fc314ff1007713e83ef46327c369b97 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 11 May 2023 16:21:33 +0200
Subject: [PATCH 431/689] Suppress MSan warning in
 NgramDistanceImpl::unrollLowering()

NgramDistanceImpl::unrollLowering() relies on the fact that PODArray has
padding and it is OK to access more items.

Here is an MSan report:

    ==656==WARNING: MemorySanitizer: use-of-uninitialized-value
        0 0x557fd825485f in DB::NgramDistanceImpl<4ul, char8_t, false, true, false>::vectorConstant(DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 63ul, 64ul> const&, DB::PODArray<unsigned long, 4096ul, Allocator<false, false>, 63ul, 64ul> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, DB::PODArray<float, 4096ul, Allocator<false, false>, 63ul, 64ul>&) (/usr/bin/clickhouse+0x124d885f) (BuildId: 76773125d8739591c75d4f4d263a2ffe7ca96855)
        1 0x557fd824eb83 in DB::FunctionsStringSimilarity<DB::NgramDistanceImpl<4ul, char8_t, false, true, false>, DB::NameNgramSearchCaseInsensitive>::executeImpl(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName>> const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long) const (/usr/bin/clickhouse+0x124d2b83) (BuildId: 76773125d8739591c75d4f4d263a2ffe7ca96855)
        2 0x557fd50023b7 in DB::FunctionToExecutableFunctionAdaptor::executeImpl() const (/usr/bin/clickhouse+0xf2863b7) (BuildId: 76773125d8739591c75d4f4d263a2ffe7ca96855)

      Uninitialized value was stored to memory at
        0 0x557fd4f8da5a in __msan_memcpy (/usr/bin/clickhouse+0xf211a5a) (BuildId: 76773125d8739591c75d4f4d263a2ffe7ca96855)
        1 0x557fd8253803 in DB::NgramDistanceImpl<4ul, char8_t, false, true, false>::vectorConstant(DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 63ul, 64ul> const&, DB::PODArray<unsigned long, 4096ul, Allocator<false, false>, 63ul, 64ul> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, DB::PODArray<float, 4096ul, Allocator<false, false>, 63ul, 64ul>&) (/usr/bin/clickhouse+0x124d7803) (BuildId: 76773125d8739591c75d4f4d263a2ffe7ca96855)
        2 0x557fd824eb83 in DB::FunctionsStringSimilarity<DB::NgramDistanceImpl<4ul, char8_t, false, true, false>, DB::NameNgramSearchCaseInsensitive>::executeImpl(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName>> const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long) const (/usr/bin/clickhouse+0x124d2b83) (BuildId: 76773125d8739591c75d4f4d263a2ffe7ca96855)
        3 0x557fd50023b7 in DB::FunctionToExecutableFunctionAdaptor::executeImpl() const (/usr/bin/clickhouse+0xf2863b7) (BuildId: 76773125d8739591c75d4f4d263a2ffe7ca96855)

      Uninitialized value was stored to memory at
        0 0x557fd4f8da5a in __msan_memcpy (/usr/bin/clickhouse+0xf211a5a) (BuildId: 76773125d8739591c75d4f4d263a2ffe7ca96855)
        1 0x5580061699f5 in detail::memcpySmallAllowReadWriteOverflow15Impl(char*, char const*, long) build_docker/./src/Common/memcpySmall.h:42:13
        2 0x5580061699f5 in memcpySmallAllowReadWriteOverflow15(void*, void const*, unsigned long) build_docker/./src/Common/memcpySmall.h:57:5
        3 0x5580061699f5 in DB::ColumnString::replicate(DB::PODArray<unsigned long, 4096ul, Allocator<false, false>, 63ul, 64ul> const&) const build_docker/./src/Columns/ColumnString.cpp:462:13
        4 0x558005d3fae4 in DB::ColumnConst::convertToFullColumn() const build_docker/./src/Columns/ColumnConst.cpp:48:18

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Functions/FunctionsStringSimilarity.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/Functions/FunctionsStringSimilarity.cpp b/src/Functions/FunctionsStringSimilarity.cpp
index faa01abb675..7c0db701302 100644
--- a/src/Functions/FunctionsStringSimilarity.cpp
+++ b/src/Functions/FunctionsStringSimilarity.cpp
@@ -108,6 +108,10 @@ struct NgramDistanceImpl
 
         if constexpr (case_insensitive)
         {
+#if defined(MEMORY_SANITIZER)
+            /// Due to PODArray padding accessing more elements should be OK
+            __msan_unpoison(code_points + (N - 1), padding_offset * sizeof(CodePoint));
+#endif
             /// We really need template lambdas with C++20 to do it inline
             unrollLowering<N - 1>(code_points, std::make_index_sequence<padding_offset>());
         }

From 84aa97b738836293bae72114d8993462999a66db Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Thu, 11 May 2023 14:52:01 +0000
Subject: [PATCH 432/689] fix arrayMap with array of tuples with single
 argument

---
 src/Functions/array/FunctionArrayMapped.h                  | 7 ++++---
 .../0_stateless/02735_array_map_array_of_tuples.reference  | 4 ++++
 .../0_stateless/02735_array_map_array_of_tuples.sql        | 4 ++++
 3 files changed, 12 insertions(+), 3 deletions(-)
 create mode 100644 tests/queries/0_stateless/02735_array_map_array_of_tuples.reference
 create mode 100644 tests/queries/0_stateless/02735_array_map_array_of_tuples.sql

diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h
index 9917a38e144..a7ab80f697a 100644
--- a/src/Functions/array/FunctionArrayMapped.h
+++ b/src/Functions/array/FunctionArrayMapped.h
@@ -135,7 +135,7 @@ public:
 
         size_t num_function_arguments = function_type->getArgumentTypes().size();
         if (is_single_array_argument
-            && tuple_argument_size
+            && tuple_argument_size > 1
             && tuple_argument_size == num_function_arguments)
         {
             assert(nested_types.size() == 1);
@@ -337,12 +337,13 @@ public:
                 }
 
                 const auto * column_tuple = checkAndGetColumn<ColumnTuple>(&column_array->getData());
-                if (is_single_array_argument && column_tuple && column_tuple->getColumns().size() == num_function_arguments)
+                size_t tuple_size = column_tuple ? column_tuple->getColumns().size() : 0;
+
+                if (is_single_array_argument && tuple_size > 1 && tuple_size == num_function_arguments)
                 {
                     const auto & type_tuple = assert_cast<const DataTypeTuple &>(*array_type->getNestedType());
                     const auto & tuple_names = type_tuple.getElementNames();
 
-                    size_t tuple_size = column_tuple->getColumns().size();
                     arrays.reserve(column_tuple->getColumns().size());
                     for (size_t j = 0; j < tuple_size; ++j)
                     {
diff --git a/tests/queries/0_stateless/02735_array_map_array_of_tuples.reference b/tests/queries/0_stateless/02735_array_map_array_of_tuples.reference
new file mode 100644
index 00000000000..80481fee564
--- /dev/null
+++ b/tests/queries/0_stateless/02735_array_map_array_of_tuples.reference
@@ -0,0 +1,4 @@
+[(1)]
+[1]
+[3]
+[3]
diff --git a/tests/queries/0_stateless/02735_array_map_array_of_tuples.sql b/tests/queries/0_stateless/02735_array_map_array_of_tuples.sql
new file mode 100644
index 00000000000..51d60aa0cd0
--- /dev/null
+++ b/tests/queries/0_stateless/02735_array_map_array_of_tuples.sql
@@ -0,0 +1,4 @@
+SELECT arrayMap((x) -> x, [tuple(1)]);
+SELECT arrayMap((x) -> x.1, [tuple(1)]);
+SELECT arrayMap((x) -> x.1 + x.2, [tuple(1, 2)]);
+SELECT arrayMap((x, y) -> x + y, [tuple(1, 2)]);

From fcf1dd52bc9150d7a019031d3617276cb88afd1f Mon Sep 17 00:00:00 2001
From: Denny Crane <denis.zhuravlov@gmail.com>
Date: Thu, 11 May 2023 14:05:30 -0300
Subject: [PATCH 433/689] Update settings.md

---
 docs/en/operations/settings/settings.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index d1851aa7761..c1d6416837d 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -1636,7 +1636,7 @@ For not replicated tables see [non_replicated_deduplication_window](merge-tree-s
 
 ### async_insert {#async-insert}
 
-Enables or disables asynchronous inserts. Note that deduplication isn't working for such inserts.
+Enables or disables asynchronous inserts. Note that deduplication is disabled by default, see [async_insert_deduplicate](#async-insert-deduplicate).
 
 If enabled, the data is combined into batches before the insertion into tables, so it is possible to do small and frequent insertions into ClickHouse (up to 15000 queries per second) without buffer tables.
 

From ef74e64336db12cd0a623488f21d653b0df9e2aa Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Thu, 11 May 2023 22:18:08 +0200
Subject: [PATCH 434/689] address comments

---
 docs/en/sql-reference/dictionaries/index.md | 2 --
 src/Dictionaries/RegExpTreeDictionary.cpp   | 5 +++++
 src/Dictionaries/RegExpTreeDictionary.h     | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index 74ab7e3c948..5801b7866cb 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -2218,8 +2218,6 @@ LAYOUT(regexp_tree)
 ...
 ```
 
-We only allow `YAMLRegExpTree` to work with regexp_tree dicitionary layout. If you want to use other sources, please set variable `regexp_dict_allow_other_sources` true.
-
 **Source**
 
 We introduce a type of source called `YAMLRegExpTree` representing the structure of Regexp Tree dictionary. An Example of a valid yaml config is like:
diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp
index a0c66f42e8b..9841cadcdca 100644
--- a/src/Dictionaries/RegExpTreeDictionary.cpp
+++ b/src/Dictionaries/RegExpTreeDictionary.cpp
@@ -683,9 +683,11 @@ std::unordered_map<String, ColumnPtr> RegExpTreeDictionary::match(
 
 Pipe RegExpTreeDictionary::read(const Names & , size_t max_block_size, size_t) const
 {
+
     auto it = regex_nodes.begin();
     size_t block_size = 0;
     BlocksList result;
+
     for (;;)
     {
         Block block;
@@ -694,6 +696,7 @@ Pipe RegExpTreeDictionary::read(const Names & , size_t max_block_size, size_t) c
         auto col_regex = std::make_shared<DataTypeString>()->createColumn();
         auto col_keys = std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())->createColumn();
         auto col_values = std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())->createColumn();
+
         for (;it != regex_nodes.end() && block_size < max_block_size; it++, block_size++)
         {
             col_id->insert(it->first);
@@ -709,6 +712,7 @@ Pipe RegExpTreeDictionary::read(const Names & , size_t max_block_size, size_t) c
             col_keys->insert(Array(keys.begin(), keys.end()));
             col_values->insert(Array(values.begin(), values.end()));
         }
+
         block.insert(ColumnWithTypeAndName(std::move(col_id),std::make_shared<DataTypeUInt64>(),kId));
         block.insert(ColumnWithTypeAndName(std::move(col_pid),std::make_shared<DataTypeUInt64>(),kParentId));
         block.insert(ColumnWithTypeAndName(std::move(col_regex),std::make_shared<DataTypeString>(),kRegExp));
@@ -719,6 +723,7 @@ Pipe RegExpTreeDictionary::read(const Names & , size_t max_block_size, size_t) c
             break;
         block_size = 0;
     }
+
     return Pipe(std::make_shared<BlocksListSource>(std::move(result)));
 }
 
diff --git a/src/Dictionaries/RegExpTreeDictionary.h b/src/Dictionaries/RegExpTreeDictionary.h
index 893d1fb7a0b..683588e688f 100644
--- a/src/Dictionaries/RegExpTreeDictionary.h
+++ b/src/Dictionaries/RegExpTreeDictionary.h
@@ -93,7 +93,7 @@ public:
         throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Dictionary {} does not support method `hasKeys`", name);
     }
 
-    Pipe read(const Names &, size_t, size_t) const override;
+    Pipe read(const Names & columns, size_t max_block_size, size_t num_streams) const override;
 
     ColumnPtr getColumn(
         const std::string & attribute_name,

From 478140f8b40da6518e532652e476021574b8bd96 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Wed, 10 May 2023 11:52:09 +0200
Subject: [PATCH 435/689] Fix LimitSeekableReadBuffer.

---
 src/IO/LimitSeekableReadBuffer.cpp            | 46 +++++++++++--------
 .../test_backup_restore_new/test.py           | 23 ++++++++++
 .../test_backup_restore_s3/test.py            | 27 +++++++++++
 3 files changed, 76 insertions(+), 20 deletions(-)

diff --git a/src/IO/LimitSeekableReadBuffer.cpp b/src/IO/LimitSeekableReadBuffer.cpp
index acd7ba6eaa0..587138cb2cf 100644
--- a/src/IO/LimitSeekableReadBuffer.cpp
+++ b/src/IO/LimitSeekableReadBuffer.cpp
@@ -19,7 +19,7 @@ LimitSeekableReadBuffer::LimitSeekableReadBuffer(std::unique_ptr<SeekableReadBuf
     , in(std::move(in_))
     , min_offset(start_offset_)
     , max_offset(start_offset_ + limit_size_)
-    , need_seek(start_offset_)
+    , need_seek(min_offset) /// We always start reading from `min_offset`.
 {
 }
 
@@ -31,44 +31,50 @@ bool LimitSeekableReadBuffer::nextImpl()
     if (need_seek)
     {
         /// Do actual seek.
-        if (in->getPosition() != *need_seek)
+        if (in->seek(*need_seek, SEEK_SET) != static_cast<off_t>(*need_seek))
         {
-            if (in->seek(*need_seek, SEEK_SET) != static_cast<off_t>(*need_seek))
-            {
-                /// Failed to seek, maybe because the new seek position is located after EOF.
-                set(in->position(), 0);
-                return false;
-            }
+            /// Failed to seek, maybe because the new seek position is located after EOF.
+            set(in->position(), 0);
+            return false;
         }
         need_seek.reset();
     }
 
-    if (in->getPosition() >= max_offset)
+    off_t seek_pos = in->getPosition();
+    off_t offset_after_min = seek_pos - min_offset;
+    off_t available_before_max = max_offset - seek_pos;
+
+    if (offset_after_min < 0 || available_before_max <= 0)
     {
         /// Limit reached.
         set(in->position(), 0);
         return false;
     }
 
-    if (in->eof())
+    if (in->eof()) /// `in->eof()` can call `in->next()`
     {
         /// EOF reached.
         set(in->position(), 0);
         return false;
     }
 
-    /// Adjust the size of the buffer (we don't allow to read more than `max_offset - min_offset`).
-    off_t size = in->buffer().size();
-    size = std::min(size, max_offset - in->getPosition());
+    /// in->eof() shouldn't change the seek position.
+    chassert(seek_pos == in->getPosition());
 
-    if (!size || (static_cast<off_t>(in->offset()) >= size))
-    {
-        /// Limit reached.
-        set(in->position(), 0);
-        return false;
-    }
+    /// Adjust the beginning and the end of the working buffer.
+    /// Because we don't want to read before `min_offset` or after `max_offset`.
+    auto * ptr = in->position();
+    auto * begin = in->buffer().begin();
+    auto * end = in->buffer().end();
+
+    if (ptr - begin > offset_after_min)
+        begin = ptr - offset_after_min;
+    if (end - ptr > available_before_max)
+        end = ptr + available_before_max;
+
+    BufferBase::set(begin, end - begin, ptr - begin);
+    chassert(position() == ptr && available());
 
-    BufferBase::set(in->buffer().begin(), size, in->offset());
     return true;
 }
 
diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py
index 53f1599a0d6..18f6ada6821 100644
--- a/tests/integration/test_backup_restore_new/test.py
+++ b/tests/integration/test_backup_restore_new/test.py
@@ -473,6 +473,29 @@ def test_incremental_backup_for_log_family():
     assert instance.query("SELECT count(), sum(x) FROM test.table2") == "102\t5081\n"
 
 
+def test_incremental_backup_append_table_def():
+    backup_name = new_backup_name()
+    create_and_fill_table()
+
+    assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
+    instance.query(f"BACKUP TABLE test.table TO {backup_name}")
+
+    instance.query("ALTER TABLE test.table MODIFY SETTING parts_to_throw_insert=100")
+
+    incremental_backup_name = new_backup_name()
+    instance.query(
+        f"BACKUP TABLE test.table TO {incremental_backup_name} SETTINGS base_backup = {backup_name}"
+    )
+
+    instance.query("DROP TABLE test.table")
+    instance.query(f"RESTORE TABLE test.table FROM {incremental_backup_name}")
+
+    assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
+    assert "parts_to_throw_insert = 100" in instance.query(
+        "SHOW CREATE TABLE test.table"
+    )
+
+
 def test_backup_not_found_or_already_exists():
     backup_name = new_backup_name()
 
diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py
index 30c7bad7b83..e42754dd2a3 100644
--- a/tests/integration/test_backup_restore_s3/test.py
+++ b/tests/integration/test_backup_restore_s3/test.py
@@ -172,3 +172,30 @@ def test_backup_to_s3_native_copy_multipart():
     assert node.contains_in_log(
         f"copyS3File: Multipart upload has completed. Bucket: root, Key: data/backups/multipart/{backup_name}/"
     )
+
+
+def test_incremental_backup_append_table_def():
+    backup_name = f"S3('http://minio1:9001/root/data/backups/{new_backup_name()}', 'minio', 'minio123')"
+
+    node.query(
+        "CREATE TABLE data (x UInt32, y String) Engine=MergeTree() ORDER BY y PARTITION BY x%10 SETTINGS storage_policy='policy_s3'"
+    )
+
+    node.query("INSERT INTO data SELECT number, toString(number) FROM numbers(100)")
+    assert node.query("SELECT count(), sum(x) FROM data") == "100\t4950\n"
+
+    node.query(f"BACKUP TABLE data TO {backup_name}")
+
+    node.query("ALTER TABLE data MODIFY SETTING parts_to_throw_insert=100")
+
+    incremental_backup_name = f"S3('http://minio1:9001/root/data/backups/{new_backup_name()}', 'minio', 'minio123')"
+
+    node.query(
+        f"BACKUP TABLE data TO {incremental_backup_name} SETTINGS base_backup = {backup_name}"
+    )
+
+    node.query("DROP TABLE data")
+    node.query(f"RESTORE TABLE data FROM {incremental_backup_name}")
+
+    assert node.query("SELECT count(), sum(x) FROM data") == "100\t4950\n"
+    assert "parts_to_throw_insert = 100" in node.query("SHOW CREATE TABLE data")

From 8ca804d40eaa37516744fc608e071f7fc2884583 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Thu, 11 May 2023 21:32:12 +0000
Subject: [PATCH 436/689] Typo: demange.cpp --> demangle.cpp

---
 src/Functions/{demange.cpp => demangle.cpp} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/Functions/{demange.cpp => demangle.cpp} (100%)

diff --git a/src/Functions/demange.cpp b/src/Functions/demangle.cpp
similarity index 100%
rename from src/Functions/demange.cpp
rename to src/Functions/demangle.cpp

From 0c0ea7bfc0ca60c3c04d516bb5ae1b7f14e67ca3 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Thu, 11 May 2023 23:07:48 +0000
Subject: [PATCH 437/689] Analyzer: apply _CAST to constants only once

---
 src/Analyzer/FunctionNode.cpp | 10 +++++++---
 tests/broken_tests.txt        |  1 -
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/Analyzer/FunctionNode.cpp b/src/Analyzer/FunctionNode.cpp
index e635750569d..2385531db4f 100644
--- a/src/Analyzer/FunctionNode.cpp
+++ b/src/Analyzer/FunctionNode.cpp
@@ -209,15 +209,19 @@ ASTPtr FunctionNode::toASTImpl(const ConvertToASTOptions & options) const
         function_ast->kind = ASTFunction::Kind::WINDOW_FUNCTION;
     }
 
+    auto new_options = options;
+    if (function_name == "_CAST")
+        new_options.add_cast_for_constants = false;
+
     const auto & parameters = getParameters();
     if (!parameters.getNodes().empty())
     {
-        function_ast->children.push_back(parameters.toAST(options));
+        function_ast->children.push_back(parameters.toAST(new_options));
         function_ast->parameters = function_ast->children.back();
     }
 
     const auto & arguments = getArguments();
-    function_ast->children.push_back(arguments.toAST(options));
+    function_ast->children.push_back(arguments.toAST(new_options));
     function_ast->arguments = function_ast->children.back();
 
     auto window_node = getWindowNode();
@@ -226,7 +230,7 @@ ASTPtr FunctionNode::toASTImpl(const ConvertToASTOptions & options) const
         if (auto * identifier_node = window_node->as<IdentifierNode>())
             function_ast->window_name = identifier_node->getIdentifier().getFullName();
         else
-            function_ast->window_definition = window_node->toAST(options);
+            function_ast->window_definition = window_node->toAST(new_options);
     }
 
     return function_ast;
diff --git a/tests/broken_tests.txt b/tests/broken_tests.txt
index f3fd0f395d4..02b539d583b 100644
--- a/tests/broken_tests.txt
+++ b/tests/broken_tests.txt
@@ -115,7 +115,6 @@
 02481_merge_array_join_sample_by
 02493_inconsistent_hex_and_binary_number
 02494_optimize_group_by_function_keys_and_alias_columns
-02511_complex_literals_as_aggregate_function_parameters
 02521_aggregation_by_partitions
 02554_fix_grouping_sets_predicate_push_down
 02575_merge_prewhere_different_default_kind

From cf2b158072b1e774c32bebbbe5f6171491361f81 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 12 May 2023 03:18:43 +0300
Subject: [PATCH 438/689] Update build-cross-riscv.md

---
 docs/en/development/build-cross-riscv.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/development/build-cross-riscv.md b/docs/en/development/build-cross-riscv.md
index a20913e7a32..e3550a046c7 100644
--- a/docs/en/development/build-cross-riscv.md
+++ b/docs/en/development/build-cross-riscv.md
@@ -11,7 +11,7 @@ This is for the case when you have Linux machine and want to use it to build `cl
 
 The cross-build for RISC-V 64 is based on the [Build instructions](../development/build.md), follow them first.
 
-## Install Clang-13
+## Install Clang-16
 
 Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup or do
 ```
@@ -23,7 +23,7 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
 ``` bash
 cd ClickHouse
 mkdir build-riscv64
-CC=clang-14 CXX=clang++-14 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF  -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DUSE_UNWIND=OFF -DENABLE_GRPC=OFF -DENABLE_HDFS=OFF -DENABLE_MYSQL=OFF
+CC=clang-16 CXX=clang++-16 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF  -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DUSE_UNWIND=OFF -DENABLE_GRPC=OFF -DENABLE_HDFS=OFF -DENABLE_MYSQL=OFF
 ninja -C build-riscv64
 ```
 

From a14ad025ee8b251be1a4a9087a1ff77d8c4faff6 Mon Sep 17 00:00:00 2001
From: DanRoscigno <dan@roscigno.com>
Date: Thu, 11 May 2023 21:33:36 -0400
Subject: [PATCH 439/689] add greatest and least docs

---
 .../aggregate-functions/reference/greatest.md | 48 +++++++++++++++++++
 .../aggregate-functions/reference/least.md    | 48 +++++++++++++++++++
 2 files changed, 96 insertions(+)
 create mode 100644 docs/en/sql-reference/aggregate-functions/reference/greatest.md
 create mode 100644 docs/en/sql-reference/aggregate-functions/reference/least.md

diff --git a/docs/en/sql-reference/aggregate-functions/reference/greatest.md b/docs/en/sql-reference/aggregate-functions/reference/greatest.md
new file mode 100644
index 00000000000..14de1fbbf16
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/greatest.md
@@ -0,0 +1,48 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/greatest
+title: greatest
+---
+
+Aggregate function that returns the greatest across a list of values.  All of the list members must be of comparable types.
+
+Examples:
+
+```sql
+SELECT
+    toTypeName(greatest(toUInt8(1), 2, toUInt8(3), 3.)),
+    greatest(1, 2, toUInt8(3), 3.)
+```
+```response
+┌─toTypeName(greatest(toUInt8(1), 2, toUInt8(3), 3.))─┬─greatest(1, 2, toUInt8(3), 3.)─┐
+│ Float64                                             │                              3 │
+└─────────────────────────────────────────────────────┴────────────────────────────────┘
+```
+
+:::note
+The type returned is a Float64 as the UInt8 must be promoted to 64 bit for the comparison.
+:::
+
+```sql
+SELECT greatest(['hello'], ['there'], ['world'])
+```
+```response
+┌─greatest(['hello'], ['there'], ['world'])─┐
+│ ['world']                                 │
+└───────────────────────────────────────────┘
+```
+
+```sql
+SELECT greatest(toDateTime32(now() + toIntervalDay(1)), toDateTime64(now(), 3))
+```
+```response
+┌─greatest(toDateTime32(plus(now(), toIntervalDay(1))), toDateTime64(now(), 3))─┐
+│                                                    2023-05-12 01:16:59.000 │
+└────────────────────────────────────────────────────────────────────────────┘
+```
+
+:::note
+The type returned is a DateTime64 as the DataTime32 must be promoted to 64 bit for the comparison.
+:::
+
+Also see [least](/docs/en/sql-reference/aggregate-functions/reference/least.md).
+
diff --git a/docs/en/sql-reference/aggregate-functions/reference/least.md b/docs/en/sql-reference/aggregate-functions/reference/least.md
new file mode 100644
index 00000000000..eb3f14a60e3
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/least.md
@@ -0,0 +1,48 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/least
+title: least
+---
+
+Aggregate function that returns the least across a list of values.  All of the list members must be of comparable types.
+
+Examples:
+
+```sql
+SELECT
+    toTypeName(least(toUInt8(1), 2, toUInt8(3), 3.)),
+    least(1, 2, toUInt8(3), 3.)
+```
+```response
+┌─toTypeName(least(toUInt8(1), 2, toUInt8(3), 3.))─┬─least(1, 2, toUInt8(3), 3.)─┐
+│ Float64                                             │                              1 │
+└─────────────────────────────────────────────────────┴────────────────────────────────┘
+```
+
+:::note
+The type returned is a Float64 as the UInt8 must be promoted to 64 bit for the comparison.
+:::
+
+```sql
+SELECT least(['hello'], ['there'], ['world'])
+```
+```response
+┌─least(['hello'], ['there'], ['world'])─┐
+│ ['hello']                                 │
+└───────────────────────────────────────────┘
+```
+
+```sql
+SELECT least(toDateTime32(now() + toIntervalDay(1)), toDateTime64(now(), 3))
+```
+```response
+┌─least(toDateTime32(plus(now(), toIntervalDay(1))), toDateTime64(now(), 3))─┐
+│                                                    2023-05-12 01:16:59.000 │
+└────────────────────────────────────────────────────────────────────────────┘
+```
+
+:::note
+The type returned is a DateTime64 as the DataTime32 must be promoted to 64 bit for the comparison.
+:::
+
+Also see [greatest](/docs/en/sql-reference/aggregate-functions/reference/greatest.md).
+

From 11f78fed2babf1cc66b0aa6d47f737cf8b4ab95d Mon Sep 17 00:00:00 2001
From: DanRoscigno <dan@roscigno.com>
Date: Thu, 11 May 2023 21:40:47 -0400
Subject: [PATCH 440/689] fix markdown spaces

---
 .../aggregate-functions/reference/greatest.md             | 4 ++--
 .../sql-reference/aggregate-functions/reference/least.md  | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/greatest.md b/docs/en/sql-reference/aggregate-functions/reference/greatest.md
index 14de1fbbf16..d5efea44790 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/greatest.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/greatest.md
@@ -36,8 +36,8 @@ SELECT greatest(toDateTime32(now() + toIntervalDay(1)), toDateTime64(now(), 3))
 ```
 ```response
 ┌─greatest(toDateTime32(plus(now(), toIntervalDay(1))), toDateTime64(now(), 3))─┐
-│                                                    2023-05-12 01:16:59.000 │
-└────────────────────────────────────────────────────────────────────────────┘
+│                                                       2023-05-12 01:16:59.000 │
+└──---──────────────────────────────────────────────────────────────────────────┘
 ```
 
 :::note
diff --git a/docs/en/sql-reference/aggregate-functions/reference/least.md b/docs/en/sql-reference/aggregate-functions/reference/least.md
index eb3f14a60e3..ae4b1d43182 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/least.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/least.md
@@ -14,8 +14,8 @@ SELECT
 ```
 ```response
 ┌─toTypeName(least(toUInt8(1), 2, toUInt8(3), 3.))─┬─least(1, 2, toUInt8(3), 3.)─┐
-│ Float64                                             │                              1 │
-└─────────────────────────────────────────────────────┴────────────────────────────────┘
+│ Float64                                          │                           1 │
+└──────────────────────────────────────────────────┴─────────────────────────────┘
 ```
 
 :::note
@@ -27,8 +27,8 @@ SELECT least(['hello'], ['there'], ['world'])
 ```
 ```response
 ┌─least(['hello'], ['there'], ['world'])─┐
-│ ['hello']                                 │
-└───────────────────────────────────────────┘
+│ ['hello']                              │
+└────────────────────────────────────────┘
 ```
 
 ```sql

From 5bb1db0fa8970f87481a694f29ef4cd03fcea235 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 12 May 2023 05:19:01 +0200
Subject: [PATCH 441/689] Allow loading dicrionaries and functions from YAML by
 default

---
 programs/server/config.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/server/config.xml b/programs/server/config.xml
index 51aa04ba0e5..82dd697084c 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -1294,10 +1294,10 @@
     <!-- Configuration of external dictionaries. See:
          https://clickhouse.com/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts
     -->
-    <dictionaries_config>*_dictionary.xml</dictionaries_config>
+    <dictionaries_config>*_dictionary.*ml</dictionaries_config>
 
     <!-- Configuration of user defined executable functions -->
-    <user_defined_executable_functions_config>*_function.xml</user_defined_executable_functions_config>
+    <user_defined_executable_functions_config>*_function.*ml</user_defined_executable_functions_config>
 
     <!-- Path in ZooKeeper to store user-defined SQL functions created by the command CREATE FUNCTION.
      If not specified they will be stored locally. -->

From 2c40dd6a4c6aaa956762b8709fe5e4686e16876c Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 2 May 2023 14:36:02 +0200
Subject: [PATCH 442/689] Switch Block::NameMap to google::dense_hash_map over
 HashMap

Since HashMap creates 2^8 elements by default, while dense_hash_map
should be good here.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Access/AuthenticationData.cpp                   |  1 +
 src/AggregateFunctions/AggregateFunctionForEach.h   |  1 +
 src/AggregateFunctions/AggregateFunctionMinMaxAny.h |  1 +
 src/CMakeLists.txt                                  |  3 ++-
 src/Core/Block.cpp                                  |  6 ++----
 src/Core/Block.h                                    |  6 ++----
 .../SerializationAggregateFunction.cpp              |  1 +
 src/Formats/ColumnMapping.cpp                       |  6 +++---
 src/Functions/FunctionBinaryArithmetic.h            |  1 +
 src/Interpreters/QueryLog.h                         |  1 +
 src/Interpreters/QueryViewsLog.h                    |  1 +
 src/Interpreters/tests/gtest_context_race.cpp       |  1 +
 .../Formats/Impl/BSONEachRowRowInputFormat.cpp      | 12 +++++++-----
 .../Formats/Impl/BSONEachRowRowInputFormat.h        |  2 +-
 .../Impl/JSONColumnsBlockInputFormatBase.cpp        |  2 +-
 .../Formats/Impl/JSONEachRowRowInputFormat.cpp      | 13 ++++++-------
 .../Formats/Impl/JSONEachRowRowInputFormat.h        |  5 ++---
 .../QueryPlan/DistributedCreateLocalPlan.h          |  1 +
 src/Storages/DataLakes/DeltaLakeMetadataParser.cpp  |  5 +++--
 src/Storages/DataLakes/HudiMetadataParser.cpp       |  1 +
 src/Storages/DataLakes/IcebergMetadataParser.cpp    |  1 +
 src/Storages/StorageS3.h                            |  1 +
 22 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/src/Access/AuthenticationData.cpp b/src/Access/AuthenticationData.cpp
index 409338209cc..3bb0be160f4 100644
--- a/src/Access/AuthenticationData.cpp
+++ b/src/Access/AuthenticationData.cpp
@@ -10,6 +10,7 @@
 #include <Common/OpenSSLHelpers.h>
 #include <Poco/SHA1Engine.h>
 #include <base/types.h>
+#include <base/hex.h>
 #include <boost/algorithm/hex.hpp>
 #include <boost/algorithm/string/case_conv.hpp>
 
diff --git a/src/AggregateFunctions/AggregateFunctionForEach.h b/src/AggregateFunctions/AggregateFunctionForEach.h
index 81ba298bb8a..480b4cc690e 100644
--- a/src/AggregateFunctions/AggregateFunctionForEach.h
+++ b/src/AggregateFunctions/AggregateFunctionForEach.h
@@ -2,6 +2,7 @@
 
 #include <Columns/ColumnArray.h>
 #include <Common/assert_cast.h>
+#include <Common/Arena.h>
 #include <base/arithmeticOverflow.h>
 #include <DataTypes/DataTypeArray.h>
 #include <AggregateFunctions/IAggregateFunction.h>
diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
index c2cf2ac418f..94c0d60be81 100644
--- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
+++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
@@ -10,6 +10,7 @@
 #include <DataTypes/IDataType.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <base/StringRef.h>
+#include <Common/Arena.h>
 #include <Common/assert_cast.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <AggregateFunctions/IAggregateFunction.h>
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index b3f4fbb7420..c04b7acad3d 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -211,6 +211,7 @@ endif()
 if (TARGET ch_contrib::jemalloc)
     target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::jemalloc)
 endif()
+target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::sparsehash)
 
 add_subdirectory(Access/Common)
 add_subdirectory(Common/ZooKeeper)
@@ -463,7 +464,7 @@ endif ()
 if (TARGET ch_contrib::ldap)
     dbms_target_link_libraries (PRIVATE ch_contrib::ldap ch_contrib::lber)
 endif ()
-dbms_target_link_libraries (PRIVATE ch_contrib::sparsehash)
+dbms_target_link_libraries (PUBLIC ch_contrib::sparsehash)
 
 if (TARGET ch_contrib::protobuf)
     dbms_target_link_libraries (PRIVATE ch_contrib::protobuf)
diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp
index 456f1d5d95e..a3bd29faab1 100644
--- a/src/Core/Block.cpp
+++ b/src/Core/Block.cpp
@@ -663,12 +663,10 @@ Names Block::getDataTypeNames() const
 
 Block::NameMap Block::getNamesToIndexesMap() const
 {
-    NameMap res;
-    res.reserve(index_by_name.size());
-
+    NameMap res(index_by_name.size());
+    res.set_empty_key(StringRef{});
     for (const auto & [name, index] : index_by_name)
         res[name] = index;
-
     return res;
 }
 
diff --git a/src/Core/Block.h b/src/Core/Block.h
index eb9d57ea6f8..7eed48d3d9f 100644
--- a/src/Core/Block.h
+++ b/src/Core/Block.h
@@ -5,13 +5,11 @@
 #include <Core/ColumnsWithTypeAndName.h>
 #include <Core/NamesAndTypes.h>
 
-#include <Common/HashTable/HashMap.h>
-
 #include <initializer_list>
 #include <list>
-#include <map>
 #include <set>
 #include <vector>
+#include <sparsehash/dense_hash_map>
 
 
 namespace DB
@@ -97,7 +95,7 @@ public:
     Names getDataTypeNames() const;
 
     /// Hash table match `column name -> position in the block`.
-    using NameMap = HashMap<StringRef, size_t, StringRefHash>;
+    using NameMap = ::google::dense_hash_map<StringRef, size_t, StringRefHash>;
     NameMap getNamesToIndexesMap() const;
 
     Serializations getSerializations() const;
diff --git a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp
index 7e192595114..c482c9623e9 100644
--- a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp
+++ b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp
@@ -7,6 +7,7 @@
 #include <Common/typeid_cast.h>
 #include <Common/assert_cast.h>
 #include <Common/AlignedBuffer.h>
+#include <Common/Arena.h>
 
 #include <Formats/FormatSettings.h>
 #include <Formats/ProtobufReader.h>
diff --git a/src/Formats/ColumnMapping.cpp b/src/Formats/ColumnMapping.cpp
index 2fab5766591..e33dfc878f4 100644
--- a/src/Formats/ColumnMapping.cpp
+++ b/src/Formats/ColumnMapping.cpp
@@ -26,8 +26,8 @@ void ColumnMapping::addColumns(
     {
         names_of_columns.push_back(name);
 
-        const auto * column_it = column_indexes_by_names.find(name);
-        if (!column_it)
+        const auto column_it = column_indexes_by_names.find(name);
+        if (column_it == column_indexes_by_names.end())
         {
             if (settings.skip_unknown_fields)
             {
@@ -43,7 +43,7 @@ void ColumnMapping::addColumns(
                             name, column_indexes_for_input_fields.size());
         }
 
-        const auto column_index = column_it->getMapped();
+        const auto column_index = column_it->second;
 
         if (read_columns[column_index])
             throw Exception(ErrorCodes::INCORRECT_DATA, "Duplicate field found while parsing format header: {}", name);
diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h
index a83bc6382b6..0174899d767 100644
--- a/src/Functions/FunctionBinaryArithmetic.h
+++ b/src/Functions/FunctionBinaryArithmetic.h
@@ -41,6 +41,7 @@
 #include <Common/FieldVisitorsAccurateComparison.h>
 #include <Common/assert_cast.h>
 #include <Common/typeid_cast.h>
+#include <Common/Arena.h>
 #include <DataTypes/DataTypeLowCardinality.h>
 #include <Interpreters/Context.h>
 
diff --git a/src/Interpreters/QueryLog.h b/src/Interpreters/QueryLog.h
index a2d434e8843..44780f530e0 100644
--- a/src/Interpreters/QueryLog.h
+++ b/src/Interpreters/QueryLog.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <Common/ProfileEvents.h>
 #include <Core/NamesAndTypes.h>
 #include <Core/NamesAndAliases.h>
 #include <Core/Settings.h>
diff --git a/src/Interpreters/QueryViewsLog.h b/src/Interpreters/QueryViewsLog.h
index 986311fc822..e28bce0b91c 100644
--- a/src/Interpreters/QueryViewsLog.h
+++ b/src/Interpreters/QueryViewsLog.h
@@ -13,6 +13,7 @@
 #include <Core/NamesAndAliases.h>
 #include <Interpreters/SystemLog.h>
 #include <base/types.h>
+#include <Common/ProfileEvents.h>
 
 namespace ProfileEvents
 {
diff --git a/src/Interpreters/tests/gtest_context_race.cpp b/src/Interpreters/tests/gtest_context_race.cpp
index 60531494592..ec61fc9467c 100644
--- a/src/Interpreters/tests/gtest_context_race.cpp
+++ b/src/Interpreters/tests/gtest_context_race.cpp
@@ -1,6 +1,7 @@
 #include <Interpreters/Context.h>
 #include <Common/tests/gtest_global_context.h>
 #include <gtest/gtest.h>
+#include <thread>
 
 using namespace DB;
 
diff --git a/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.cpp
index 370cddd2dcb..57598fb507f 100644
--- a/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.cpp
+++ b/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.cpp
@@ -64,20 +64,22 @@ inline size_t BSONEachRowRowInputFormat::columnIndex(const StringRef & name, siz
     /// Optimization by caching the order of fields (which is almost always the same)
     /// and a quick check to match the next expected field, instead of searching the hash table.
 
-    if (prev_positions.size() > key_index && prev_positions[key_index] && name == prev_positions[key_index]->getKey())
+    if (prev_positions.size() > key_index
+        && prev_positions[key_index] != Block::NameMap::const_iterator{}
+        && name == prev_positions[key_index]->first)
     {
-        return prev_positions[key_index]->getMapped();
+        return prev_positions[key_index]->second;
     }
     else
     {
-        auto * it = name_map.find(name);
+        const auto it = name_map.find(name);
 
-        if (it)
+        if (it != name_map.end())
         {
             if (key_index < prev_positions.size())
                 prev_positions[key_index] = it;
 
-            return it->getMapped();
+            return it->second;
         }
         else
             return UNKNOWN_FIELD;
diff --git a/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.h b/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.h
index ad6d712b6dd..538a59e05c3 100644
--- a/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.h
+++ b/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.h
@@ -91,7 +91,7 @@ private:
     Block::NameMap name_map;
 
     /// Cached search results for previous row (keyed as index in JSON object) - used as a hint.
-    std::vector<Block::NameMap::LookupResult> prev_positions;
+    std::vector<Block::NameMap::const_iterator> prev_positions;
 
     DataTypes types;
 
diff --git a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.cpp b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.cpp
index 3d052e23c21..2e264c59f56 100644
--- a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.cpp
+++ b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.cpp
@@ -128,7 +128,7 @@ Chunk JSONColumnsBlockInputFormatBase::generate()
         {
             /// Check if this name appears in header. If no, skip this column or throw
             /// an exception according to setting input_format_skip_unknown_fields
-            if (!name_to_index.has(*column_name))
+            if (name_to_index.find(*column_name) == name_to_index.end())
             {
                 if (!format_settings.skip_unknown_fields)
                     throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown column found in input data: {}", *column_name);
diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
index 22ac31c7824..e5f52936021 100644
--- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
+++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
@@ -71,21 +71,20 @@ inline size_t JSONEachRowRowInputFormat::columnIndex(StringRef name, size_t key_
     /// and a quick check to match the next expected field, instead of searching the hash table.
 
     if (prev_positions.size() > key_index
-        && prev_positions[key_index]
-        && name == prev_positions[key_index]->getKey())
+        && prev_positions[key_index] != Block::NameMap::const_iterator{}
+        && name == prev_positions[key_index]->first)
     {
-        return prev_positions[key_index]->getMapped();
+        return prev_positions[key_index]->second;
     }
     else
     {
-        auto * it = name_map.find(name);
-
-        if (it)
+        const auto it = name_map.find(name);
+        if (it != name_map.end())
         {
             if (key_index < prev_positions.size())
                 prev_positions[key_index] = it;
 
-            return it->getMapped();
+            return it->second;
         }
         else
             return UNKNOWN_FIELD;
diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h
index beee9e95821..ce42071585e 100644
--- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h
+++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h
@@ -71,11 +71,10 @@ private:
     /// for row like {..., "non-nullable column name" : null, ...}
 
     /// Hash table match `field name -> position in the block`. NOTE You can use perfect hash map.
-    using NameMap = HashMap<StringRef, size_t, StringRefHash>;
-    NameMap name_map;
+    Block::NameMap name_map;
 
     /// Cached search results for previous row (keyed as index in JSON object) - used as a hint.
-    std::vector<NameMap::LookupResult> prev_positions;
+    std::vector<Block::NameMap::const_iterator> prev_positions;
 
     bool allow_new_rows = true;
 
diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.h b/src/Processors/QueryPlan/DistributedCreateLocalPlan.h
index 16bf1c565ff..1afdc07fa4d 100644
--- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.h
+++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Core/QueryProcessingStage.h>
+#include <Core/UUID.h>
 #include <Parsers/IAST_fwd.h>
 #include <Processors/QueryPlan/QueryPlan.h>
 #include <Processors/ResizeProcessor.h>
diff --git a/src/Storages/DataLakes/DeltaLakeMetadataParser.cpp b/src/Storages/DataLakes/DeltaLakeMetadataParser.cpp
index 45189ca325a..309aa54909a 100644
--- a/src/Storages/DataLakes/DeltaLakeMetadataParser.cpp
+++ b/src/Storages/DataLakes/DeltaLakeMetadataParser.cpp
@@ -11,10 +11,11 @@
 #include <Processors/Formats/Impl/ParquetBlockInputFormat.h>
 #include <Processors/Formats/Impl/ArrowColumnToCHColumn.h>
 #include <Formats/FormatFactory.h>
-#include <boost/algorithm/string/case_conv.hpp>
-#include <parquet/arrow/reader.h>
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnNullable.h>
+#include <IO/ReadHelpers.h>
+#include <boost/algorithm/string/case_conv.hpp>
+#include <parquet/arrow/reader.h>
 #include <ranges>
 
 namespace fs = std::filesystem;
diff --git a/src/Storages/DataLakes/HudiMetadataParser.cpp b/src/Storages/DataLakes/HudiMetadataParser.cpp
index 74e556f2bf2..a1f35a5ae42 100644
--- a/src/Storages/DataLakes/HudiMetadataParser.cpp
+++ b/src/Storages/DataLakes/HudiMetadataParser.cpp
@@ -1,6 +1,7 @@
 #include <Storages/DataLakes/HudiMetadataParser.h>
 #include <Common/logger_useful.h>
 #include <ranges>
+#include <base/find_symbols.h>
 #include <Poco/String.h>
 #include "config.h"
 #include <filesystem>
diff --git a/src/Storages/DataLakes/IcebergMetadataParser.cpp b/src/Storages/DataLakes/IcebergMetadataParser.cpp
index 4e90da41721..3ab90e271cf 100644
--- a/src/Storages/DataLakes/IcebergMetadataParser.cpp
+++ b/src/Storages/DataLakes/IcebergMetadataParser.cpp
@@ -12,6 +12,7 @@
 #include <Storages/StorageS3.h>
 #include <Processors/Formats/Impl/AvroRowInputFormat.h>
 #include <Formats/FormatFactory.h>
+#include <IO/ReadHelpers.h>
 
 #include <Poco/JSON/Array.h>
 #include <Poco/JSON/Object.h>
diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h
index c3b862f6bbd..12573ab513f 100644
--- a/src/Storages/StorageS3.h
+++ b/src/Storages/StorageS3.h
@@ -16,6 +16,7 @@
 #include <Poco/URI.h>
 #include <IO/S3/getObjectInfo.h>
 #include <IO/CompressionMethod.h>
+#include <IO/SeekableReadBuffer.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/threadPoolCallbackRunner.h>
 #include <Storages/Cache/SchemaCache.h>

From 9c42f7f113d670bc891145b5bb827f983ea73066 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 11 May 2023 12:46:16 +0200
Subject: [PATCH 443/689] Fix settings aliases in native protocol

The initial test (tests/queries/0_stateless/02539_settings_alias.sh)
works only because of the clickhouse-client, while in native protocol
aliases does not work.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Core/BaseSettings.h                       |   6 +-
 .../02750_settings_alias_tcp_protocol.python  | 218 ++++++++++++++++++
 ...2750_settings_alias_tcp_protocol.reference |   1 +
 .../02750_settings_alias_tcp_protocol.sh      |   9 +
 4 files changed, 232 insertions(+), 2 deletions(-)
 create mode 100644 tests/queries/0_stateless/02750_settings_alias_tcp_protocol.python
 create mode 100644 tests/queries/0_stateless/02750_settings_alias_tcp_protocol.reference
 create mode 100755 tests/queries/0_stateless/02750_settings_alias_tcp_protocol.sh

diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h
index 521422b780e..a14cec9cc7d 100644
--- a/src/Core/BaseSettings.h
+++ b/src/Core/BaseSettings.h
@@ -501,9 +501,11 @@ void BaseSettings<TTraits>::read(ReadBuffer & in, SettingsWriteFormat format)
     const auto & accessor = Traits::Accessor::instance();
     while (true)
     {
-        String name = BaseSettingsHelpers::readString(in);
-        if (name.empty() /* empty string is a marker of the end of settings */)
+        String read_name = BaseSettingsHelpers::readString(in);
+        if (read_name.empty() /* empty string is a marker of the end of settings */)
             break;
+
+        std::string_view name = TTraits::resolveName(read_name);
         size_t index = accessor.find(name);
 
         using Flags = BaseSettingsHelpers::Flags;
diff --git a/tests/queries/0_stateless/02750_settings_alias_tcp_protocol.python b/tests/queries/0_stateless/02750_settings_alias_tcp_protocol.python
new file mode 100644
index 00000000000..768fb2144e3
--- /dev/null
+++ b/tests/queries/0_stateless/02750_settings_alias_tcp_protocol.python
@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+
+import socket
+import os
+import uuid
+import json
+
+CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1")
+CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "900000"))
+CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default")
+
+
+def writeVarUInt(x, ba):
+    for _ in range(0, 9):
+        byte = x & 0x7F
+        if x > 0x7F:
+            byte |= 0x80
+
+        ba.append(byte)
+
+        x >>= 7
+        if x == 0:
+            return
+
+
+def writeStringBinary(s, ba):
+    b = bytes(s, "utf-8")
+    writeVarUInt(len(s), ba)
+    ba.extend(b)
+
+
+def readStrict(s, size=1):
+    res = bytearray()
+    while size:
+        cur = s.recv(size)
+        # if not res:
+        #     raise "Socket is closed"
+        size -= len(cur)
+        res.extend(cur)
+
+    return res
+
+
+def readUInt(s, size=1):
+    res = readStrict(s, size)
+    val = 0
+    for i in range(len(res)):
+        val += res[i] << (i * 8)
+    return val
+
+
+def readUInt8(s):
+    return readUInt(s)
+
+
+def readUInt16(s):
+    return readUInt(s, 2)
+
+
+def readUInt32(s):
+    return readUInt(s, 4)
+
+
+def readUInt64(s):
+    return readUInt(s, 8)
+
+
+def readVarUInt(s):
+    x = 0
+    for i in range(9):
+        byte = readStrict(s)[0]
+        x |= (byte & 0x7F) << (7 * i)
+
+        if not byte & 0x80:
+            return x
+
+    return x
+
+
+def readStringBinary(s):
+    size = readVarUInt(s)
+    s = readStrict(s, size)
+    return s.decode("utf-8")
+
+
+def sendHello(s):
+    ba = bytearray()
+    writeVarUInt(0, ba)  # Hello
+    writeStringBinary("simple native protocol", ba)
+    writeVarUInt(21, ba)
+    writeVarUInt(9, ba)
+    writeVarUInt(54449, ba)
+    writeStringBinary(CLICKHOUSE_DATABASE, ba)  # database
+    writeStringBinary("default", ba)  # user
+    writeStringBinary("", ba)  # pwd
+    s.sendall(ba)
+
+
+def receiveHello(s):
+    p_type = readVarUInt(s)
+    assert p_type == 0  # Hello
+    _server_name = readStringBinary(s)
+    _server_version_major = readVarUInt(s)
+    _server_version_minor = readVarUInt(s)
+    _server_revision = readVarUInt(s)
+    _server_timezone = readStringBinary(s)
+    _server_display_name = readStringBinary(s)
+    _server_version_patch = readVarUInt(s)
+
+
+def serializeClientInfo(ba, query_id):
+    writeStringBinary("default", ba)  # initial_user
+    writeStringBinary(query_id, ba)  # initial_query_id
+    writeStringBinary("127.0.0.1:9000", ba)  # initial_address
+    ba.extend([0] * 8)  # initial_query_start_time_microseconds
+    ba.append(1)  # TCP
+    writeStringBinary("os_user", ba)  # os_user
+    writeStringBinary("client_hostname", ba)  # client_hostname
+    writeStringBinary("client_name", ba)  # client_name
+    writeVarUInt(21, ba)
+    writeVarUInt(9, ba)
+    writeVarUInt(54449, ba)
+    writeStringBinary("", ba)  # quota_key
+    writeVarUInt(0, ba)  # distributed_depth
+    writeVarUInt(1, ba)  # client_version_patch
+    ba.append(0)  # No telemetry
+
+
+def sendQuery(s, query, settings):
+    ba = bytearray()
+    query_id = uuid.uuid4().hex
+    writeVarUInt(1, ba)  # query
+    writeStringBinary(query_id, ba)
+
+    ba.append(1)  # INITIAL_QUERY
+
+    # client info
+    serializeClientInfo(ba, query_id)
+
+    # Settings
+    for key, value in settings.items():
+        writeStringBinary(key, ba)
+        writeVarUInt(1, ba)  # is_important
+        writeStringBinary(str(value), ba)
+    writeStringBinary("", ba)  # End of settings
+
+    writeStringBinary("", ba)  # No interserver secret
+    writeVarUInt(2, ba)  # Stage - Complete
+    ba.append(0)  # No compression
+    writeStringBinary(query, ba)  # query, finally
+    s.sendall(ba)
+
+
+def serializeBlockInfo(ba):
+    writeVarUInt(1, ba)  # 1
+    ba.append(0)  # is_overflows
+    writeVarUInt(2, ba)  # 2
+    writeVarUInt(0, ba)  # 0
+    ba.extend([0] * 4)  # bucket_num
+
+
+def sendEmptyBlock(s):
+    ba = bytearray()
+    writeVarUInt(2, ba)  # Data
+    writeStringBinary("", ba)
+    serializeBlockInfo(ba)
+    writeVarUInt(0, ba)  # rows
+    writeVarUInt(0, ba)  # columns
+    s.sendall(ba)
+
+
+def assertPacket(packet, expected):
+    assert packet == expected, "Got: {}, expected: {}".format(packet, expected)
+
+
+def readResponse(s):
+    packet_type = readVarUInt(s)
+    if packet_type == 2:  # Exception
+        raise RuntimeError(readException(s))
+
+    if packet_type == 1:  # Data
+        return None
+    if packet_type == 3:  # Progress
+        return None
+    if packet_type == 5:  # End stream
+        return None
+
+    raise RuntimeError("Unexpected packet: {}".format(packet_type))
+
+
+def readException(s):
+    code = readUInt32(s)
+    _name = readStringBinary(s)
+    text = readStringBinary(s)
+    readStringBinary(s)  # trace
+    assertPacket(readUInt8(s), 0)  # has_nested
+    return "code {}: {}".format(code, text.replace("DB::Exception:", ""))
+
+
+def main():
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.settimeout(30)
+        s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT))
+        sendHello(s)
+        receiveHello(s)
+        sendQuery(s, "select 1", {"replication_alter_partitions_sync": 1})
+        # external tables
+        sendEmptyBlock(s)
+
+        while readResponse(s) is not None:
+            pass
+
+        s.close()
+    print("OK")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/queries/0_stateless/02750_settings_alias_tcp_protocol.reference b/tests/queries/0_stateless/02750_settings_alias_tcp_protocol.reference
new file mode 100644
index 00000000000..d86bac9de59
--- /dev/null
+++ b/tests/queries/0_stateless/02750_settings_alias_tcp_protocol.reference
@@ -0,0 +1 @@
+OK
diff --git a/tests/queries/0_stateless/02750_settings_alias_tcp_protocol.sh b/tests/queries/0_stateless/02750_settings_alias_tcp_protocol.sh
new file mode 100755
index 00000000000..35d685c1580
--- /dev/null
+++ b/tests/queries/0_stateless/02750_settings_alias_tcp_protocol.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# NOTE: this sh wrapper is required because of shell_config
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+python3 "$CURDIR"/02750_settings_alias_tcp_protocol.python

From fa2074e67b42f56564aa23f0b0a3236aa632d16b Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Fri, 12 May 2023 10:22:43 +0200
Subject: [PATCH 444/689] Use CLOCK_MONOTONIC_RAW over CLOCK_MONOTONIC on Linux
 (fixes non monotonic clock)

There is no bug in Linux, the issue that CLOCK_MONOTONIC returns values
less then previous calls likely happens due to adjtime(3) (NTP), since
CLOCK_MONOTONIC is affected by it, and I've seen lots of slight time
modifications due to NTP on the servers. And even on my desktop (I also
have NTP enabled):

    CLOCK_MONOTONIC:     189292.803 (2 days +  4h 34m 52s)
    CLOCK_MONOTONIC_RAW: 189290.016 (2 days +  4h 34m 50s)

However on Linux there is CLOCK_MONOTONIC_RAW, it is similar to
CLOCK_MONOTONIC, but does not affected by the adjtime(3).

About performance, it is the same:

    CLOCK_MONOTONIC 10e6:     real=0m0.191s user=0m0.190s sys=0m0.000s
    CLOCK_MONOTONIC_RAW 10e6: real=0m0.191s user=0m0.191s sys=0m0.000s

Ops/s:
- AMD Threadripper: 52.3e6
- Xeon Silver 4216 2.10: 46.5e6

Fixes: c5d631ca548b93e6
Fixes: #29811 (cc @tavplubix)
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Common/Stopwatch.h | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/Common/Stopwatch.h b/src/Common/Stopwatch.h
index 6b59340f05d..d6864a476e1 100644
--- a/src/Common/Stopwatch.h
+++ b/src/Common/Stopwatch.h
@@ -8,19 +8,34 @@
 #include <atomic>
 #include <memory>
 
+/// From clock_getres(2):
+///
+///    Similar to CLOCK_MONOTONIC, but provides access to a raw hardware-based
+///    time that is not subject to NTP adjustments or the incremental
+///    adjustments performed by adjtime(3).
+#ifdef CLOCK_MONOTONIC_RAW
+static constexpr clockid_t STOPWATCH_DEFAULT_CLOCK = CLOCK_MONOTONIC_RAW;
+#else
+static constexpr clockid_t STOPWATCH_DEFAULT_CLOCK = CLOCK_MONOTONIC;
+#endif
 
-inline UInt64 clock_gettime_ns(clockid_t clock_type = CLOCK_MONOTONIC)
+inline UInt64 clock_gettime_ns(clockid_t clock_type = STOPWATCH_DEFAULT_CLOCK)
 {
     struct timespec ts;
     clock_gettime(clock_type, &ts);
     return UInt64(ts.tv_sec * 1000000000LL + ts.tv_nsec);
 }
 
-/// Sometimes monotonic clock may not be monotonic (due to bug in kernel?).
-/// It may cause some operations to fail with "Timeout exceeded: elapsed 18446744073.709553 seconds".
 /// Takes previously returned value and returns it again if time stepped back for some reason.
-inline UInt64 clock_gettime_ns_adjusted(UInt64 prev_time, clockid_t clock_type = CLOCK_MONOTONIC)
+///
+/// You should use this if OS does not support CLOCK_MONOTONIC_RAW
+inline UInt64 clock_gettime_ns_adjusted(UInt64 prev_time, clockid_t clock_type = STOPWATCH_DEFAULT_CLOCK)
 {
+#ifdef CLOCK_MONOTONIC_RAW
+    if (likely(clock_type == CLOCK_MONOTONIC_RAW))
+        return clock_gettime_ns(clock_type);
+#endif
+
     UInt64 current_time = clock_gettime_ns(clock_type);
     if (likely(prev_time <= current_time))
         return current_time;
@@ -36,10 +51,10 @@ inline UInt64 clock_gettime_ns_adjusted(UInt64 prev_time, clockid_t clock_type =
 class Stopwatch
 {
 public:
-    /** CLOCK_MONOTONIC works relatively efficient (~15 million calls/sec) and doesn't lead to syscall.
+    /** CLOCK_MONOTONIC/CLOCK_MONOTONIC_RAW works relatively efficient (~40-50 million calls/sec) and doesn't lead to syscall.
       * Pass CLOCK_MONOTONIC_COARSE, if you need better performance with acceptable cost of several milliseconds of inaccuracy.
       */
-    explicit Stopwatch(clockid_t clock_type_ = CLOCK_MONOTONIC) : clock_type(clock_type_) { start(); }
+    explicit Stopwatch(clockid_t clock_type_ = STOPWATCH_DEFAULT_CLOCK) : clock_type(clock_type_) { start(); }
     explicit Stopwatch(clockid_t clock_type_, UInt64 start_nanoseconds, bool is_running_)
         : start_ns(start_nanoseconds), clock_type(clock_type_), is_running(is_running_)
     {
@@ -75,7 +90,7 @@ using StopwatchUniquePtr = std::unique_ptr<Stopwatch>;
 class AtomicStopwatch
 {
 public:
-    explicit AtomicStopwatch(clockid_t clock_type_ = CLOCK_MONOTONIC) : clock_type(clock_type_) { restart(); }
+    explicit AtomicStopwatch(clockid_t clock_type_ = STOPWATCH_DEFAULT_CLOCK) : clock_type(clock_type_) { restart(); }
 
     void restart()                     { start_ns = nanoseconds(0); }
     UInt64 elapsed() const

From eb62030fa425af8ef176e7f7f333b8e34fa4dc4f Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Wed, 10 May 2023 05:02:52 +0200
Subject: [PATCH 445/689] Fix assigning a setting to NULL in settings profile's
 definition.

---
 src/Access/SettingsConstraints.cpp            | 12 ++++----
 src/Access/SettingsProfileElement.cpp         | 30 +++++++++----------
 src/Access/SettingsProfileElement.h           |  6 ++--
 .../Access/ASTSettingsProfileElement.cpp      | 12 ++++----
 .../Access/ASTSettingsProfileElement.h        |  6 ++--
 .../Access/ParserSettingsProfileElement.cpp   | 24 +++++++--------
 .../StorageSystemSettingsProfileElements.cpp  | 12 ++++----
 7 files changed, 51 insertions(+), 51 deletions(-)

diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp
index e83ab264f4f..12f584cab83 100644
--- a/src/Access/SettingsConstraints.cpp
+++ b/src/Access/SettingsConstraints.cpp
@@ -105,21 +105,21 @@ void SettingsConstraints::check(const Settings & current_settings, const Setting
         if (SettingsProfileElements::isAllowBackupSetting(element.setting_name))
             continue;
 
-        if (!element.value.isNull())
+        if (element.value)
         {
-            SettingChange value(element.setting_name, element.value);
+            SettingChange value(element.setting_name, *element.value);
             check(current_settings, value);
         }
 
-        if (!element.min_value.isNull())
+        if (element.min_value)
         {
-            SettingChange value(element.setting_name, element.min_value);
+            SettingChange value(element.setting_name, *element.min_value);
             check(current_settings, value);
         }
 
-        if (!element.max_value.isNull())
+        if (element.max_value)
         {
-            SettingChange value(element.setting_name, element.max_value);
+            SettingChange value(element.setting_name, *element.max_value);
             check(current_settings, value);
         }
 
diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp
index ce56782d887..9358391cb93 100644
--- a/src/Access/SettingsProfileElement.cpp
+++ b/src/Access/SettingsProfileElement.cpp
@@ -63,18 +63,18 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A
         max_value = ast.max_value;
         writability = ast.writability;
 
-        if (!value.isNull())
-            value = Settings::castValueUtil(setting_name, value);
-        if (!min_value.isNull())
-            min_value = Settings::castValueUtil(setting_name, min_value);
-        if (!max_value.isNull())
-            max_value = Settings::castValueUtil(setting_name, max_value);
+        if (value)
+            value = Settings::castValueUtil(setting_name, *value);
+        if (min_value)
+            min_value = Settings::castValueUtil(setting_name, *min_value);
+        if (max_value)
+            max_value = Settings::castValueUtil(setting_name, *max_value);
     }
 }
 
 bool SettingsProfileElement::isConstraint() const
 {
-    return this->writability || !this->min_value.isNull() || !this->max_value.isNull();
+    return this->writability || this->min_value || this->max_value;
 }
 
 std::shared_ptr<ASTSettingsProfileElement> SettingsProfileElement::toAST() const
@@ -187,8 +187,8 @@ Settings SettingsProfileElements::toSettings() const
     Settings res;
     for (const auto & elem : *this)
     {
-        if (!elem.setting_name.empty() && !isAllowBackupSetting(elem.setting_name) && !elem.value.isNull())
-            res.set(elem.setting_name, elem.value);
+        if (!elem.setting_name.empty() && !isAllowBackupSetting(elem.setting_name) && elem.value)
+            res.set(elem.setting_name, *elem.value);
     }
     return res;
 }
@@ -200,8 +200,8 @@ SettingsChanges SettingsProfileElements::toSettingsChanges() const
     {
         if (!elem.setting_name.empty() && !isAllowBackupSetting(elem.setting_name))
         {
-            if (!elem.value.isNull())
-                res.push_back({elem.setting_name, elem.value});
+            if (elem.value)
+                res.push_back({elem.setting_name, *elem.value});
         }
     }
     return res;
@@ -214,8 +214,8 @@ SettingsConstraints SettingsProfileElements::toSettingsConstraints(const AccessC
         if (!elem.setting_name.empty() && elem.isConstraint() && !isAllowBackupSetting(elem.setting_name))
             res.set(
                 elem.setting_name,
-                elem.min_value,
-                elem.max_value,
+                elem.min_value ? *elem.min_value : Field{},
+                elem.max_value ? *elem.max_value : Field{},
                 elem.writability ? *elem.writability : SettingConstraintWritability::WRITABLE);
     return res;
 }
@@ -240,8 +240,8 @@ bool SettingsProfileElements::isBackupAllowed() const
 {
     for (const auto & setting : *this)
     {
-        if (isAllowBackupSetting(setting.setting_name))
-            return static_cast<bool>(SettingFieldBool{setting.value});
+        if (isAllowBackupSetting(setting.setting_name) && setting.value)
+            return static_cast<bool>(SettingFieldBool{*setting.value});
     }
     return true;
 }
diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h
index 7f9379c1e47..7078f565295 100644
--- a/src/Access/SettingsProfileElement.h
+++ b/src/Access/SettingsProfileElement.h
@@ -23,9 +23,9 @@ struct SettingsProfileElement
     std::optional<UUID> parent_profile;
 
     String setting_name;
-    Field value;
-    Field min_value;
-    Field max_value;
+    std::optional<Field> value;
+    std::optional<Field> min_value;
+    std::optional<Field> max_value;
     std::optional<SettingConstraintWritability> writability;
 
     auto toTuple() const { return std::tie(parent_profile, setting_name, value, min_value, max_value, writability); }
diff --git a/src/Parsers/Access/ASTSettingsProfileElement.cpp b/src/Parsers/Access/ASTSettingsProfileElement.cpp
index 76973c428b2..7b29b15cb29 100644
--- a/src/Parsers/Access/ASTSettingsProfileElement.cpp
+++ b/src/Parsers/Access/ASTSettingsProfileElement.cpp
@@ -35,21 +35,21 @@ void ASTSettingsProfileElement::formatImpl(const FormatSettings & settings, Form
 
     formatSettingName(setting_name, settings.ostr);
 
-    if (!value.isNull())
+    if (value)
     {
-        settings.ostr << " = " << applyVisitor(FieldVisitorToString{}, value);
+        settings.ostr << " = " << applyVisitor(FieldVisitorToString{}, *value);
     }
 
-    if (!min_value.isNull())
+    if (min_value)
     {
         settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " MIN " << (settings.hilite ? IAST::hilite_none : "")
-                      << applyVisitor(FieldVisitorToString{}, min_value);
+                      << applyVisitor(FieldVisitorToString{}, *min_value);
     }
 
-    if (!max_value.isNull())
+    if (max_value)
     {
         settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " MAX " << (settings.hilite ? IAST::hilite_none : "")
-                      << applyVisitor(FieldVisitorToString{}, max_value);
+                      << applyVisitor(FieldVisitorToString{}, *max_value);
     }
 
     if (writability)
diff --git a/src/Parsers/Access/ASTSettingsProfileElement.h b/src/Parsers/Access/ASTSettingsProfileElement.h
index 275257e4f8e..13c1926d9b0 100644
--- a/src/Parsers/Access/ASTSettingsProfileElement.h
+++ b/src/Parsers/Access/ASTSettingsProfileElement.h
@@ -14,9 +14,9 @@ class ASTSettingsProfileElement : public IAST
 public:
     String parent_profile;
     String setting_name;
-    Field value;
-    Field min_value;
-    Field max_value;
+    std::optional<Field> value;
+    std::optional<Field> min_value;
+    std::optional<Field> max_value;
     std::optional<SettingConstraintWritability> writability;
     bool id_mode = false;  /// If true then `parent_profile` keeps UUID, not a name.
     bool use_inherit_keyword = false;  /// If true then this element is a part of ASTCreateSettingsProfileQuery.
diff --git a/src/Parsers/Access/ParserSettingsProfileElement.cpp b/src/Parsers/Access/ParserSettingsProfileElement.cpp
index db23a806a12..36330b96622 100644
--- a/src/Parsers/Access/ParserSettingsProfileElement.cpp
+++ b/src/Parsers/Access/ParserSettingsProfileElement.cpp
@@ -52,7 +52,7 @@ namespace
     }
 
 
-    bool parseValue(IParserBase::Pos & pos, Expected & expected, Field & res)
+    bool parseValue(IParserBase::Pos & pos, Expected & expected, std::optional<Field> & res)
     {
         return IParserBase::wrapParseImpl(pos, [&]
         {
@@ -69,7 +69,7 @@ namespace
     }
 
 
-    bool parseMinMaxValue(IParserBase::Pos & pos, Expected & expected, Field & min_value, Field & max_value)
+    bool parseMinMaxValue(IParserBase::Pos & pos, Expected & expected, std::optional<Field> & min_value, std::optional<Field> & max_value)
     {
         return IParserBase::wrapParseImpl(pos, [&]
         {
@@ -124,9 +124,9 @@ namespace
         IParserBase::Pos & pos,
         Expected & expected,
         String & setting_name,
-        Field & value,
-        Field & min_value,
-        Field & max_value,
+        std::optional<Field> & value,
+        std::optional<Field> & min_value,
+        std::optional<Field> & max_value,
         std::optional<SettingConstraintWritability> & writability)
     {
         return IParserBase::wrapParseImpl(pos, [&]
@@ -136,9 +136,9 @@ namespace
                 return false;
 
             String res_setting_name = getIdentifierName(name_ast);
-            Field res_value;
-            Field res_min_value;
-            Field res_max_value;
+            std::optional<Field> res_value;
+            std::optional<Field> res_min_value;
+            std::optional<Field> res_max_value;
             std::optional<SettingConstraintWritability> res_writability;
 
             bool has_value_or_constraint = false;
@@ -151,7 +151,7 @@ namespace
             if (!has_value_or_constraint)
                 return false;
 
-            if (boost::iequals(res_setting_name, "PROFILE") && res_value.isNull() && res_min_value.isNull() && res_max_value.isNull()
+            if (boost::iequals(res_setting_name, "PROFILE") && !res_value && !res_min_value && !res_max_value
                 && res_writability == SettingConstraintWritability::CONST)
             {
                 /// Ambiguity: "profile readonly" can be treated either as a profile named "readonly" or
@@ -181,9 +181,9 @@ namespace
         {
             String parent_profile;
             String setting_name;
-            Field value;
-            Field min_value;
-            Field max_value;
+            std::optional<Field> value;
+            std::optional<Field> min_value;
+            std::optional<Field> max_value;
             std::optional<SettingConstraintWritability> writability;
 
             bool ok = parseSettingNameWithValueOrConstraints(pos, expected, setting_name, value, min_value, max_value, writability);
diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.cpp b/src/Storages/System/StorageSystemSettingsProfileElements.cpp
index 6785a4392e1..e01d3cb0ace 100644
--- a/src/Storages/System/StorageSystemSettingsProfileElements.cpp
+++ b/src/Storages/System/StorageSystemSettingsProfileElements.cpp
@@ -87,27 +87,27 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns
         size_t current_index = index++;
 
         bool inserted_value = false;
-        if (!element.value.isNull() && !element.setting_name.empty())
+        if (element.value && !element.setting_name.empty())
         {
-            String str = Settings::valueToStringUtil(element.setting_name, element.value);
+            String str = Settings::valueToStringUtil(element.setting_name, *element.value);
             column_value.insertData(str.data(), str.length());
             column_value_null_map.push_back(false);
             inserted_value = true;
         }
 
         bool inserted_min = false;
-        if (!element.min_value.isNull() && !element.setting_name.empty())
+        if (element.min_value && !element.setting_name.empty())
         {
-            String str = Settings::valueToStringUtil(element.setting_name, element.min_value);
+            String str = Settings::valueToStringUtil(element.setting_name, *element.min_value);
             column_min.insertData(str.data(), str.length());
             column_min_null_map.push_back(false);
             inserted_min = true;
         }
 
         bool inserted_max = false;
-        if (!element.max_value.isNull() && !element.setting_name.empty())
+        if (element.max_value && !element.setting_name.empty())
         {
-            String str = Settings::valueToStringUtil(element.setting_name, element.max_value);
+            String str = Settings::valueToStringUtil(element.setting_name, *element.max_value);
             column_max.insertData(str.data(), str.length());
             column_max_null_map.push_back(false);
             inserted_max = true;

From 4d1b113d36db612a2b07b727d93023a3924da8a4 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 12 May 2023 09:41:13 +0000
Subject: [PATCH 446/689] README.md: 4 --> 5

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ee464900035..f4065439465 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ curl https://clickhouse.com/ | sh
 
 ## Upcoming Events
 
-* [**v23.5 Release Webinar**](https://clickhouse.com/company/events/v23-5-release-webinar?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-05) - May 31 - 23.4 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
+* [**v23.5 Release Webinar**](https://clickhouse.com/company/events/v23-5-release-webinar?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-05) - May 31 - 23.5 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
 * [**ClickHouse Meetup in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/292892466) - May 16 
 * [**ClickHouse Meetup in Barcelona**](https://www.meetup.com/clickhouse-barcelona-user-group/events/292892669) - May 25 
 * [**ClickHouse Meetup in London**](https://www.meetup.com/clickhouse-london-user-group/events/292892824) - May 25 

From 8fc0083a264cb4f1afb821bd9884ea1f6f414710 Mon Sep 17 00:00:00 2001
From: "Diego Nieto (lesandie)" <lesandie@gmail.com>
Date: Fri, 12 May 2023 11:45:38 +0200
Subject: [PATCH 447/689] Rewrite following conversation/comments

---
 docs/en/operations/storing-data.md | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md
index d0102ce38d4..e8b043e7a27 100644
--- a/docs/en/operations/storing-data.md
+++ b/docs/en/operations/storing-data.md
@@ -172,7 +172,7 @@ Example of configuration for versions earlier than 22.8:
     </storage_configuration>
 ```
 
-File Cache **configuration settings**:
+File Cache **disk configuration settings**:
 
 These settings should be defined in the disk configuration section.
 
@@ -180,7 +180,7 @@ These settings should be defined in the disk configuration section.
 
 - `max_size` - maximum size of the cache in bytes or in readable format, e.g. `ki, Mi, Gi, etc`, example `10Gi` (such format works starting from `22.10` version). When the limit is reached, cache files are evicted according to the cache eviction policy. Default: None, this setting is obligatory.
 
-- `cache_on_write_operations` - allow to turn on `write-through` cache (caching data on any write operations: `INSERT` queries, background merges). Default: `false`. 
+- `cache_on_write_operations` - allow to turn on `write-through` cache (caching data on any write operations: `INSERT` queries, background merges). Default: `false`. The `write-through` cache can be disabled per query using setting `enable_filesystem_cache_on_write_operations` (data is cached only if both cache config settings and corresponding query setting are enabled).
 
 - `enable_filesystem_query_cache_limit` - allow to limit the size of cache which is downloaded within each query (depends on user setting `max_query_cache_size`). Default: `false`.
 
@@ -192,23 +192,23 @@ These settings should be defined in the disk configuration section.
 
 - `max_elements` - a limit for a number of cache files. Default: `1048576`.
 
-File Cache **query settings**:
+File Cache **query/profile settings**:
 
-Some of these settings will disable cache per query features that are enabled by default. For example, setting `cache_on_write_operations` to 1 means that general file and per query cache are enabled but also setting `enable_filesystem_cache_on_write_operations` to 1 means that file cache is enabled but disabled per query cache. The same approach must be used for `enable_filesystem_cache`, if set to 1, cache per query is disabled.
+Some of these settings will disable cache features per query/profile that are enabled by default. For example, you can enable cache in disk configuration and disable it per query/profile setting `enable_filesystem_cache` to `false`. Also setting `cache_on_write_operations` to `true` in disk configuration means that general file and per query cache are enabled. But if you need to disable this general setting per specific queries then setting `enable_filesystem_cache_on_write_operations` to `false` means that write operations cache will be disabled for a specific query/profile.
 
-- `enable_filesystem_cache` - allows to disable cache per query even if storage policy was configured with `cache` disk type. Default: `true`. This setting should be defined in the disk configuration section.
+- `enable_filesystem_cache` - allows to disable cache per query even if storage policy was configured with `cache` disk type. Default: `true`.
 
-- `read_from_filesystem_cache_if_exists_otherwise_bypass_cache` - allows to use cache in query only if it already exists, otherwise query data will not be written to local cache storage. Default: `false`. This is a profile level configuration setting.
+- `read_from_filesystem_cache_if_exists_otherwise_bypass_cache` - allows to use cache in query only if it already exists, otherwise query data will not be written to local cache storage. Default: `false`.
 
-- `enable_filesystem_cache_on_write_operations` - turn on `write-through` cache. This setting works only if setting `cache_on_write_operations` in cache configuration is turned on. This setting should be defined in the disk configuration section.
+- `enable_filesystem_cache_on_write_operations` - turn on `write-through` cache. This setting works only if setting `cache_on_write_operations` in cache configuration is turned on. Default: `false`.
 
-- `enable_filesystem_cache_log` - turn on logging to `system.filesystem_cache_log` table. Gives a detailed view of cache usage per query. Default: `false`. This is a profile level configuration setting.
+- `enable_filesystem_cache_log` - turn on logging to `system.filesystem_cache_log` table. Gives a detailed view of cache usage per query. It can be turn on for specific queries or enabled in a profile. Default: `false`.
 
-- `max_query_cache_size` - a limit for the cache size, which can be written to local cache storage. Requires enabled `enable_filesystem_query_cache_limit` in cache configuration. Default: `false`. This setting should be defined in the disk configuration section.
+- `max_query_cache_size` - a limit for the cache size, which can be written to local cache storage. Requires enabled `enable_filesystem_query_cache_limit` in cache configuration. Default: `false`.
 
-- `skip_download_if_exceeds_query_cache` - allows to change the behaviour of setting `max_query_cache_size`. Default: `true`. If this setting is turned on and cache download limit during query was reached, no more cache will be downloaded to cache storage. If this setting is turned off and cache download limit during query was reached, cache will still be written by cost of evicting previously downloaded (within current query) data, e.g. second behaviour allows to preserve `last recently used` behaviour while keeping query cache limit. This is a profile level configuration setting.
+- `skip_download_if_exceeds_query_cache` - allows to change the behaviour of setting `max_query_cache_size`. Default: `true`. If this setting is turned on and cache download limit during query was reached, no more cache will be downloaded to cache storage. If this setting is turned off and cache download limit during query was reached, cache will still be written by cost of evicting previously downloaded (within current query) data, e.g. second behaviour allows to preserve `last recently used` behaviour while keeping query cache limit.
 
-** Warning **
+**Warning**
 Cache configuration settings and cache query settings correspond to the latest ClickHouse version, for earlier versions something might not be supported.
 
 Cache **system tables**:

From 22f7aa8d89107910f10c8ff5fb92296856c81283 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Fri, 12 May 2023 12:00:15 +0200
Subject: [PATCH 448/689] make special build pass

---
 src/IO/WriteBufferFromS3.h            |   5 --
 src/IO/tests/gtest_writebuffer_s3.cpp | 104 +++++++++++++-------------
 2 files changed, 52 insertions(+), 57 deletions(-)

diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h
index b0d8d329589..e65127872fa 100644
--- a/src/IO/WriteBufferFromS3.h
+++ b/src/IO/WriteBufferFromS3.h
@@ -16,11 +16,6 @@
 #include <vector>
 #include <list>
 
-namespace Aws::S3
-{
-class Client;
-}
-
 namespace DB
 {
 /**
diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp
index d7661d3e3d0..c0bd6742ea3 100644
--- a/src/IO/tests/gtest_writebuffer_s3.cpp
+++ b/src/IO/tests/gtest_writebuffer_s3.cpp
@@ -60,12 +60,12 @@ private:
 class BucketMemStore
 {
 public:
-    typedef std::string Key;
-    typedef std::string Data;
-    typedef std::string ETag;
-    typedef std::string MPU_ID;
-    typedef std::map<ETag, Data> MPUPartsInProgress;
-    typedef std::vector<Data> MPUParts;
+    using Key = std::string;
+    using Data = std::string;
+    using ETag = std::string;
+    using MPU_ID = std::string;
+    using MPUPartsInProgress = std::map<ETag, Data>;
+    using MPUParts = std::vector<Data>;
 
 
     std::map<Key, Data> objects;
@@ -129,7 +129,7 @@ public:
     {
         std::vector<size_t> result;
         result.reserve(parts.size());
-        for (auto & part_data : parts)
+        for (const auto & part_data : parts)
             result.push_back(part_data.size());
 
         return result;
@@ -142,7 +142,7 @@ class S3MemStrore
 public:
     void CreateBucket(const std::string & bucket)
     {
-        assert(buckets.count(bucket) == 0);
+        assert(!buckets.contains(bucket));
         buckets.emplace(bucket, BucketMemStore{});
     }
 
@@ -193,14 +193,14 @@ struct InjectionModel
 
 struct Client : DB::S3::Client
 {
-    Client(std::shared_ptr<S3MemStrore> mock_s3_store)
+    explicit Client(std::shared_ptr<S3MemStrore> mock_s3_store)
         : DB::S3::Client(
                100,
                DB::S3::ServerSideEncryptionKMSConfig(),
                std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>("", ""),
                GetClientConfiguration(),
                Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
-               /* useVirtualAddressing = */ true)
+               /* use_virtual_addressing = */ true)
         , store(mock_s3_store)
     { }
 
@@ -425,8 +425,8 @@ struct BaseSyncPolicy
 {
     virtual ~BaseSyncPolicy() = default;
     virtual DB::ThreadPoolCallbackRunner<void> getScheduler() { return {}; }
-    virtual void execute(size_t = 0) {}
-    virtual void setAutoExecute(bool = true) {}
+    virtual void execute(size_t) {}
+    virtual void setAutoExecute(bool) {}
 
     virtual size_t size() const { return 0; }
     virtual bool empty() const { return size() == 0; }
@@ -437,7 +437,7 @@ struct SimpleAsyncTasks : BaseSyncPolicy
     bool auto_execute = false;
     std::deque<std::packaged_task<void()>> queue;
 
-    virtual DB::ThreadPoolCallbackRunner<void> getScheduler() override
+    DB::ThreadPoolCallbackRunner<void> getScheduler() override
     {
         return [this] (std::function<void()> && operation, size_t /*priority*/)
         {
@@ -453,7 +453,7 @@ struct SimpleAsyncTasks : BaseSyncPolicy
         };
     }
 
-    virtual void execute(size_t limit = 0) override
+    void execute(size_t limit) override
     {
         if (limit == 0)
             limit = queue.size();
@@ -468,14 +468,14 @@ struct SimpleAsyncTasks : BaseSyncPolicy
         }
     }
 
-    virtual void setAutoExecute(bool value = true) override
+    void setAutoExecute(bool value) override
     {
         auto_execute = value;
         if (auto_execute)
-            execute();
+            execute(0);
     }
 
-    virtual size_t size() const override { return queue.size(); }
+    size_t size() const override { return queue.size(); }
 };
 
 }
@@ -545,7 +545,7 @@ public:
             auto buffer = getWriteBuffer("file");
             writeMethod(*buffer, size);
 
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
             buffer->finalize();
 
             expected_counters.writtenSize = size;
@@ -585,13 +585,13 @@ protected:
     std::shared_ptr<MockS3::Client> client;
     std::unique_ptr<MockS3::BaseSyncPolicy> async_policy;
 
-    virtual void SetUp() override
+    void SetUp() override
     {
         client = MockS3::Client::CreateClient(bucket);
         async_policy = std::make_unique<MockS3::BaseSyncPolicy>();
     }
 
-    virtual void TearDown() override
+    void TearDown() override
     {
         client.reset();
         async_policy.reset();
@@ -603,13 +603,13 @@ class SyncAsync : public WBS3Test, public ::testing::WithParamInterface<bool>
 protected:
     bool test_with_pool = false;
 
-    virtual void SetUp() override
+    void SetUp() override
     {
         test_with_pool = GetParam();
         client = MockS3::Client::CreateClient(bucket);
         if (test_with_pool)
             async_policy = std::make_unique<MockS3::SimpleAsyncTasks>();
-         else
+        else
             async_policy = std::make_unique<MockS3::BaseSyncPolicy>();
     }
 };
@@ -622,7 +622,7 @@ INSTANTIATE_TEST_SUITE_P(WBS3
         return name;
   });
 
-TEST_P(SyncAsync, exception_on_head) {
+TEST_P(SyncAsync, ExceptionOnHead) {
     setInjectionModel(std::make_shared<MockS3::HeadObjectFailIngection>());
 
     getSettings().s3_check_objects_after_upload = true;
@@ -633,7 +633,7 @@ TEST_P(SyncAsync, exception_on_head) {
             buffer->write('A');
             buffer->next();
 
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
             buffer->finalize();
         }
         catch( const DB::Exception& e )
@@ -645,7 +645,7 @@ TEST_P(SyncAsync, exception_on_head) {
     }, DB::S3Exception);
 }
 
-TEST_P(SyncAsync, exception_on_put) {
+TEST_P(SyncAsync, ExceptionOnPut) {
     setInjectionModel(std::make_shared<MockS3::PutObjectFailIngection>());
 
     EXPECT_THROW({
@@ -655,7 +655,7 @@ TEST_P(SyncAsync, exception_on_put) {
             buffer->write('A');
             buffer->next();
 
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
             buffer->finalize();
         }
         catch( const DB::Exception& e )
@@ -671,7 +671,7 @@ TEST_P(SyncAsync, exception_on_put) {
             auto buffer = getWriteBuffer("exception_on_put_2");
             buffer->write('A');
 
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
             buffer->finalize();
         }
         catch( const DB::Exception& e )
@@ -686,10 +686,10 @@ TEST_P(SyncAsync, exception_on_put) {
         try {
             auto buffer = getWriteBuffer("exception_on_put_3");
             buffer->write('A');
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
             buffer->preFinalize();
 
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
             buffer->finalize();
         }
         catch( const DB::Exception& e )
@@ -702,7 +702,7 @@ TEST_P(SyncAsync, exception_on_put) {
 
 }
 
-TEST_P(SyncAsync, exception_on_create_mpu) {
+TEST_P(SyncAsync, ExceptionOnCreateMPU) {
     setInjectionModel(std::make_shared<MockS3::CreateMPUFailIngection>());
 
     getSettings().s3_max_single_part_upload_size = 0; // no single part
@@ -716,7 +716,7 @@ TEST_P(SyncAsync, exception_on_create_mpu) {
             buffer->write('A');
             buffer->next();
 
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
             buffer->finalize();
         }
         catch( const DB::Exception& e )
@@ -733,7 +733,7 @@ TEST_P(SyncAsync, exception_on_create_mpu) {
             buffer->write('A');
             buffer->preFinalize();
 
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
             buffer->finalize();
         }
         catch( const DB::Exception& e )
@@ -749,7 +749,7 @@ TEST_P(SyncAsync, exception_on_create_mpu) {
             auto buffer = getWriteBuffer("exception_on_create_mpu_2");
             buffer->write('A');
 
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
             buffer->finalize();
         }
         catch( const DB::Exception& e )
@@ -762,7 +762,7 @@ TEST_P(SyncAsync, exception_on_create_mpu) {
 }
 
 
-TEST_P(SyncAsync, exception_on_complete_mpu) {
+TEST_P(SyncAsync, ExceptionOnCompleteMPU) {
     setInjectionModel(std::make_shared<MockS3::CompleteMPUFailIngection>());
 
     getSettings().s3_max_single_part_upload_size = 0; // no single part
@@ -773,7 +773,7 @@ TEST_P(SyncAsync, exception_on_complete_mpu) {
             auto buffer = getWriteBuffer("exception_on_complete_mpu_1");
             buffer->write('A');
 
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
             buffer->finalize();
         }
         catch(const DB::Exception & e)
@@ -785,7 +785,7 @@ TEST_P(SyncAsync, exception_on_complete_mpu) {
       }, DB::S3Exception);
 }
 
-TEST_P(SyncAsync, exception_on_upload_part) {
+TEST_P(SyncAsync, ExceptionOnUploadPart) {
     setInjectionModel(std::make_shared<MockS3::UploadPartFailIngection>());
 
     getSettings().s3_max_single_part_upload_size = 0; // no single part
@@ -804,7 +804,7 @@ TEST_P(SyncAsync, exception_on_upload_part) {
             buffer->write('A');
             buffer->next();
 
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
 
             buffer->finalize();
         }
@@ -820,7 +820,7 @@ TEST_P(SyncAsync, exception_on_upload_part) {
     EXPECT_THROW({
         try {
             auto buffer = getWriteBuffer("exception_on_upload_part_2");
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
 
             buffer->write('A');
             buffer->next();
@@ -848,7 +848,7 @@ TEST_P(SyncAsync, exception_on_upload_part) {
 
             buffer->preFinalize();
 
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
             buffer->finalize();
         }
         catch(const DB::Exception & e)
@@ -865,7 +865,7 @@ TEST_P(SyncAsync, exception_on_upload_part) {
             auto buffer = getWriteBuffer("exception_on_upload_part_4");
             buffer->write('A');
 
-            getAsyncPolicy().setAutoExecute();
+            getAsyncPolicy().setAutoExecute(true);
             buffer->finalize();
         }
         catch(const DB::Exception & e)
@@ -879,7 +879,7 @@ TEST_P(SyncAsync, exception_on_upload_part) {
 }
 
 
-TEST_F(WBS3Test, prefinalize_called_multiple_times) {
+TEST_F(WBS3Test, PrefinalizeCalledMultipleTimes) {
 #ifdef ABORT_ON_LOGICAL_ERROR
     GTEST_SKIP() << "this test trigger LOGICAL_ERROR, runs only if ABORT_ON_LOGICAL_ERROR is not defined";
 #else
@@ -904,14 +904,14 @@ TEST_F(WBS3Test, prefinalize_called_multiple_times) {
 #endif
 }
 
-TEST_P(SyncAsync, empty_file) {
+TEST_P(SyncAsync, EmptyFile) {
     getSettings().s3_check_objects_after_upload = true;
 
     MockS3::EventCounts counters = {.headObject = 2, .putObject = 1};
     runSimpleScenario(counters, 0);
 }
 
-TEST_P(SyncAsync, manual_next_calls) {
+TEST_P(SyncAsync, ManualNextCalls) {
     getSettings().s3_check_objects_after_upload = true;
 
     {
@@ -920,7 +920,7 @@ TEST_P(SyncAsync, manual_next_calls) {
         auto buffer = getWriteBuffer("manual_next_calls_1");
         buffer->next();
 
-        getAsyncPolicy().setAutoExecute();
+        getAsyncPolicy().setAutoExecute(true);
         buffer->finalize();
 
         assertCountersEQ(counters);
@@ -933,7 +933,7 @@ TEST_P(SyncAsync, manual_next_calls) {
         buffer->next();
         buffer->next();
 
-        getAsyncPolicy().setAutoExecute();
+        getAsyncPolicy().setAutoExecute(true);
         buffer->finalize();
 
         assertCountersEQ(counters);
@@ -947,7 +947,7 @@ TEST_P(SyncAsync, manual_next_calls) {
         buffer->write('A');
         buffer->next();
 
-        getAsyncPolicy().setAutoExecute();
+        getAsyncPolicy().setAutoExecute(true);
         buffer->finalize();
 
         assertCountersEQ(counters);
@@ -963,14 +963,14 @@ TEST_P(SyncAsync, manual_next_calls) {
         buffer->next();
         buffer->next();
 
-        getAsyncPolicy().setAutoExecute();
+        getAsyncPolicy().setAutoExecute(true);
         buffer->finalize();
 
         assertCountersEQ(counters);
      }
 }
 
-TEST_P(SyncAsync, small_file_is_one_put_request) {
+TEST_P(SyncAsync, SmallFileIsOnePutRequest) {
     getSettings().s3_check_objects_after_upload = true;
 
     {
@@ -999,7 +999,7 @@ TEST_P(SyncAsync, small_file_is_one_put_request) {
     }
 }
 
-TEST_P(SyncAsync, little_bigger_file_is_multi_part_upload) {
+TEST_P(SyncAsync, LittleBiggerFileIsMultiPartUpload) {
     getSettings().s3_check_objects_after_upload = true;
 
     {
@@ -1026,7 +1026,7 @@ TEST_P(SyncAsync, little_bigger_file_is_multi_part_upload) {
     }
 }
 
-TEST_P(SyncAsync, bigger_file_is_multi_part_upload) {
+TEST_P(SyncAsync, BiggerFileIsMultiPartUpload) {
     getSettings().s3_check_objects_after_upload = true;
 
     {
@@ -1059,7 +1059,7 @@ TEST_P(SyncAsync, bigger_file_is_multi_part_upload) {
     }
 }
 
-TEST_P(SyncAsync, increase_upload_buffer) {
+TEST_P(SyncAsync, IncreaseUploadBuffer) {
     getSettings().s3_check_objects_after_upload = true;
 
     {
@@ -1092,7 +1092,7 @@ TEST_P(SyncAsync, increase_upload_buffer) {
     }
 }
 
-TEST_P(SyncAsync, increase_limited) {
+TEST_P(SyncAsync, IncreaseLimited) {
     getSettings().s3_check_objects_after_upload = true;
 
     {

From a4694ac1858890bcd3d35547cf5c4417252933c9 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Fri, 12 May 2023 12:29:29 +0200
Subject: [PATCH 449/689] Add test.

---
 .../01418_custom_settings.reference           | 16 +++++++++----
 .../0_stateless/01418_custom_settings.sql     | 24 +++++++++++++++----
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/tests/queries/0_stateless/01418_custom_settings.reference b/tests/queries/0_stateless/01418_custom_settings.reference
index cf0cb35c72a..8484a5d0e6f 100644
--- a/tests/queries/0_stateless/01418_custom_settings.reference
+++ b/tests/queries/0_stateless/01418_custom_settings.reference
@@ -1,3 +1,4 @@
+--- assigning ---
 5	UInt8
 -177	Int16
 98.11	Float64
@@ -6,7 +7,7 @@ custom_a	UInt64_5
 custom_b	Int64_-177
 custom_c	Float64_98.11
 custom_d	\'abc def\'
-
+--- modifying ---
 changed	String
 \N	Nullable(Nothing)
 50000	UInt16
@@ -15,9 +16,10 @@ custom_a	\'changed\'
 custom_b	NULL
 custom_c	UInt64_50000
 custom_d	Float64_1.11
-
+--- undefined setting ---
 404	UInt16
-
+--- wrong prefix ---
+--- using query context ---
 -0.333	Float64
 custom_e	Float64_-0.333
 404	UInt16
@@ -25,7 +27,13 @@ custom_e	UInt64_404
 word	String
 custom_f	\'word\'
 0
-
+--- compound identifier ---
 test	String
 custom_compound.identifier.v1	\'test\'
 CREATE SETTINGS PROFILE s1_01418 SETTINGS custom_compound.identifier.v2 = 100
+--- null type ---
+\N	Nullable(Nothing)
+custom_null	NULL
+\N	Nullable(Nothing)
+custom_null	NULL
+CREATE SETTINGS PROFILE s2_01418 SETTINGS custom_null = NULL
diff --git a/tests/queries/0_stateless/01418_custom_settings.sql b/tests/queries/0_stateless/01418_custom_settings.sql
index 95051db3a34..be18f553589 100644
--- a/tests/queries/0_stateless/01418_custom_settings.sql
+++ b/tests/queries/0_stateless/01418_custom_settings.sql
@@ -1,3 +1,6 @@
+DROP SETTINGS PROFILE IF EXISTS s1_01418, s2_01418;
+
+SELECT '--- assigning ---';
 SET custom_a = 5;
 SET custom_b = -177;
 SET custom_c = 98.11;
@@ -8,7 +11,7 @@ SELECT getSetting('custom_c') as v, toTypeName(v);
 SELECT getSetting('custom_d') as v, toTypeName(v);
 SELECT name, value FROM system.settings WHERE name LIKE 'custom_%' ORDER BY name;
 
-SELECT '';
+SELECT '--- modifying ---';
 SET custom_a = 'changed';
 SET custom_b = NULL;
 SET custom_c = 50000;
@@ -19,14 +22,15 @@ SELECT getSetting('custom_c') as v, toTypeName(v);
 SELECT getSetting('custom_d') as v, toTypeName(v);
 SELECT name, value FROM system.settings WHERE name LIKE 'custom_%' ORDER BY name;
 
-SELECT '';
+SELECT '--- undefined setting ---';
 SELECT getSetting('custom_e') as v, toTypeName(v); -- { serverError 115 } -- Setting not found.
 SET custom_e = 404;
 SELECT getSetting('custom_e') as v, toTypeName(v);
 
+SELECT '--- wrong prefix ---';
 SET invalid_custom = 8; -- { serverError 115 } -- Setting is neither a builtin nor started with one of the registered prefixes for user-defined settings.
 
-SELECT '';
+SELECT '--- using query context ---';
 SELECT getSetting('custom_e') as v, toTypeName(v) SETTINGS custom_e = -0.333;
 SELECT name, value FROM system.settings WHERE name = 'custom_e' SETTINGS custom_e = -0.333;
 SELECT getSetting('custom_e') as v, toTypeName(v);
@@ -37,7 +41,7 @@ SELECT name, value FROM system.settings WHERE name = 'custom_f' SETTINGS custom_
 SELECT getSetting('custom_f') as v, toTypeName(v); -- { serverError 115 } -- Setting not found.
 SELECT COUNT() FROM system.settings WHERE name = 'custom_f';
 
-SELECT '';
+SELECT '--- compound identifier ---';
 SET custom_compound.identifier.v1 = 'test';
 SELECT getSetting('custom_compound.identifier.v1') as v, toTypeName(v);
 SELECT name, value FROM system.settings WHERE name = 'custom_compound.identifier.v1';
@@ -45,3 +49,15 @@ SELECT name, value FROM system.settings WHERE name = 'custom_compound.identifier
 CREATE SETTINGS PROFILE s1_01418 SETTINGS custom_compound.identifier.v2 = 100;
 SHOW CREATE SETTINGS PROFILE s1_01418;
 DROP SETTINGS PROFILE s1_01418;
+
+SELECT '--- null type ---';
+SELECT getSetting('custom_null') as v, toTypeName(v) SETTINGS custom_null = NULL;
+SELECT name, value FROM system.settings WHERE name = 'custom_null' SETTINGS custom_null = NULL;
+
+SET custom_null = NULL;
+SELECT getSetting('custom_null') as v, toTypeName(v);
+SELECT name, value FROM system.settings WHERE name = 'custom_null';
+
+CREATE SETTINGS PROFILE s2_01418 SETTINGS custom_null = NULL;
+SHOW CREATE SETTINGS PROFILE s2_01418;
+DROP SETTINGS PROFILE s2_01418;

From 5706e05757f69a011688b62359ce6c0f070b2641 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Fri, 12 May 2023 13:00:56 +0200
Subject: [PATCH 450/689] adding more comments

---
 src/Storages/MergeTree/DataPartsExchange.cpp   |  3 +++
 src/Storages/MergeTree/IMergeTreeDataPart.h    | 10 ++++++++--
 src/Storages/MergeTree/MergeTask.cpp           |  3 +++
 src/Storages/MergeTree/MergeTreeData.cpp       |  3 +++
 src/Storages/MergeTree/MergeTreeDataWriter.cpp |  3 +++
 src/Storages/MergeTree/MutateTask.cpp          |  7 +++++++
 6 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp
index a9b7fec41e7..4f2ad823c3a 100644
--- a/src/Storages/MergeTree/DataPartsExchange.cpp
+++ b/src/Storages/MergeTree/DataPartsExchange.cpp
@@ -919,6 +919,9 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk(
 
         new_data_part->version.setCreationTID(Tx::PrehistoricTID, nullptr);
         new_data_part->is_temp = true;
+        /// In case of replicated merge tree with zero copy replication
+        /// Here Clickhouse claims that this new part can be deleted in temporary state without unlocking the blobs
+        /// The blobs have to stay intact, this temporary part does not own them and does not share them yet.
         new_data_part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::PRESERVE_BLOBS;
         new_data_part->modification_time = time(nullptr);
         new_data_part->loadColumnsChecksumsIndexes(true, false);
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h
index 634e275bd64..ff40c1da8d1 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.h
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.h
@@ -218,14 +218,20 @@ public:
     /// FIXME Why do we need this flag? What's difference from Temporary and DeleteOnDestroy state? Can we get rid of this?
     bool is_temp = false;
 
+    /// This type and the field remove_tmp_policy is used as a hint
+    /// to help avoid communication with keeper when temporary part is deleting.
+    /// The common procedure is to ask the keeper with unlock request to release a references to the blobs.
+    /// And then follow the keeper answer decide remove or preserve the blobs in that part from s3.
+    /// However in some special cases Clickhouse can make a decision without asking keeper.
     enum class BlobsRemovalPolicyForTemporaryParts
     {
+        /// decision about removing blobs is determined by keeper, the common case
         ASK_KEEPER,
+        /// is set when Clickhouse is sure that the blobs in the part are belong only to it, other replicas have not seen them yet
         REMOVE_BLOBS,
+        /// is set when Clickhouse is sure that the blobs belong to other replica and current replica has not locked them on s3 yet
         PRESERVE_BLOBS,
     };
-    /// That field is used by replicated merge tree with zero copy replication
-    /// Usually the data has to bo unlocked in keeper unless explicitly otherwise stated
     BlobsRemovalPolicyForTemporaryParts remove_tmp_policy = BlobsRemovalPolicyForTemporaryParts::ASK_KEEPER;
 
     /// If true it means that there are no ZooKeeper node for this part, so it should be deleted only from filesystem
diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp
index 2248b08b1da..e58a9fc283c 100644
--- a/src/Storages/MergeTree/MergeTask.cpp
+++ b/src/Storages/MergeTree/MergeTask.cpp
@@ -209,6 +209,9 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare()
     global_ctx->new_data_part->uuid = global_ctx->future_part->uuid;
     global_ctx->new_data_part->partition.assign(global_ctx->future_part->getPartition());
     global_ctx->new_data_part->is_temp = global_ctx->parent_part == nullptr;
+    /// In case of replicated merge tree with zero copy replication
+    /// Here Clickhouse claims that this new part can be deleted in temporary state without unlocking the blobs
+    /// The blobs have to be removed along with the part, this temporary part owns them and does not share them yet.
     global_ctx->new_data_part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS;
 
     ctx->need_remove_expired_values = false;
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index eeabaa9e145..ec141f594bc 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -8282,6 +8282,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createEmptyPart(
 
     new_data_part->minmax_idx = std::move(minmax_idx);
     new_data_part->is_temp = true;
+    /// In case of replicated merge tree with zero copy replication
+    /// Here Clickhouse claims that this new part can be deleted in temporary state without unlocking the blobs
+    /// The blobs have to be removed along with the part, this temporary part owns them and does not share them yet.
     new_data_part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS;
 
     auto new_data_part_storage = new_data_part->getDataPartStoragePtr();
diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index b5b982948b7..db486b163eb 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -469,6 +469,9 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl(
     new_data_part->partition = std::move(partition);
     new_data_part->minmax_idx = std::move(minmax_idx);
     new_data_part->is_temp = true;
+    /// In case of replicated merge tree with zero copy replication
+    /// Here Clickhouse claims that this new part can be deleted in temporary state without unlocking the blobs
+    /// The blobs have to be removed along with the part, this temporary part owns them and does not share them yet.
     new_data_part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS;
 
     SyncGuardPtr sync_guard;
diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp
index b2c2b4a96c0..b12dbadf1ed 100644
--- a/src/Storages/MergeTree/MutateTask.cpp
+++ b/src/Storages/MergeTree/MutateTask.cpp
@@ -1838,6 +1838,9 @@ bool MutateTask::prepare()
     if (!isWidePart(ctx->source_part) || !isFullPartStorage(ctx->source_part->getDataPartStorage())
         || (ctx->interpreter && ctx->interpreter->isAffectingAllColumns()))
     {
+        /// In case of replicated merge tree with zero copy replication
+        /// Here Clickhouse claims that this new part can be deleted in temporary state without unlocking the blobs
+        /// The blobs have to be removed along with the part, this temporary part owns them and does not share them yet.
         ctx->new_data_part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS;
 
         task = std::make_unique<MutateAllPartColumnsTask>(ctx);
@@ -1867,6 +1870,10 @@ bool MutateTask::prepare()
             ctx->for_file_renames,
             ctx->mrk_extension);
 
+        /// In case of replicated merge tree with zero copy replication
+        /// Here Clickhouse has to follow the common procedure when deleting new part in temporary state
+        /// Some of the files within the blobs are shared with source part, some belongs only to the part
+        /// Keeper has to be asked with unlock request to release the references to the blobs
         ctx->new_data_part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::ASK_KEEPER;
 
         task = std::make_unique<MutateSomePartColumnsTask>(ctx);

From 47b27ca5fb04f5b3d9e4eeed6ee986886a2f899c Mon Sep 17 00:00:00 2001
From: Denny Crane <denis.zhuravlov@gmail.com>
Date: Fri, 12 May 2023 08:23:02 -0300
Subject: [PATCH 451/689] Update hash-functions.md

---
 docs/en/sql-reference/functions/hash-functions.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md
index fe842732b89..5836d853fa6 100644
--- a/docs/en/sql-reference/functions/hash-functions.md
+++ b/docs/en/sql-reference/functions/hash-functions.md
@@ -279,6 +279,8 @@ cityHash64(par1,...)
 
 This is a fast non-cryptographic hash function. It uses the CityHash algorithm for string parameters and implementation-specific fast non-cryptographic hash function for parameters with other data types. The function uses the CityHash combinator to get the final results.
 
+Note that Google CityHash has changed the algorithm after it have been added it into ClickHouse. Now results of Google CityHash and ClickHouse cityHash64 are different. ClickHouse cityHash64 results are consistent with CityHash 1.0.2. Currently [farmhash64](#farmhash64) is consistent with Google CityHash.
+
 **Arguments**
 
 The function takes a variable number of input parameters. Arguments can be any of the [supported data types](/docs/en/sql-reference/data-types/index.md). For some data types calculated value of hash function may be the same for the same values even if types of arguments differ (integers of different size, named and unnamed `Tuple` with the same data, `Map` and the corresponding `Array(Tuple(key, value))` type with the same data).

From 051456ba9286f92d1435a50e06d983ab5675fe02 Mon Sep 17 00:00:00 2001
From: Denny Crane <denis.zhuravlov@gmail.com>
Date: Fri, 12 May 2023 08:33:27 -0300
Subject: [PATCH 452/689] Update hash-functions.md

---
 docs/en/sql-reference/functions/hash-functions.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md
index 5836d853fa6..802ad31e5f1 100644
--- a/docs/en/sql-reference/functions/hash-functions.md
+++ b/docs/en/sql-reference/functions/hash-functions.md
@@ -279,7 +279,7 @@ cityHash64(par1,...)
 
 This is a fast non-cryptographic hash function. It uses the CityHash algorithm for string parameters and implementation-specific fast non-cryptographic hash function for parameters with other data types. The function uses the CityHash combinator to get the final results.
 
-Note that Google CityHash has changed the algorithm after it have been added it into ClickHouse. Now results of Google CityHash and ClickHouse cityHash64 are different. ClickHouse cityHash64 results are consistent with CityHash 1.0.2. Currently [farmhash64](#farmhash64) is consistent with Google CityHash.
+Note that Google CityHash has changed the algorithm after it have been added to ClickHouse. Now results of Google CityHash and ClickHouse cityHash64 are different. ClickHouse cityHash64 results are consistent with CityHash 1.0.2. Currently [farmhash64](#farmhash64) is consistent with Google CityHash.
 
 **Arguments**
 

From 8f7546740cc47b16736c4b2ecea04ac3bfffcde8 Mon Sep 17 00:00:00 2001
From: Salvatore Mesoraca <salvatore.mesoraca@aiven.io>
Date: Fri, 12 May 2023 11:15:04 +0200
Subject: [PATCH 453/689] Make 01533_multiple_nested test more reliable

Tests run with random values for `min_bytes_to_use_direct_io`.
As a consequence, tests open files using Direct I/O, occasionally.

If the filesystem doesn't support DIRECT_IO (e.g. tmpfs) the open
will fail and it will be attempted again without that flag, but the
failed open is still accounted for in `ProfileEvents['FileOpen']`.

This change is needed to prevent the test from failing
just because the target FS doesn't support DIRECT_IO.
---
 tests/queries/0_stateless/01533_multiple_nested.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01533_multiple_nested.sql b/tests/queries/0_stateless/01533_multiple_nested.sql
index f39b56bbaa2..1a6f0ec395e 100644
--- a/tests/queries/0_stateless/01533_multiple_nested.sql
+++ b/tests/queries/0_stateless/01533_multiple_nested.sql
@@ -37,7 +37,7 @@ SELECT col1.a FROM nested FORMAT Null;
 
 -- 4 files: (col1.size0, col1.a) x2
 SYSTEM FLUSH LOGS;
-SELECT ProfileEvents['FileOpen']
+SELECT ProfileEvents['FileOpen'] - ProfileEvents['CreatedReadBufferDirectIOFailed']
 FROM system.query_log
 WHERE (type = 'QueryFinish') AND (lower(query) LIKE lower('SELECT col1.a FROM %nested%'))
     AND event_date >= yesterday() AND current_database = currentDatabase();
@@ -47,7 +47,7 @@ SELECT col3.n2.s FROM nested FORMAT Null;
 
 -- 6 files: (col3.size0, col3.n2.size1, col3.n2.s) x2
 SYSTEM FLUSH LOGS;
-SELECT ProfileEvents['FileOpen']
+SELECT ProfileEvents['FileOpen'] - ProfileEvents['CreatedReadBufferDirectIOFailed']
 FROM system.query_log
 WHERE (type = 'QueryFinish') AND (lower(query) LIKE lower('SELECT col3.n2.s FROM %nested%'))
     AND event_date >= yesterday() AND current_database = currentDatabase();

From 5807737804db1eb0717503817fe4b8a9fcc15045 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 12 May 2023 12:02:45 +0000
Subject: [PATCH 454/689] temporarily disable JobFailure test

---
 src/Common/tests/gtest_async_loader.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 2f6c74a0f96..5b04e67e763 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -387,7 +387,7 @@ TEST(AsyncLoader, CancelExecutingTask)
     }
 }
 
-TEST(AsyncLoader, JobFailure)
+TEST(AsyncLoader, DISABLEDJobFailure)
 {
     AsyncLoaderTest t;
     t.loader.start();

From 8dc2cad027eb423e0acd7044df7709f12a51961d Mon Sep 17 00:00:00 2001
From: Eridanus <maodingxuan@foxmail.com>
Date: Fri, 12 May 2023 10:40:09 +0800
Subject: [PATCH 455/689] test(parser): add hint for create quota overflow test

---
 .../0_stateless/01297_create_quota.reference  |  7 +++--
 .../0_stateless/01297_create_quota.sql        | 27 ++++++++++---------
 2 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/tests/queries/0_stateless/01297_create_quota.reference b/tests/queries/0_stateless/01297_create_quota.reference
index 76040b5dba5..3ff2d9bfec6 100644
--- a/tests/queries/0_stateless/01297_create_quota.reference
+++ b/tests/queries/0_stateless/01297_create_quota.reference
@@ -85,10 +85,9 @@ CREATE QUOTA q16_01297 FOR INTERVAL 1 minute MAX execution_time = 1.5
 -- overflow test
 CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 18446744073709551615
 CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 18446744073
-CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 0
-CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 9223372036.854776
-CREATE QUOTA q3_01297 FOR INTERVAL 1 minute MAX execution_time = 9223372036.854776
-CREATE QUOTA q4_01297 FOR INTERVAL 1 minute MAX execution_time = 9223372036.854776
+-- FIXME
+-- the overflow happens somewhere else before reaching the up bound of Quota Value , not in the parser
+-- the overflow have different behavior in different environment
 -- zero test
 CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 0
 CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 0
diff --git a/tests/queries/0_stateless/01297_create_quota.sql b/tests/queries/0_stateless/01297_create_quota.sql
index c47f68d69ae..1f4c3c729b2 100644
--- a/tests/queries/0_stateless/01297_create_quota.sql
+++ b/tests/queries/0_stateless/01297_create_quota.sql
@@ -197,18 +197,21 @@ SHOW CREATE QUOTA q1_01297;
 SHOW CREATE QUOTA q2_01297;
 DROP QUOTA IF EXISTS q1_01297;
 DROP QUOTA IF EXISTS q2_01297;
-CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '18446744073709551616';
-CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1T';
-CREATE QUOTA q3_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1Ti';
-CREATE QUOTA q4_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '18446744074';
-SHOW CREATE QUOTA q1_01297;
-SHOW CREATE QUOTA q2_01297;
-SHOW CREATE QUOTA q3_01297;
-SHOW CREATE QUOTA q4_01297;
-DROP QUOTA IF EXISTS q1_01297;
-DROP QUOTA IF EXISTS q2_01297;
-DROP QUOTA IF EXISTS q3_01297;
-DROP QUOTA IF EXISTS q4_01297;
+SELECT '-- FIXME';
+SELECT '-- the overflow happens somewhere else before reaching the up bound of Quota Value , not in the parser';
+SELECT '-- the overflow have different behavior in different environment';
+-- CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '18446744073709551616';
+-- CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1T';
+-- CREATE QUOTA q3_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1Ti';
+-- CREATE QUOTA q4_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '18446744074';
+-- SHOW CREATE QUOTA q1_01297;
+-- SHOW CREATE QUOTA q2_01297;
+-- SHOW CREATE QUOTA q3_01297;
+-- SHOW CREATE QUOTA q4_01297;
+-- DROP QUOTA IF EXISTS q1_01297;
+-- DROP QUOTA IF EXISTS q2_01297;
+-- DROP QUOTA IF EXISTS q3_01297;
+-- DROP QUOTA IF EXISTS q4_01297;
 SELECT '-- zero test';
 CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '0';
 CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '0';

From 0b9e5755884388254b6dfa79823560f4149ca675 Mon Sep 17 00:00:00 2001
From: DanRoscigno <dan@roscigno.com>
Date: Fri, 12 May 2023 08:05:38 -0400
Subject: [PATCH 456/689] uplevel UDFs and link

---
 docs/en/sql-reference/functions/index.md      | 242 +----------------
 docs/en/sql-reference/functions/udf.md        | 249 ++++++++++++++++++
 .../statements/create/function.md             |   4 +-
 3 files changed, 254 insertions(+), 241 deletions(-)
 create mode 100644 docs/en/sql-reference/functions/udf.md

diff --git a/docs/en/sql-reference/functions/index.md b/docs/en/sql-reference/functions/index.md
index 1577c01eec9..42d402e9d44 100644
--- a/docs/en/sql-reference/functions/index.md
+++ b/docs/en/sql-reference/functions/index.md
@@ -59,244 +59,6 @@ A lambda function that accepts multiple arguments can also be passed to a higher
 
 For some functions the first argument (the lambda function) can be omitted. In this case, identical mapping is assumed.
 
-## SQL User Defined Functions
+## User Defined Functions (UDFs)
 
-Custom functions from lambda expressions can be created using the [CREATE FUNCTION](../statements/create/function.md) statement. To delete these functions use the [DROP FUNCTION](../statements/drop.md#drop-function) statement.
-
-## Executable User Defined Functions
-ClickHouse can call any external executable program or script to process data.
-
-The configuration of executable user defined functions can be located in one or more xml-files. The path to the configuration is specified in the [user_defined_executable_functions_config](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_defined_executable_functions_config) parameter.
-
-A function configuration contains the following settings:
-
-- `name` - a function name.
-- `command` - script name to execute or command if `execute_direct` is false.
-- `argument` - argument description with the `type`, and optional `name` of an argument. Each argument is described in a separate setting. Specifying name is necessary if argument names are part of serialization for user defined function format like [Native](../../interfaces/formats.md#native) or [JSONEachRow](../../interfaces/formats.md#jsoneachrow). Default argument name value is `c` + argument_number.
-- `format` - a [format](../../interfaces/formats.md) in which arguments are passed to the command.
-- `return_type` - the type of a returned value.
-- `return_name` - name of retuned value. Specifying return name is necessary if return name is part of serialization for user defined function format like [Native](../../interfaces/formats.md#native) or [JSONEachRow](../../interfaces/formats.md#jsoneachrow). Optional. Default value is `result`.
-- `type` - an executable type. If `type` is set to `executable` then single command is started. If it is set to `executable_pool` then a pool of commands is created.
-- `max_command_execution_time` - maximum execution time in seconds for processing block of data. This setting is valid for `executable_pool` commands only. Optional. Default value is `10`.
-- `command_termination_timeout` - time in seconds during which a command should finish after its pipe is closed. After that time `SIGTERM` is sent to the process executing the command. Optional. Default value is `10`.
-- `command_read_timeout` - timeout for reading data from command stdout in milliseconds. Default value 10000. Optional parameter.
-- `command_write_timeout` - timeout for writing data to command stdin in milliseconds. Default value 10000. Optional parameter.
-- `pool_size` - the size of a command pool. Optional. Default value is `16`.
-- `send_chunk_header` - controls whether to send row count before sending a chunk of data to process. Optional. Default value is `false`.
-- `execute_direct` - If `execute_direct` = `1`, then `command` will be searched inside user_scripts folder specified by [user_scripts_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_scripts_path). Additional script arguments can be specified using whitespace separator. Example: `script_name arg1 arg2`. If `execute_direct` = `0`, `command` is passed as argument for `bin/sh -c`. Default value is `1`. Optional parameter.
-- `lifetime` - the reload interval of a function in seconds. If it is set to `0` then the function is not reloaded. Default value is `0`. Optional parameter.
-
-The command must read arguments from `STDIN` and must output the result to `STDOUT`. The command must process arguments iteratively. That is after processing a chunk of arguments it must wait for the next chunk.
-
-**Example**
-
-Creating `test_function` using XML configuration.
-File `test_function.xml` (`/etc/clickhouse-server/test_function.xml` with default path settings).
-```xml
-<functions>
-    <function>
-        <type>executable</type>
-        <name>test_function_python</name>
-        <return_type>String</return_type>
-        <argument>
-            <type>UInt64</type>
-            <name>value</name>
-        </argument>
-        <format>TabSeparated</format>
-        <command>test_function.py</command>
-    </function>
-</functions>
-```
-
-Script file inside `user_scripts` folder `test_function.py` (`/var/lib/clickhouse/user_scripts/test_function.py` with default path settings).
-
-```python
-#!/usr/bin/python3
-
-import sys
-
-if __name__ == '__main__':
-    for line in sys.stdin:
-        print("Value " + line, end='')
-        sys.stdout.flush()
-```
-
-Query:
-
-``` sql
-SELECT test_function_python(toUInt64(2));
-```
-
-Result:
-
-``` text
-┌─test_function_python(2)─┐
-│ Value 2                 │
-└─────────────────────────┘
-```
-
-Creating `test_function_sum` manually specifying `execute_direct` to `0` using XML configuration.
-File `test_function.xml` (`/etc/clickhouse-server/test_function.xml` with default path settings).
-```xml
-<functions>
-    <function>
-        <type>executable</type>
-        <name>test_function_sum</name>
-        <return_type>UInt64</return_type>
-        <argument>
-            <type>UInt64</type>
-            <name>lhs</name>
-        </argument>
-        <argument>
-            <type>UInt64</type>
-            <name>rhs</name>
-        </argument>
-        <format>TabSeparated</format>
-        <command>cd /; clickhouse-local --input-format TabSeparated --output-format TabSeparated --structure 'x UInt64, y UInt64' --query "SELECT x + y FROM table"</command>
-        <execute_direct>0</execute_direct>
-    </function>
-</functions>
-```
-
-Query:
-
-``` sql
-SELECT test_function_sum(2, 2);
-```
-
-Result:
-
-``` text
-┌─test_function_sum(2, 2)─┐
-│                       4 │
-└─────────────────────────┘
-```
-
-Creating `test_function_sum_json` with named arguments and format [JSONEachRow](../../interfaces/formats.md#jsoneachrow) using XML configuration.
-File `test_function.xml` (`/etc/clickhouse-server/test_function.xml` with default path settings).
-```xml
-<functions>
-    <function>
-        <type>executable</type>
-        <name>test_function_sum_json</name>
-        <return_type>UInt64</return_type>
-        <return_name>result_name</return_name>
-        <argument>
-            <type>UInt64</type>
-            <name>argument_1</name>
-        </argument>
-        <argument>
-            <type>UInt64</type>
-            <name>argument_2</name>
-        </argument>
-        <format>JSONEachRow</format>
-        <command>test_function_sum_json.py</command>
-    </function>
-</functions>
-```
-
-Script file inside `user_scripts` folder `test_function_sum_json.py` (`/var/lib/clickhouse/user_scripts/test_function_sum_json.py` with default path settings).
-
-```python
-#!/usr/bin/python3
-
-import sys
-import json
-
-if __name__ == '__main__':
-    for line in sys.stdin:
-        value = json.loads(line)
-        first_arg = int(value['argument_1'])
-        second_arg = int(value['argument_2'])
-        result = {'result_name': first_arg + second_arg}
-        print(json.dumps(result), end='\n')
-        sys.stdout.flush()
-```
-
-Query:
-
-``` sql
-SELECT test_function_sum_json(2, 2);
-```
-
-Result:
-
-``` text
-┌─test_function_sum_json(2, 2)─┐
-│                            4 │
-└──────────────────────────────┘
-```
-
-Executable user defined functions can take constant parameters configured in `command` setting (works only for user defined functions with `executable` type).
-File `test_function_parameter_python.xml` (`/etc/clickhouse-server/test_function_parameter_python.xml` with default path settings).
-```xml
-<functions>
-    <function>
-        <type>executable</type>
-        <name>test_function_parameter_python</name>
-        <return_type>String</return_type>
-        <argument>
-            <type>UInt64</type>
-        </argument>
-        <format>TabSeparated</format>
-        <command>test_function_parameter_python.py {test_parameter:UInt64}</command>
-    </function>
-</functions>
-```
-
-Script file inside `user_scripts` folder `test_function_parameter_python.py` (`/var/lib/clickhouse/user_scripts/test_function_parameter_python.py` with default path settings).
-
-```python
-#!/usr/bin/python3
-
-import sys
-
-if __name__ == "__main__":
-    for line in sys.stdin:
-        print("Parameter " + str(sys.argv[1]) + " value " + str(line), end="")
-        sys.stdout.flush()
-```
-
-Query:
-
-``` sql
-SELECT test_function_parameter_python(1)(2);
-```
-
-Result:
-
-``` text
-┌─test_function_parameter_python(1)(2)─┐
-│ Parameter 1 value 2                  │
-└──────────────────────────────────────┘
-```
-
-## Error Handling
-
-Some functions might throw an exception if the data is invalid. In this case, the query is canceled and an error text is returned to the client. For distributed processing, when an exception occurs on one of the servers, the other servers also attempt to abort the query.
-
-## Evaluation of Argument Expressions
-
-In almost all programming languages, one of the arguments might not be evaluated for certain operators. This is usually the operators `&&`, `||`, and `?:`.
-But in ClickHouse, arguments of functions (operators) are always evaluated. This is because entire parts of columns are evaluated at once, instead of calculating each row separately.
-
-## Performing Functions for Distributed Query Processing
-
-For distributed query processing, as many stages of query processing as possible are performed on remote servers, and the rest of the stages (merging intermediate results and everything after that) are performed on the requestor server.
-
-This means that functions can be performed on different servers.
-For example, in the query `SELECT f(sum(g(x))) FROM distributed_table GROUP BY h(y),`
-
-- if a `distributed_table` has at least two shards, the functions ‘g’ and ‘h’ are performed on remote servers, and the function ‘f’ is performed on the requestor server.
-- if a `distributed_table` has only one shard, all the ‘f’, ‘g’, and ‘h’ functions are performed on this shard’s server.
-
-The result of a function usually does not depend on which server it is performed on. However, sometimes this is important.
-For example, functions that work with dictionaries use the dictionary that exists on the server they are running on.
-Another example is the `hostName` function, which returns the name of the server it is running on in order to make `GROUP BY` by servers in a `SELECT` query.
-
-If a function in a query is performed on the requestor server, but you need to perform it on remote servers, you can wrap it in an ‘any’ aggregate function or add it to a key in `GROUP BY`.
-
-
-## Related Content
-
-- [User-defined functions in ClickHouse Cloud](https://clickhouse.com/blog/user-defined-functions-clickhouse-udfs)
+ClickHouse supports user-defined functions. See [UDFs](/docs/en/sql-reference/functions/udf.md).
diff --git a/docs/en/sql-reference/functions/udf.md b/docs/en/sql-reference/functions/udf.md
new file mode 100644
index 00000000000..a58c1364780
--- /dev/null
+++ b/docs/en/sql-reference/functions/udf.md
@@ -0,0 +1,249 @@
+---
+slug: /en/sql-reference/functions/udf
+sidebar_position: 15
+sidebar_label: UDF
+---
+
+# UDFs User Defined Functions
+
+
+## Executable User Defined Functions
+ClickHouse can call any external executable program or script to process data.
+
+The configuration of executable user defined functions can be located in one or more xml-files. The path to the configuration is specified in the [user_defined_executable_functions_config](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_defined_executable_functions_config) parameter.
+
+A function configuration contains the following settings:
+
+- `name` - a function name.
+- `command` - script name to execute or command if `execute_direct` is false.
+- `argument` - argument description with the `type`, and optional `name` of an argument. Each argument is described in a separate setting. Specifying name is necessary if argument names are part of serialization for user defined function format like [Native](../../interfaces/formats.md#native) or [JSONEachRow](../../interfaces/formats.md#jsoneachrow). Default argument name value is `c` + argument_number.
+- `format` - a [format](../../interfaces/formats.md) in which arguments are passed to the command.
+- `return_type` - the type of a returned value.
+- `return_name` - name of retuned value. Specifying return name is necessary if return name is part of serialization for user defined function format like [Native](../../interfaces/formats.md#native) or [JSONEachRow](../../interfaces/formats.md#jsoneachrow). Optional. Default value is `result`.
+- `type` - an executable type. If `type` is set to `executable` then single command is started. If it is set to `executable_pool` then a pool of commands is created.
+- `max_command_execution_time` - maximum execution time in seconds for processing block of data. This setting is valid for `executable_pool` commands only. Optional. Default value is `10`.
+- `command_termination_timeout` - time in seconds during which a command should finish after its pipe is closed. After that time `SIGTERM` is sent to the process executing the command. Optional. Default value is `10`.
+- `command_read_timeout` - timeout for reading data from command stdout in milliseconds. Default value 10000. Optional parameter.
+- `command_write_timeout` - timeout for writing data to command stdin in milliseconds. Default value 10000. Optional parameter.
+- `pool_size` - the size of a command pool. Optional. Default value is `16`.
+- `send_chunk_header` - controls whether to send row count before sending a chunk of data to process. Optional. Default value is `false`.
+- `execute_direct` - If `execute_direct` = `1`, then `command` will be searched inside user_scripts folder specified by [user_scripts_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_scripts_path). Additional script arguments can be specified using whitespace separator. Example: `script_name arg1 arg2`. If `execute_direct` = `0`, `command` is passed as argument for `bin/sh -c`. Default value is `1`. Optional parameter.
+- `lifetime` - the reload interval of a function in seconds. If it is set to `0` then the function is not reloaded. Default value is `0`. Optional parameter.
+
+The command must read arguments from `STDIN` and must output the result to `STDOUT`. The command must process arguments iteratively. That is after processing a chunk of arguments it must wait for the next chunk.
+
+**Example**
+
+Creating `test_function` using XML configuration.
+File `test_function.xml` (`/etc/clickhouse-server/test_function.xml` with default path settings).
+```xml
+<functions>
+    <function>
+        <type>executable</type>
+        <name>test_function_python</name>
+        <return_type>String</return_type>
+        <argument>
+            <type>UInt64</type>
+            <name>value</name>
+        </argument>
+        <format>TabSeparated</format>
+        <command>test_function.py</command>
+    </function>
+</functions>
+```
+
+Script file inside `user_scripts` folder `test_function.py` (`/var/lib/clickhouse/user_scripts/test_function.py` with default path settings).
+
+```python
+#!/usr/bin/python3
+
+import sys
+
+if __name__ == '__main__':
+    for line in sys.stdin:
+        print("Value " + line, end='')
+        sys.stdout.flush()
+```
+
+Query:
+
+``` sql
+SELECT test_function_python(toUInt64(2));
+```
+
+Result:
+
+``` text
+┌─test_function_python(2)─┐
+│ Value 2                 │
+└─────────────────────────┘
+```
+
+Creating `test_function_sum` manually specifying `execute_direct` to `0` using XML configuration.
+File `test_function.xml` (`/etc/clickhouse-server/test_function.xml` with default path settings).
+```xml
+<functions>
+    <function>
+        <type>executable</type>
+        <name>test_function_sum</name>
+        <return_type>UInt64</return_type>
+        <argument>
+            <type>UInt64</type>
+            <name>lhs</name>
+        </argument>
+        <argument>
+            <type>UInt64</type>
+            <name>rhs</name>
+        </argument>
+        <format>TabSeparated</format>
+        <command>cd /; clickhouse-local --input-format TabSeparated --output-format TabSeparated --structure 'x UInt64, y UInt64' --query "SELECT x + y FROM table"</command>
+        <execute_direct>0</execute_direct>
+    </function>
+</functions>
+```
+
+Query:
+
+``` sql
+SELECT test_function_sum(2, 2);
+```
+
+Result:
+
+``` text
+┌─test_function_sum(2, 2)─┐
+│                       4 │
+└─────────────────────────┘
+```
+
+Creating `test_function_sum_json` with named arguments and format [JSONEachRow](../../interfaces/formats.md#jsoneachrow) using XML configuration.
+File `test_function.xml` (`/etc/clickhouse-server/test_function.xml` with default path settings).
+```xml
+<functions>
+    <function>
+        <type>executable</type>
+        <name>test_function_sum_json</name>
+        <return_type>UInt64</return_type>
+        <return_name>result_name</return_name>
+        <argument>
+            <type>UInt64</type>
+            <name>argument_1</name>
+        </argument>
+        <argument>
+            <type>UInt64</type>
+            <name>argument_2</name>
+        </argument>
+        <format>JSONEachRow</format>
+        <command>test_function_sum_json.py</command>
+    </function>
+</functions>
+```
+
+Script file inside `user_scripts` folder `test_function_sum_json.py` (`/var/lib/clickhouse/user_scripts/test_function_sum_json.py` with default path settings).
+
+```python
+#!/usr/bin/python3
+
+import sys
+import json
+
+if __name__ == '__main__':
+    for line in sys.stdin:
+        value = json.loads(line)
+        first_arg = int(value['argument_1'])
+        second_arg = int(value['argument_2'])
+        result = {'result_name': first_arg + second_arg}
+        print(json.dumps(result), end='\n')
+        sys.stdout.flush()
+```
+
+Query:
+
+``` sql
+SELECT test_function_sum_json(2, 2);
+```
+
+Result:
+
+``` text
+┌─test_function_sum_json(2, 2)─┐
+│                            4 │
+└──────────────────────────────┘
+```
+
+Executable user defined functions can take constant parameters configured in `command` setting (works only for user defined functions with `executable` type).
+File `test_function_parameter_python.xml` (`/etc/clickhouse-server/test_function_parameter_python.xml` with default path settings).
+```xml
+<functions>
+    <function>
+        <type>executable</type>
+        <name>test_function_parameter_python</name>
+        <return_type>String</return_type>
+        <argument>
+            <type>UInt64</type>
+        </argument>
+        <format>TabSeparated</format>
+        <command>test_function_parameter_python.py {test_parameter:UInt64}</command>
+    </function>
+</functions>
+```
+
+Script file inside `user_scripts` folder `test_function_parameter_python.py` (`/var/lib/clickhouse/user_scripts/test_function_parameter_python.py` with default path settings).
+
+```python
+#!/usr/bin/python3
+
+import sys
+
+if __name__ == "__main__":
+    for line in sys.stdin:
+        print("Parameter " + str(sys.argv[1]) + " value " + str(line), end="")
+        sys.stdout.flush()
+```
+
+Query:
+
+``` sql
+SELECT test_function_parameter_python(1)(2);
+```
+
+Result:
+
+``` text
+┌─test_function_parameter_python(1)(2)─┐
+│ Parameter 1 value 2                  │
+└──────────────────────────────────────┘
+```
+
+## Error Handling
+
+Some functions might throw an exception if the data is invalid. In this case, the query is canceled and an error text is returned to the client. For distributed processing, when an exception occurs on one of the servers, the other servers also attempt to abort the query.
+
+## Evaluation of Argument Expressions
+
+In almost all programming languages, one of the arguments might not be evaluated for certain operators. This is usually the operators `&&`, `||`, and `?:`.
+But in ClickHouse, arguments of functions (operators) are always evaluated. This is because entire parts of columns are evaluated at once, instead of calculating each row separately.
+
+## Performing Functions for Distributed Query Processing
+
+For distributed query processing, as many stages of query processing as possible are performed on remote servers, and the rest of the stages (merging intermediate results and everything after that) are performed on the requestor server.
+
+This means that functions can be performed on different servers.
+For example, in the query `SELECT f(sum(g(x))) FROM distributed_table GROUP BY h(y),`
+
+- if a `distributed_table` has at least two shards, the functions ‘g’ and ‘h’ are performed on remote servers, and the function ‘f’ is performed on the requestor server.
+- if a `distributed_table` has only one shard, all the ‘f’, ‘g’, and ‘h’ functions are performed on this shard’s server.
+
+The result of a function usually does not depend on which server it is performed on. However, sometimes this is important.
+For example, functions that work with dictionaries use the dictionary that exists on the server they are running on.
+Another example is the `hostName` function, which returns the name of the server it is running on in order to make `GROUP BY` by servers in a `SELECT` query.
+
+If a function in a query is performed on the requestor server, but you need to perform it on remote servers, you can wrap it in an ‘any’ aggregate function or add it to a key in `GROUP BY`.
+
+## SQL User Defined Functions
+
+Custom functions from lambda expressions can be created using the [CREATE FUNCTION](../statements/create/function.md) statement. To delete these functions use the [DROP FUNCTION](../statements/drop.md#drop-function) statement.
+
+## Related Content
+
+### [User-defined functions in ClickHouse Cloud](https://clickhouse.com/blog/user-defined-functions-clickhouse-udfs)
diff --git a/docs/en/sql-reference/statements/create/function.md b/docs/en/sql-reference/statements/create/function.md
index 08946934f0c..15c2356445b 100644
--- a/docs/en/sql-reference/statements/create/function.md
+++ b/docs/en/sql-reference/statements/create/function.md
@@ -61,4 +61,6 @@ Result:
 
 ## Related Content
 
-- [User-defined functions in ClickHouse Cloud](https://clickhouse.com/blog/user-defined-functions-clickhouse-udfs)
+### [Executable UDFs](/docs/en/sql-reference/functions/udf.md).
+
+### [User-defined functions in ClickHouse Cloud](https://clickhouse.com/blog/user-defined-functions-clickhouse-udfs)

From c4344c1de9efe86b589dd1b835c0506557847834 Mon Sep 17 00:00:00 2001
From: AnneClickHouse <97107736+AnneClickHouse@users.noreply.github.com>
Date: Fri, 12 May 2023 14:59:01 +0200
Subject: [PATCH 457/689] Update README.md

Adding careers to bottom of readme
---
 README.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/README.md b/README.md
index f4065439465..a33e62be231 100644
--- a/README.md
+++ b/README.md
@@ -35,3 +35,12 @@ Also, keep an eye out for upcoming meetups in Amsterdam, Boston, NYC, Beijing, a
 * **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
 * **Recording available**: [**v23.4 Release Webinar**](https://www.youtube.com/watch?v=4rrf6bk_mOg) Faster Parquet Reading, Asynchonous Connections to Reoplicas, Trailing Comma before FROM, extractKeyValuePairs, integrations updates, and so much more! Watch it now!
 * **All release webinar recordings**: [YouTube playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3jAlSy1JxyP8zluvXaN3nxU)
+
+  
+ ## Interested in joining ClickHouse and making it your full time job? 
+  
+We are a globally diverse and distributed team, united behind a common goal of creating industry-leading, real-time analytics. Here, you will have an opportunity to solve some of the most cutting edge technical challenges and have direct ownership of your work and vision. If you are a contributor by nature, a thinker as well as a doer - we’ll definitely click!
+  
+Check out our **current openings** here: https://clickhouse.com/company/careers 
+  
+Cant find what you are looking for, but want to let us know you are interested in joining ClickHouse? Email careers@clickhouse.com ! 

From 809fdc7b16271bfc3bf44a832697deedeee56e87 Mon Sep 17 00:00:00 2001
From: HarryLeeIBM <hleeatwork@outlook.com>
Date: Fri, 12 May 2023 06:00:47 -0700
Subject: [PATCH 458/689] Fix DefaultHash64 for s390x

---
 src/Common/HashTable/Hash.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/Common/HashTable/Hash.h b/src/Common/HashTable/Hash.h
index 9e14851eb2a..189603fcbf5 100644
--- a/src/Common/HashTable/Hash.h
+++ b/src/Common/HashTable/Hash.h
@@ -251,7 +251,10 @@ requires (sizeof(T) <= sizeof(UInt64))
 inline size_t DefaultHash64(T key)
 {
     DB::UInt64 out {0};
-    std::memcpy(&out, &key, sizeof(T));
+    if constexpr (std::endian::native == std::endian::little)
+        std::memcpy(&out, &key, sizeof(T));
+    else
+        std::memcpy(reinterpret_cast<char*>(&out) + sizeof(DB::UInt64) - sizeof(T), &key, sizeof(T));
     return intHash64(out);
 }
 

From fcce60ad25243f128d97a264cdec4c025e67a1b3 Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <440544+davenger@users.noreply.github.com>
Date: Fri, 12 May 2023 15:09:34 +0200
Subject: [PATCH 459/689] Randomize enable_multiple_prewhere_read_steps

---
 tests/clickhouse-test | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index e279b899a93..09bc15fbe3d 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -530,6 +530,7 @@ class SettingsRandomizer:
         "max_threads": lambda: random.randint(1, 64),
         "optimize_or_like_chain": lambda: random.randint(0, 1),
         "optimize_read_in_order": lambda: random.randint(0, 1),
+        "enable_multiple_prewhere_read_steps": lambda: random.randint(0, 1),
         "read_in_order_two_level_merge_threshold": lambda: random.randint(0, 100),
         "optimize_aggregation_in_order": lambda: random.randint(0, 1),
         "aggregation_in_order_max_block_bytes": lambda: random.randint(0, 50000000),

From 0f7c0859b036ee1d9ede3da0fa0f325f7314364a Mon Sep 17 00:00:00 2001
From: DanRoscigno <dan@roscigno.com>
Date: Fri, 12 May 2023 09:22:29 -0400
Subject: [PATCH 460/689] add allow_experimental_parallel_reading_from_replicas
 docs

---
 docs/en/operations/settings/settings.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index c1d6416837d..c560b8e2ad8 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -1366,6 +1366,12 @@ Possible values:
 
 Default value: `default`.
 
+## allow_experimental_parallel_reading_from_replicas
+
+If true, ClickHouse will send a SELECT query to all replicas of a table (up to `max_parallel_replicas`) . It will work for any kind of MergeTree table.
+
+Default value: `false`.
+
 ## compile_expressions {#compile-expressions}
 
 Enables or disables compilation of frequently used simple functions and operators to native code with LLVM at runtime.

From 1e3b7af97a5845c0dd02299a40e65bad5e6468fb Mon Sep 17 00:00:00 2001
From: Arthur Passos <arthur.ti@outlook.com>
Date: Fri, 12 May 2023 10:26:05 -0300
Subject: [PATCH 461/689] Add setting to limit the max number of pairs produced
 by extractKeyValuePairs

---
 src/Common/ErrorCodes.cpp                     |  1 +
 src/Core/Settings.h                           |  1 +
 .../keyvaluepair/extractKeyValuePairs.cpp     | 13 ++++++--
 .../impl/CHKeyValuePairExtractor.h            | 15 +++++++---
 .../impl/KeyValuePairExtractorBuilder.cpp     | 14 ++++++---
 .../impl/KeyValuePairExtractorBuilder.h       |  3 ++
 ...t_key_value_pairs_multiple_input.reference | 16 ++++++++++
 ...extract_key_value_pairs_multiple_input.sql | 30 +++++++++++++++++++
 8 files changed, 82 insertions(+), 11 deletions(-)

diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp
index e9dc5649245..382b8ed8019 100644
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@@ -576,6 +576,7 @@
     M(691, UNKNOWN_ELEMENT_OF_ENUM) \
     M(692, TOO_MANY_MUTATIONS) \
     M(693, AWS_ERROR) \
+    M(694, TOO_LARGE_MAP_SIZE) \
     \
     M(999, KEEPER_EXCEPTION) \
     M(1000, POCO_EXCEPTION) \
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index f60632fae91..3d1640e21b1 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -748,6 +748,7 @@ class IColumn;
     M(Bool, optimize_distinct_in_order, false, "This optimization has a bug and it is disabled. Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
     M(Bool, allow_experimental_undrop_table_query, false, "Allow to use undrop query to restore dropped table in a limited time", 0) \
     M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \
+    M(UInt64, extract_kvp_max_pairs_per_row, 1000, "Max number pairs that can be produced by extractKeyValuePairs function. Used to safeguard against consuming too much memory.", 0) \
     // End of COMMON_SETTINGS
     // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS.
 
diff --git a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
index a1b140001ac..00588052870 100644
--- a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
+++ b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
@@ -7,6 +7,8 @@
 #include <DataTypes/DataTypeMap.h>
 #include <DataTypes/DataTypeString.h>
 
+#include <Interpreters/Context.h>
+
 #include <Functions/keyvaluepair/impl/KeyValuePairExtractor.h>
 #include <Functions/keyvaluepair/impl/KeyValuePairExtractorBuilder.h>
 #include <Functions/keyvaluepair/ArgumentExtractor.h>
@@ -41,6 +43,8 @@ class ExtractKeyValuePairs : public IFunction
             builder.withQuotingCharacter(parsed_arguments.quoting_character.value());
         }
 
+        builder.withMaxNumberOfPairs(context->getSettingsRef().extract_kvp_max_pairs_per_row);
+
         return builder.build();
     }
 
@@ -73,7 +77,7 @@ class ExtractKeyValuePairs : public IFunction
     }
 
 public:
-    ExtractKeyValuePairs() = default;
+    explicit ExtractKeyValuePairs(ContextPtr context_) : context(context_) {}
 
     static constexpr auto name = Name::name;
 
@@ -82,9 +86,9 @@ public:
         return name;
     }
 
-    static FunctionPtr create(ContextPtr)
+    static FunctionPtr create(ContextPtr context)
     {
-        return std::make_shared<ExtractKeyValuePairs>();
+        return std::make_shared<ExtractKeyValuePairs>(context);
     }
 
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
@@ -120,6 +124,9 @@ public:
     {
         return {1, 2, 3, 4};
     }
+
+private:
+    ContextPtr context;
 };
 
 struct NameExtractKeyValuePairs
diff --git a/src/Functions/keyvaluepair/impl/CHKeyValuePairExtractor.h b/src/Functions/keyvaluepair/impl/CHKeyValuePairExtractor.h
index 139417579de..24c8d3cd89a 100644
--- a/src/Functions/keyvaluepair/impl/CHKeyValuePairExtractor.h
+++ b/src/Functions/keyvaluepair/impl/CHKeyValuePairExtractor.h
@@ -13,6 +13,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
+    extern const int TOO_LARGE_MAP_SIZE;
 }
 
 /*
@@ -25,8 +26,8 @@ class CHKeyValuePairExtractor : public KeyValuePairExtractor
     using NextState = DB::extractKV::StateHandler::NextState;
 
 public:
-    explicit CHKeyValuePairExtractor(StateHandler state_handler_)
-        : state_handler(std::move(state_handler_))
+    explicit CHKeyValuePairExtractor(StateHandler state_handler_, uint64_t max_number_of_pairs_)
+        : state_handler(std::move(state_handler_)), max_number_of_pairs(max_number_of_pairs_)
     {}
 
     uint64_t extract(const std::string & data, ColumnString::MutablePtr & keys, ColumnString::MutablePtr & values) override
@@ -113,11 +114,16 @@ private:
     NextState flushPair(const std::string_view & file, auto & key,
                         auto & value, uint64_t & row_offset)
     {
+        row_offset++;
+
+        if (row_offset > max_number_of_pairs)
+        {
+            throw Exception(ErrorCodes::TOO_LARGE_MAP_SIZE, "Number of pairs produced exceeded the limit of {}", max_number_of_pairs);
+        }
+
         key.commit();
         value.commit();
 
-        row_offset++;
-
         return {0, file.empty() ? State::END : State::WAITING_KEY};
     }
 
@@ -128,6 +134,7 @@ private:
     }
 
     StateHandler state_handler;
+    uint64_t max_number_of_pairs;
 };
 
 }
diff --git a/src/Functions/keyvaluepair/impl/KeyValuePairExtractorBuilder.cpp b/src/Functions/keyvaluepair/impl/KeyValuePairExtractorBuilder.cpp
index 188285dc4bd..7f2a6449ab0 100644
--- a/src/Functions/keyvaluepair/impl/KeyValuePairExtractorBuilder.cpp
+++ b/src/Functions/keyvaluepair/impl/KeyValuePairExtractorBuilder.cpp
@@ -31,6 +31,12 @@ KeyValuePairExtractorBuilder & KeyValuePairExtractorBuilder::withEscaping()
     return *this;
 }
 
+KeyValuePairExtractorBuilder & KeyValuePairExtractorBuilder::withMaxNumberOfPairs(uint64_t max_number_of_pairs_)
+{
+    max_number_of_pairs = max_number_of_pairs_;
+    return *this;
+}
+
 std::shared_ptr<KeyValuePairExtractor> KeyValuePairExtractorBuilder::build() const
 {
     if (with_escaping)
@@ -46,9 +52,9 @@ namespace
 using namespace extractKV;
 
 template <typename T>
-auto makeStateHandler(const T && handler)
+auto makeStateHandler(const T && handler, uint64_t max_number_of_pairs)
 {
-    return std::make_shared<CHKeyValuePairExtractor<T>>(handler);
+    return std::make_shared<CHKeyValuePairExtractor<T>>(handler, max_number_of_pairs);
 }
 
 }
@@ -57,14 +63,14 @@ std::shared_ptr<KeyValuePairExtractor> KeyValuePairExtractorBuilder::buildWithou
 {
     auto configuration = ConfigurationFactory::createWithoutEscaping(key_value_delimiter, quoting_character, item_delimiters);
 
-    return makeStateHandler(NoEscapingStateHandler(configuration));
+    return makeStateHandler(NoEscapingStateHandler(configuration), max_number_of_pairs);
 }
 
 std::shared_ptr<KeyValuePairExtractor> KeyValuePairExtractorBuilder::buildWithEscaping() const
 {
     auto configuration = ConfigurationFactory::createWithEscaping(key_value_delimiter, quoting_character, item_delimiters);
 
-    return makeStateHandler(InlineEscapingStateHandler(configuration));
+    return makeStateHandler(InlineEscapingStateHandler(configuration), max_number_of_pairs);
 }
 
 }
diff --git a/src/Functions/keyvaluepair/impl/KeyValuePairExtractorBuilder.h b/src/Functions/keyvaluepair/impl/KeyValuePairExtractorBuilder.h
index 5746c58dccc..0c673f12ccf 100644
--- a/src/Functions/keyvaluepair/impl/KeyValuePairExtractorBuilder.h
+++ b/src/Functions/keyvaluepair/impl/KeyValuePairExtractorBuilder.h
@@ -20,6 +20,8 @@ public:
 
     KeyValuePairExtractorBuilder & withEscaping();
 
+    KeyValuePairExtractorBuilder & withMaxNumberOfPairs(uint64_t max_number_of_pairs_);
+
     std::shared_ptr<KeyValuePairExtractor> build() const;
 
 private:
@@ -27,6 +29,7 @@ private:
     char key_value_delimiter = ':';
     char quoting_character = '"';
     std::vector<char> item_delimiters = {' ', ',', ';'};
+    uint64_t max_number_of_pairs = std::numeric_limits<uint64_t>::max();
 
     std::shared_ptr<KeyValuePairExtractor> buildWithEscaping() const;
 
diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
index 125afa19427..ec51b61a382 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
@@ -292,6 +292,22 @@ SELECT
     x;
 {'age':'31','last_key':'last_value','name':'neymar','nationality':'brazil','team':'psg'}
 -- { echoOn }
+
+SET extract_kvp_max_pairs_per_row = 2;
+-- Should be allowed because it no longer exceeds the max number of pairs
+-- expected output: {'key1':'value1','key2':'value2'}
+WITH
+    extractKeyValuePairs('key1:value1,key2:value2') AS s_map,
+    CAST(
+            arrayMap(
+                    (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
+                ),
+            'Map(String,String)'
+        ) AS x
+SELECT
+    x;
+{'key1':'value1','key2':'value2'}
+SET extract_kvp_max_pairs_per_row = 99999;
 -- should not fail because pair delimiters contains 8 characters, which is within the limit
 WITH
     extractKeyValuePairs('not_important', ':', '12345678', '\'') AS s_map,
diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
index 0a1500b1796..7dfcae879b0 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
@@ -414,7 +414,37 @@ WITH
 SELECT
     x; -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
 
+-- Should fail allowed because it exceeds the max number of pairs
+SET extract_kvp_max_pairs_per_row = 1;
+WITH
+    extractKeyValuePairs('key1:value1,key2:value2') AS s_map,
+    CAST(
+            arrayMap(
+                    (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
+                ),
+            'Map(String,String)'
+        ) AS x
+SELECT
+    x; -- {serverError TOO_LARGE_MAP_SIZE}
+
 -- { echoOn }
+
+SET extract_kvp_max_pairs_per_row = 2;
+-- Should be allowed because it no longer exceeds the max number of pairs
+-- expected output: {'key1':'value1','key2':'value2'}
+WITH
+    extractKeyValuePairs('key1:value1,key2:value2') AS s_map,
+    CAST(
+            arrayMap(
+                    (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
+                ),
+            'Map(String,String)'
+        ) AS x
+SELECT
+    x;
+
+SET extract_kvp_max_pairs_per_row = 99999;
+
 -- should not fail because pair delimiters contains 8 characters, which is within the limit
 WITH
     extractKeyValuePairs('not_important', ':', '12345678', '\'') AS s_map,

From 224646698e2748baaf732f1d9b59e61450d9d56b Mon Sep 17 00:00:00 2001
From: DanRoscigno <dan@roscigno.com>
Date: Fri, 12 May 2023 09:30:06 -0400
Subject: [PATCH 462/689] update settings header

---
 src/Core/Settings.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 215e5b38056..99e48a5b411 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -156,7 +156,7 @@ class IColumn;
     M(ParallelReplicasCustomKeyFilterType, parallel_replicas_custom_key_filter_type, ParallelReplicasCustomKeyFilterType::DEFAULT, "Type of filter to use with custom key for parallel replicas. default - use modulo operation on the custom key, range - use range filter on custom key using all possible values for the value type of custom key.", 0) \
     \
     M(String, cluster_for_parallel_replicas, "default", "Cluster for a shard in which current server is located", 0) \
-    M(Bool, allow_experimental_parallel_reading_from_replicas, false, "If true, ClickHouse will send a SELECT query to all replicas of a table. It will work for any kind on MergeTree table.", 0) \
+    M(Bool, allow_experimental_parallel_reading_from_replicas, false, "If true, ClickHouse will send a SELECT query to all replicas of a table (up to `max_parallel_replicas`). It will work for any kind of MergeTree table.", 0) \
     M(Float, parallel_replicas_single_task_marks_count_multiplier, 2, "A multiplier which will be added during calculation for minimal number of marks to retrieve from coordinator. This will be applied only for remote replicas.", 0) \
     M(Bool, parallel_replicas_for_non_replicated_merge_tree, false, "If true, ClickHouse will use parallel replicas algorithm also for non-replicated MergeTree tables", 0) \
     \

From e72a6fe9734b6078603472454619df593ba468e4 Mon Sep 17 00:00:00 2001
From: jinjunzh <jasper.zhu@intel.com>
Date: Fri, 12 May 2023 09:56:01 -0400
Subject: [PATCH 463/689] qpl-deflate codec lower the minimum SIMD version to
 SSE 4.2

---
 contrib/CMakeLists.txt                        |   4 +-
 contrib/qpl                                   |   2 +-
 contrib/qpl-cmake/CMakeLists.txt              | 206 ++++++++++--------
 .../building_and_benchmarking_deflate_qpl.md  |   6 +-
 .../sql-reference/statements/create/table.md  |   4 +-
 5 files changed, 122 insertions(+), 100 deletions(-)

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 901df1b2432..cc2a6096554 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -178,10 +178,10 @@ add_contrib (sqlite-cmake sqlite-amalgamation)
 add_contrib (s2geometry-cmake s2geometry)
 add_contrib (c-ares-cmake c-ares)
 
-if (OS_LINUX AND ARCH_AMD64 AND (ENABLE_AVX2 OR ENABLE_AVX512))
+if (OS_LINUX AND ARCH_AMD64 AND ENABLE_SSE42)
     option (ENABLE_QPL "Enable Intel® Query Processing Library" ${ENABLE_LIBRARIES})
 elseif(ENABLE_QPL)
-    message (${RECONFIGURE_MESSAGE_LEVEL} "QPL library is only supported on x86_64 arch with avx2/avx512 support")
+    message (${RECONFIGURE_MESSAGE_LEVEL} "QPL library is only supported on x86_64 arch with SSE 4.2 or higher")
 endif()
 if (ENABLE_QPL)
     add_contrib (idxd-config-cmake idxd-config)
diff --git a/contrib/qpl b/contrib/qpl
index 0bce2b03423..3f8f5cea277 160000
--- a/contrib/qpl
+++ b/contrib/qpl
@@ -1 +1 @@
-Subproject commit 0bce2b03423f6fbeb8bce66cc8be0bf558058848
+Subproject commit 3f8f5cea27739f5261e8fd577dc233ffe88bf679
diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt
index 034106b51b6..21d665d12e4 100644
--- a/contrib/qpl-cmake/CMakeLists.txt
+++ b/contrib/qpl-cmake/CMakeLists.txt
@@ -22,8 +22,8 @@ GetLibraryVersion("${HEADER_CONTENT}" QPL_VERSION)
 message(STATUS "Intel QPL version: ${QPL_VERSION}")
 
 # There are 5 source subdirectories under $QPL_SRC_DIR: isal, c_api, core-sw, middle-layer, c_api.
-# Generate 7 library targets: middle_layer_lib, isal, isal_asm, qplcore_px, qplcore_avx512, core_iaa, middle_layer_lib.
-# Output ch_contrib::qpl by linking with 7 library targets.
+# Generate 8 library targets: middle_layer_lib, isal, isal_asm, qplcore_px, qplcore_avx512, qplcore_sw_dispatcher, core_iaa, middle_layer_lib.
+# Output ch_contrib::qpl by linking with 8 library targets.
 
 include("${QPL_PROJECT_DIR}/cmake/CompileOptions.cmake")
 
@@ -88,31 +88,36 @@ set(ISAL_ASM_SRC ${QPL_SRC_DIR}/isal/igzip/igzip_body.asm
 add_library(isal OBJECT ${ISAL_C_SRC})
 add_library(isal_asm OBJECT ${ISAL_ASM_SRC})
 
+set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS
+             $<TARGET_OBJECTS:isal>)
+
+set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS
+             $<TARGET_OBJECTS:isal_asm>)
+
 # Setting external and internal interfaces for ISA-L library
 target_include_directories(isal
-                           PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/isal/include>
-                           PRIVATE ${QPL_SRC_DIR}/isal/include
-                           PUBLIC ${QPL_SRC_DIR}/isal/igzip)
+                        PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/isal/include>
+                        PRIVATE ${QPL_SRC_DIR}/isal/include
+                        PUBLIC ${QPL_SRC_DIR}/isal/igzip)
+
+set_target_properties(isal PROPERTIES
+                        CXX_STANDARD 11
+                        C_STANDARD 99)
 
 target_compile_options(isal PRIVATE
                        "$<$<C_COMPILER_ID:GNU>:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}>"
                        "$<$<CONFIG:Debug>:>"
                        "$<$<CONFIG:Release>:>")
 
+# AS_FEATURE_LEVEL=10 means "Check SIMD capabilities of the target system at runtime and use up to AVX512 if available".
+# HAVE_KNOWS_AVX512 means rely on AVX512 being available on the target system.
 target_compile_options(isal_asm PRIVATE "-I${QPL_SRC_DIR}/isal/include/"
                        PRIVATE "-I${QPL_SRC_DIR}/isal/igzip/"
                        PRIVATE "-I${QPL_SRC_DIR}/isal/crc/"
+                       PRIVATE "-DHAVE_AS_KNOWS_AVX512"
+                       PRIVATE "-DAS_FEATURE_LEVEL=10"
                        PRIVATE "-DQPL_LIB")
 
-# AS_FEATURE_LEVEL=10 means "Check SIMD capabilities of the target system at runtime and use up to AVX512 if available".
-# AS_FEATURE_LEVEL=5 means "Check SIMD capabilities of the target system at runtime and use up to AVX2 if available".
-# HAVE_KNOWS_AVX512 means rely on AVX512 being available on the target system.
-if (ENABLE_AVX512)
-    target_compile_options(isal_asm PRIVATE "-DHAVE_AS_KNOWS_AVX512" "-DAS_FEATURE_LEVEL=10")
-else()
-    target_compile_options(isal_asm PRIVATE "-DAS_FEATURE_LEVEL=5")
-endif()
-
 # Here must remove "-fno-sanitize=undefined" from COMPILE_OPTIONS.
 # Otherwise nasm compiler would fail to proceed due to unrecognition of "-fno-sanitize=undefined"
 if (SANITIZE STREQUAL "undefined")
@@ -126,78 +131,97 @@ target_compile_definitions(isal PUBLIC
                            NDEBUG)
 
 # [SUBDIR]core-sw
-# Two libraries:qplcore_avx512/qplcore_px for SW fallback will be created which are implemented by AVX512 and non-AVX512 instructions respectively.
+# Create set of libraries corresponding to supported platforms for SW fallback which are implemented by AVX512 and non-AVX512 instructions respectively.
 # The upper level QPL API will check SIMD capabilities of the target system at runtime and decide to call AVX512 function or non-AVX512 function.
-# Hence, here we don't need put qplcore_avx512 under an ENABLE_AVX512 CMake switch. 
-# Actually, if we do that, some undefined symbols errors would happen because both of AVX512 function and non-AVX512 function are referenced by QPL API.
-# PLATFORM=2 means AVX512 implementation; PLATFORM=0 means non-AVX512 implementation.
+# Hence, here we don't need put ENABLE_AVX512 CMake switch.
 
-# Find Core Sources
-file(GLOB SOURCES
-     ${QPL_SRC_DIR}/core-sw/src/checksums/*.c
-     ${QPL_SRC_DIR}/core-sw/src/filtering/*.c
-     ${QPL_SRC_DIR}/core-sw/src/other/*.c
-     ${QPL_SRC_DIR}/core-sw/src/compression/*.c)
+get_list_of_supported_optimizations(PLATFORMS_LIST)
 
-file(GLOB DATA_SOURCES
-     ${QPL_SRC_DIR}/core-sw/src/data/*.c)
+foreach(PLATFORM_ID IN LISTS PLATFORMS_LIST)
+    # Find Core Sources
+    file(GLOB SOURCES
+        ${QPL_SRC_DIR}/core-sw/src/checksums/*.c
+        ${QPL_SRC_DIR}/core-sw/src/filtering/*.c
+        ${QPL_SRC_DIR}/core-sw/src/other/*.c
+        ${QPL_SRC_DIR}/core-sw/src/compression/*.c)
 
-# Create avx512 library
-add_library(qplcore_avx512 OBJECT ${SOURCES})
+    file(GLOB DATA_SOURCES
+        ${QPL_SRC_DIR}/core-sw/src/data/*.c)
 
-target_compile_definitions(qplcore_avx512 PRIVATE PLATFORM=2)
+    # Create library
+    add_library(qplcore_${PLATFORM_ID} OBJECT ${SOURCES})
 
-target_include_directories(qplcore_avx512
-                           PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/include>
-                           PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/src/include>
-                           PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/src/compression/include>
-                           PRIVATE $<TARGET_PROPERTY:isal,INTERFACE_INCLUDE_DIRECTORIES>)
+    set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS
+                $<TARGET_OBJECTS:qplcore_${PLATFORM_ID}>)
 
-set_target_properties(qplcore_avx512 PROPERTIES
-        $<$<C_COMPILER_ID:GNU>:C_STANDARD 17>)
+    target_include_directories(qplcore_${PLATFORM_ID}
+                                PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw>
+                                PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/include>
+                                PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/src/include>
+                                PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/src/compression/include>
+                                PRIVATE $<TARGET_PROPERTY:isal,INTERFACE_INCLUDE_DIRECTORIES>)
 
-target_link_libraries(qplcore_avx512
-        PRIVATE isal
-        PRIVATE ${CMAKE_DL_LIBS})
+    set_target_properties(qplcore_${PLATFORM_ID} PROPERTIES
+                                $<$<C_COMPILER_ID:GNU>:C_STANDARD 17>)
 
-target_compile_options(qplcore_avx512
-                        PRIVATE ${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}
-                        PRIVATE -march=skylake-avx512
-                        PRIVATE "$<$<CONFIG:Debug>:>"
-                        PRIVATE "$<$<CONFIG:Release>:-O3;-D_FORTIFY_SOURCE=2>")
+    target_compile_options(qplcore_${PLATFORM_ID}
+                            PRIVATE ${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}
+                            PRIVATE "$<$<CONFIG:Debug>:>"
+                            PRIVATE "$<$<CONFIG:Release>:-O3;-D_FORTIFY_SOURCE=2>")
 
+    # Set specific compiler options and/or definitions based on a platform
+    if (${PLATFORM_ID} MATCHES "avx512")
+        target_compile_definitions(qplcore_${PLATFORM_ID} PRIVATE PLATFORM=2)
+        target_compile_options(qplcore_${PLATFORM_ID} PRIVATE -march=skylake-avx512)
+    else() # Create default px library
+        target_compile_definitions(qplcore_${PLATFORM_ID} PRIVATE PLATFORM=0)
+    endif()
 
-target_compile_definitions(qplcore_avx512 PUBLIC QPL_BADARG_CHECK)
+    target_link_libraries(qplcore_${PLATFORM_ID} isal)
+endforeach()
 
 #
-# Create px library
+# Create dispatcher between platforms and auto-generated wrappers
 #
-#set(CMAKE_INCLUDE_CURRENT_DIR ON)
+file(GLOB SW_DISPATCHER_SOURCES ${QPL_SRC_DIR}/core-sw/dispatcher/*.cpp)
 
-# Create library
-add_library(qplcore_px OBJECT ${SOURCES} ${DATA_SOURCES})
+add_library(qplcore_sw_dispatcher OBJECT ${SW_DISPATCHER_SOURCES})
 
-target_compile_definitions(qplcore_px PRIVATE PLATFORM=0)
+set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS
+             $<TARGET_OBJECTS:qplcore_sw_dispatcher>)
 
-target_include_directories(qplcore_px
-                           PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/include>
-                           PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/src/include>
-                           PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/src/compression/include>
-                           PRIVATE $<TARGET_PROPERTY:isal,INTERFACE_INCLUDE_DIRECTORIES>)
+target_include_directories(qplcore_sw_dispatcher
+                          PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/dispatcher>)
 
-set_target_properties(qplcore_px PROPERTIES
-        $<$<C_COMPILER_ID:GNU>:C_STANDARD 17>)
+# Generate kernel wrappers
+generate_unpack_kernel_arrays(${QPL_BINARY_DIR} "${PLATFORMS_LIST}")
 
-target_link_libraries(qplcore_px
-        PRIVATE isal
-        PRIVATE ${CMAKE_DL_LIBS})
+foreach(PLATFORM_ID IN LISTS PLATFORMS_LIST)
+    file(GLOB GENERATED_${PLATFORM_ID}_TABLES_SRC ${QPL_BINARY_DIR}/generated/${PLATFORM_ID}_*.cpp)
 
-target_compile_options(qplcore_px
-                        PRIVATE ${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}
-                        PRIVATE "$<$<CONFIG:Debug>:>"
-                        PRIVATE "$<$<CONFIG:Release>:-O3;-D_FORTIFY_SOURCE=2>")
+    target_sources(qplcore_sw_dispatcher PRIVATE ${GENERATED_${PLATFORM_ID}_TABLES_SRC})
 
-target_compile_definitions(qplcore_px PUBLIC QPL_BADARG_CHECK)
+    # Set specific compiler options and/or definitions based on a platform
+    if (${PLATFORM_ID} MATCHES "avx512")
+        set_source_files_properties(${GENERATED_${PLATFORM_ID}_TABLES_SRC} PROPERTIES COMPILE_DEFINITIONS PLATFORM=2)
+    else()
+        set_source_files_properties(${GENERATED_${PLATFORM_ID}_TABLES_SRC} PROPERTIES COMPILE_DEFINITIONS PLATFORM=0)
+    endif()
+
+    target_include_directories(qplcore_sw_dispatcher
+                                PUBLIC $<TARGET_PROPERTY:qplcore_${PLATFORM_ID},INTERFACE_INCLUDE_DIRECTORIES>)
+endforeach()
+
+set_target_properties(qplcore_sw_dispatcher PROPERTIES CXX_STANDARD 17)
+
+# w/a for build compatibility with ISAL codebase
+target_compile_definitions(qplcore_sw_dispatcher PUBLIC -DQPL_LIB)
+
+target_compile_options(qplcore_sw_dispatcher
+        PRIVATE $<$<C_COMPILER_ID:GNU>:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS};
+                                       ${QPL_LINUX_TOOLCHAIN_DYNAMIC_LIBRARY_FLAGS};
+                                       $<$<CONFIG:Release>:-O3;-D_FORTIFY_SOURCE=2>>
+        PRIVATE $<$<COMPILE_LANG_AND_ID:CXX,GNU>:${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}>)
 
 # [SUBDIR]core-iaa
 file(GLOB HW_PATH_SRC ${QPL_SRC_DIR}/core-iaa/sources/aecs/*.c
@@ -211,13 +235,20 @@ file(GLOB HW_PATH_SRC ${QPL_SRC_DIR}/core-iaa/sources/aecs/*.c
 # Create library
 add_library(core_iaa OBJECT ${HW_PATH_SRC})
 
+set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS
+             $<TARGET_OBJECTS:core_iaa>)
+
 target_include_directories(core_iaa
         PRIVATE ${UUID_DIR}
         PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-iaa/include>
         PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-iaa/sources/include>
         PRIVATE $<BUILD_INTERFACE:${QPL_PROJECT_DIR}/include>  # status.h in own_checkers.h
         PRIVATE $<BUILD_INTERFACE:${QPL_PROJECT_DIR}/sources/c_api> # own_checkers.h
-        PRIVATE $<TARGET_PROPERTY:qplcore_avx512,INTERFACE_INCLUDE_DIRECTORIES>)
+        PRIVATE $<TARGET_PROPERTY:qplcore_sw_dispatcher,INTERFACE_INCLUDE_DIRECTORIES>)
+
+set_target_properties(core_iaa PROPERTIES
+        $<$<C_COMPILER_ID:GNU>:C_STANDARD 17>
+        CXX_STANDARD 17)
 
 target_compile_options(core_iaa
         PRIVATE $<$<C_COMPILER_ID:GNU>:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS};
@@ -227,11 +258,10 @@ target_compile_features(core_iaa PRIVATE c_std_11)
 
 target_compile_definitions(core_iaa PRIVATE QPL_BADARG_CHECK
         PRIVATE $<$<BOOL:${BLOCK_ON_FAULT}>: BLOCK_ON_FAULT_ENABLED>
-        PRIVATE $<$<BOOL:${LOG_HW_INIT}>:LOG_HW_INIT>)
+        PRIVATE $<$<BOOL:${LOG_HW_INIT}>:LOG_HW_INIT>
+        PRIVATE $<$<BOOL:${DYNAMIC_LOADING_LIBACCEL_CONFIG}>:DYNAMIC_LOADING_LIBACCEL_CONFIG>)
 
 # [SUBDIR]middle-layer
-generate_unpack_kernel_arrays(${QPL_BINARY_DIR})
-
 file(GLOB MIDDLE_LAYER_SRC
         ${QPL_SRC_DIR}/middle-layer/analytics/*.cpp
         ${QPL_SRC_DIR}/middle-layer/c_wrapper/*.cpp
@@ -246,14 +276,12 @@ file(GLOB MIDDLE_LAYER_SRC
         ${QPL_SRC_DIR}/middle-layer/inflate/*.cpp
         ${QPL_SRC_DIR}/core-iaa/sources/accelerator/*.cpp) # todo
 
-file(GLOB GENERATED_PX_TABLES_SRC ${QPL_BINARY_DIR}/generated/px_*.cpp)
-file(GLOB GENERATED_AVX512_TABLES_SRC ${QPL_BINARY_DIR}/generated/avx512_*.cpp)
-
 add_library(middle_layer_lib OBJECT
-        ${GENERATED_PX_TABLES_SRC}
-        ${GENERATED_AVX512_TABLES_SRC}
         ${MIDDLE_LAYER_SRC})
 
+set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS
+        $<TARGET_OBJECTS:middle_layer_lib>)
+
 target_compile_options(middle_layer_lib
         PRIVATE $<$<C_COMPILER_ID:GNU>:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS};
                                        ${QPL_LINUX_TOOLCHAIN_DYNAMIC_LIBRARY_FLAGS};
@@ -264,17 +292,16 @@ target_compile_definitions(middle_layer_lib
         PUBLIC QPL_VERSION="${QPL_VERSION}"
         PUBLIC $<$<BOOL:${LOG_HW_INIT}>:LOG_HW_INIT>
         PUBLIC $<$<BOOL:${EFFICIENT_WAIT}>:QPL_EFFICIENT_WAIT>
-        PUBLIC QPL_BADARG_CHECK)
+        PUBLIC QPL_BADARG_CHECK
+        PUBLIC $<$<BOOL:${DYNAMIC_LOADING_LIBACCEL_CONFIG}>:DYNAMIC_LOADING_LIBACCEL_CONFIG>)
 
-set_source_files_properties(${GENERATED_PX_TABLES_SRC} PROPERTIES COMPILE_DEFINITIONS PLATFORM=0)
-set_source_files_properties(${GENERATED_AVX512_TABLES_SRC} PROPERTIES COMPILE_DEFINITIONS PLATFORM=2)
+set_target_properties(middle_layer_lib PROPERTIES CXX_STANDARD 17)
 
 target_include_directories(middle_layer_lib
         PRIVATE ${UUID_DIR}
         PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/middle-layer>
         PUBLIC $<TARGET_PROPERTY:_qpl,INTERFACE_INCLUDE_DIRECTORIES>
-        PUBLIC $<TARGET_PROPERTY:qplcore_px,INTERFACE_INCLUDE_DIRECTORIES>
-        PUBLIC $<TARGET_PROPERTY:qplcore_avx512,INTERFACE_INCLUDE_DIRECTORIES>
+        PUBLIC $<TARGET_PROPERTY:qplcore_sw_dispatcher,INTERFACE_INCLUDE_DIRECTORIES>
         PUBLIC $<TARGET_PROPERTY:isal,INTERFACE_INCLUDE_DIRECTORIES>
         PUBLIC $<TARGET_PROPERTY:core_iaa,INTERFACE_INCLUDE_DIRECTORIES>)
 
@@ -285,20 +312,19 @@ file(GLOB_RECURSE QPL_C_API_SRC
         ${QPL_SRC_DIR}/c_api/*.c
         ${QPL_SRC_DIR}/c_api/*.cpp)
 
-add_library(_qpl STATIC ${QPL_C_API_SRC}
-         $<TARGET_OBJECTS:middle_layer_lib>
-        $<TARGET_OBJECTS:isal>
-        $<TARGET_OBJECTS:isal_asm>
-        $<TARGET_OBJECTS:qplcore_px>
-        $<TARGET_OBJECTS:qplcore_avx512>
-        $<TARGET_OBJECTS:core_iaa>
-        $<TARGET_OBJECTS:middle_layer_lib>)
+get_property(LIB_DEPS GLOBAL PROPERTY QPL_LIB_DEPS)
+
+add_library(_qpl STATIC ${QPL_C_API_SRC} ${LIB_DEPS})
 
 target_include_directories(_qpl
-        PUBLIC $<BUILD_INTERFACE:${QPL_PROJECT_DIR}/include/>
+        PUBLIC $<BUILD_INTERFACE:${QPL_PROJECT_DIR}/include/> $<INSTALL_INTERFACE:include>
         PRIVATE $<TARGET_PROPERTY:middle_layer_lib,INTERFACE_INCLUDE_DIRECTORIES>
         PRIVATE $<BUILD_INTERFACE:${QPL_SRC_DIR}/c_api>)
 
+set_target_properties(_qpl PROPERTIES
+        $<$<C_COMPILER_ID:GNU>:C_STANDARD 17>
+        CXX_STANDARD 17)
+
 target_compile_options(_qpl
         PRIVATE $<$<C_COMPILER_ID:GNU>:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS};
                                        ${QPL_LINUX_TOOLCHAIN_DYNAMIC_LIBRARY_FLAGS};
@@ -308,12 +334,12 @@ target_compile_options(_qpl
 target_compile_definitions(_qpl
         PRIVATE -DQPL_LIB
         PRIVATE -DQPL_BADARG_CHECK
+        PRIVATE $<$<BOOL:${DYNAMIC_LOADING_LIBACCEL_CONFIG}>:DYNAMIC_LOADING_LIBACCEL_CONFIG>
         PUBLIC -DENABLE_QPL_COMPRESSION)
 
 target_link_libraries(_qpl
         PRIVATE ch_contrib::accel-config
-        PRIVATE ch_contrib::isal
-        PRIVATE ${CMAKE_DL_LIBS})
+        PRIVATE ch_contrib::isal)
 
 target_include_directories(_qpl SYSTEM BEFORE
         PUBLIC "${QPL_PROJECT_DIR}/include"
diff --git a/docs/en/development/building_and_benchmarking_deflate_qpl.md b/docs/en/development/building_and_benchmarking_deflate_qpl.md
index 60d6b1c7b76..a492a30abae 100644
--- a/docs/en/development/building_and_benchmarking_deflate_qpl.md
+++ b/docs/en/development/building_and_benchmarking_deflate_qpl.md
@@ -8,11 +8,7 @@ description: How to build Clickhouse and run benchmark with DEFLATE_QPL Codec
 - Make sure your target machine meet the QPL required [Prerequisites](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#prerequisites)
 - Pass the following flag to CMake when building ClickHouse, depending on the capabilities of your target machine:
 ``` bash
-cmake -DENABLE_AVX2=1 -DENABLE_QPL=1 ..
-```
-or
-``` bash
-cmake -DENABLE_AVX512=1 -DENABLE_QPL=1 ..
+cmake -DENABLE_SSE42=1 -DENABLE_QPL=1 ..
 ```
 - For generic requirements, please refer to Clickhouse generic [build instructions](/docs/en/development/build.md)
 
diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md
index f8f031e1551..2011d12131c 100644
--- a/docs/en/sql-reference/statements/create/table.md
+++ b/docs/en/sql-reference/statements/create/table.md
@@ -381,9 +381,9 @@ High compression levels are useful for asymmetric scenarios, like compress once,
 `DEFLATE_QPL` — [Deflate compression algorithm](https://github.com/intel/qpl) implemented by Intel® Query Processing Library. Some limitations apply:
 
 - DEFLATE_QPL is experimental and can only be used after setting configuration parameter `allow_experimental_codecs=1`.
-- DEFLATE_QPL only works if ClickHouse was compiled with support for AVX2 or AVX512 instructions. Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details.
+- DEFLATE_QPL requires build ClickHouse with SSE 4.2 enabled. Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details.
 - DEFLATE_QPL works best if the system has a Intel® IAA (In-Memory Analytics Accelerator) offloading device. Refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration) and [Benchmark with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Run-Benchmark-with-DEFLATE_QPL) for more details.
-- DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with support for AVX2/AVX512
+- DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with SSE 4.2 enabled.
 
 ### Specialized Codecs
 

From 5fdaef851c8787d7d2bfb4a25c9729d986b8005b Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 12 May 2023 14:14:40 +0000
Subject: [PATCH 464/689] Handle constants carefully

---
 src/Analyzer/FunctionNode.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Analyzer/FunctionNode.cpp b/src/Analyzer/FunctionNode.cpp
index 2385531db4f..f5bcdc103d2 100644
--- a/src/Analyzer/FunctionNode.cpp
+++ b/src/Analyzer/FunctionNode.cpp
@@ -210,7 +210,8 @@ ASTPtr FunctionNode::toASTImpl(const ConvertToASTOptions & options) const
     }
 
     auto new_options = options;
-    if (function_name == "_CAST")
+    /// To avoid surrounding constants with several internal casts.
+    if (function_name == "_CAST" && (*getArguments().begin())->getNodeType() == QueryTreeNodeType::CONSTANT)
         new_options.add_cast_for_constants = false;
 
     const auto & parameters = getParameters();

From b8b4723aa627f5487f954b76a29690a3941274a3 Mon Sep 17 00:00:00 2001
From: DanRoscigno <dan@roscigno.com>
Date: Fri, 12 May 2023 10:19:30 -0400
Subject: [PATCH 465/689] add max_rows_in_set_to_optimize_join docs

---
 docs/en/operations/settings/settings.md | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index c1d6416837d..cddde2090f8 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -608,6 +608,17 @@ See also:
 
 - [JOIN strictness](../../sql-reference/statements/select/join.md/#join-settings)
 
+## max_rows_in_set_to_optimize_join
+
+Maximal size of the set to filter joined tables by each other's row sets before joining.
+
+Possible values:
+
+- 0 — Disable.
+- Any positive integer.
+
+Default value: 100000.
+
 ## temporary_files_codec {#temporary_files_codec}
 
 Sets compression codec for temporary files used in sorting and joining operations on disk.

From b1549a19a50048a93f5cc0b48c482160fcb66613 Mon Sep 17 00:00:00 2001
From: Arthur Passos <arthur.ti@outlook.com>
Date: Fri, 12 May 2023 11:19:35 -0300
Subject: [PATCH 466/689] Use 0 as unlimited

---
 .../keyvaluepair/extractKeyValuePairs.cpp         |  7 ++++++-
 ...tract_key_value_pairs_multiple_input.reference | 15 ++++++++++++++-
 ...499_extract_key_value_pairs_multiple_input.sql | 14 +++++++++++++-
 3 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
index 00588052870..9956eaf01f1 100644
--- a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
+++ b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
@@ -43,7 +43,12 @@ class ExtractKeyValuePairs : public IFunction
             builder.withQuotingCharacter(parsed_arguments.quoting_character.value());
         }
 
-        builder.withMaxNumberOfPairs(context->getSettingsRef().extract_kvp_max_pairs_per_row);
+        bool is_number_of_pairs_unlimited = context->getSettingsRef().extract_kvp_max_pairs_per_row == 0;
+
+        if (!is_number_of_pairs_unlimited)
+        {
+            builder.withMaxNumberOfPairs(context->getSettingsRef().extract_kvp_max_pairs_per_row);
+        }
 
         return builder.build();
     }
diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
index ec51b61a382..a4ff1464fb8 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
@@ -307,7 +307,20 @@ WITH
 SELECT
     x;
 {'key1':'value1','key2':'value2'}
-SET extract_kvp_max_pairs_per_row = 99999;
+SET extract_kvp_max_pairs_per_row = 0;
+-- Should be allowed because max pairs per row is set to 0 (unlimited)
+-- expected output: {'key1':'value1','key2':'value2'}
+WITH
+    extractKeyValuePairs('key1:value1,key2:value2') AS s_map,
+    CAST(
+            arrayMap(
+                    (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
+                ),
+            'Map(String,String)'
+        ) AS x
+SELECT
+    x;
+{'key1':'value1','key2':'value2'}
 -- should not fail because pair delimiters contains 8 characters, which is within the limit
 WITH
     extractKeyValuePairs('not_important', ':', '12345678', '\'') AS s_map,
diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
index 7dfcae879b0..8c5f0365cc7 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
@@ -443,7 +443,19 @@ WITH
 SELECT
     x;
 
-SET extract_kvp_max_pairs_per_row = 99999;
+SET extract_kvp_max_pairs_per_row = 0;
+-- Should be allowed because max pairs per row is set to 0 (unlimited)
+-- expected output: {'key1':'value1','key2':'value2'}
+WITH
+    extractKeyValuePairs('key1:value1,key2:value2') AS s_map,
+    CAST(
+            arrayMap(
+                    (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
+                ),
+            'Map(String,String)'
+        ) AS x
+SELECT
+    x;
 
 -- should not fail because pair delimiters contains 8 characters, which is within the limit
 WITH

From d2e185d86df4f7e6a7b19134d7de2f960e61a644 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 12 May 2023 14:36:54 +0000
Subject: [PATCH 467/689] Analyzer: do not optimize GROUP BY keys with ROLLUP
 and CUBE

---
 src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp   | 3 +++
 .../queries/0_stateless/02734_optimize_group_by.reference | 8 ++++++++
 tests/queries/0_stateless/02734_optimize_group_by.sql     | 7 +++++++
 3 files changed, 18 insertions(+)
 create mode 100644 tests/queries/0_stateless/02734_optimize_group_by.reference
 create mode 100644 tests/queries/0_stateless/02734_optimize_group_by.sql

diff --git a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp
index c97645219da..5ed52f1210b 100644
--- a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp
+++ b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp
@@ -38,6 +38,9 @@ public:
         if (!query->hasGroupBy())
             return;
 
+        if (query->isGroupByWithCube() || query->isGroupByWithRollup())
+            return;
+
         auto & group_by = query->getGroupBy().getNodes();
         if (query->isGroupByWithGroupingSets())
         {
diff --git a/tests/queries/0_stateless/02734_optimize_group_by.reference b/tests/queries/0_stateless/02734_optimize_group_by.reference
new file mode 100644
index 00000000000..3f5ef03cb61
--- /dev/null
+++ b/tests/queries/0_stateless/02734_optimize_group_by.reference
@@ -0,0 +1,8 @@
+a	b
+a	b
+a	b
+	b
+a	b
+a	b
+a	b
+a	b
diff --git a/tests/queries/0_stateless/02734_optimize_group_by.sql b/tests/queries/0_stateless/02734_optimize_group_by.sql
new file mode 100644
index 00000000000..28e86c04b0f
--- /dev/null
+++ b/tests/queries/0_stateless/02734_optimize_group_by.sql
@@ -0,0 +1,7 @@
+SELECT 'a' AS key, 'b' as value GROUP BY key WITH CUBE SETTINGS allow_experimental_analyzer = 0;
+SELECT 'a' AS key, 'b' as value GROUP BY key WITH CUBE SETTINGS allow_experimental_analyzer = 1;
+
+SELECT 'a' AS key, 'b' as value GROUP BY ignore(1) WITH CUBE;
+
+SELECT 'a' AS key, 'b' as value GROUP BY ignore(1);
+SELECT 'a' AS key, 'b' as value GROUP BY key;

From 5235f52de3f767b72c8358af31a79fa966cf1d8f Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 12 May 2023 16:38:32 +0200
Subject: [PATCH 468/689] Small rewrite, removed note about farm hash (it
 seemed fragile:
 https://github.com/ClickHouse/ClickHouse/issues/8354#issuecomment-974808920)

---
 docs/en/sql-reference/functions/hash-functions.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md
index 802ad31e5f1..30bb63614ae 100644
--- a/docs/en/sql-reference/functions/hash-functions.md
+++ b/docs/en/sql-reference/functions/hash-functions.md
@@ -279,7 +279,7 @@ cityHash64(par1,...)
 
 This is a fast non-cryptographic hash function. It uses the CityHash algorithm for string parameters and implementation-specific fast non-cryptographic hash function for parameters with other data types. The function uses the CityHash combinator to get the final results.
 
-Note that Google CityHash has changed the algorithm after it have been added to ClickHouse. Now results of Google CityHash and ClickHouse cityHash64 are different. ClickHouse cityHash64 results are consistent with CityHash 1.0.2. Currently [farmhash64](#farmhash64) is consistent with Google CityHash.
+Note that Google changed the algorithm of CityHash after it has been added to ClickHouse. In other words, ClickHouse's cityHash64 and Google's upstrea CityHash now produce different results. ClickHouse cityHash64 corresponds to CityHash v1.0.2.
 
 **Arguments**
 

From 195eafcaed3ab9d49a18c0d15f66aac65d7bdf72 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 12 May 2023 14:51:33 +0000
Subject: [PATCH 469/689] fix disable test

---
 src/Common/tests/gtest_async_loader.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp
index 5b04e67e763..5666c4b923e 100644
--- a/src/Common/tests/gtest_async_loader.cpp
+++ b/src/Common/tests/gtest_async_loader.cpp
@@ -387,7 +387,7 @@ TEST(AsyncLoader, CancelExecutingTask)
     }
 }
 
-TEST(AsyncLoader, DISABLEDJobFailure)
+TEST(AsyncLoader, DISABLED_JobFailure)
 {
     AsyncLoaderTest t;
     t.loader.start();

From 5133597661672faa8d4f4c7ffac92f74336273cf Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 12 May 2023 16:51:47 +0200
Subject: [PATCH 470/689] +m

---
 docs/en/sql-reference/functions/hash-functions.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md
index 30bb63614ae..635c8f4e0ec 100644
--- a/docs/en/sql-reference/functions/hash-functions.md
+++ b/docs/en/sql-reference/functions/hash-functions.md
@@ -279,7 +279,7 @@ cityHash64(par1,...)
 
 This is a fast non-cryptographic hash function. It uses the CityHash algorithm for string parameters and implementation-specific fast non-cryptographic hash function for parameters with other data types. The function uses the CityHash combinator to get the final results.
 
-Note that Google changed the algorithm of CityHash after it has been added to ClickHouse. In other words, ClickHouse's cityHash64 and Google's upstrea CityHash now produce different results. ClickHouse cityHash64 corresponds to CityHash v1.0.2.
+Note that Google changed the algorithm of CityHash after it has been added to ClickHouse. In other words, ClickHouse's cityHash64 and Google's upstream CityHash now produce different results. ClickHouse cityHash64 corresponds to CityHash v1.0.2.
 
 **Arguments**
 

From eb6cbe63ee807ca10f1eea44577b2bb0a9038fa8 Mon Sep 17 00:00:00 2001
From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com>
Date: Fri, 12 May 2023 11:54:53 -0300
Subject: [PATCH 471/689] Minor correction to example in COMMENT COLUMN

---
 docs/en/sql-reference/statements/alter/column.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md
index ff55f700023..378f41c1199 100644
--- a/docs/en/sql-reference/statements/alter/column.md
+++ b/docs/en/sql-reference/statements/alter/column.md
@@ -132,7 +132,7 @@ Comments are stored in the `comment_expression` column returned by the [DESCRIBE
 Example:
 
 ``` sql
-ALTER TABLE visits COMMENT COLUMN browser 'The table shows the browser used for accessing the site.'
+ALTER TABLE visits COMMENT COLUMN browser 'This column shows the browser used for accessing the site.'
 ```
 
 ## MODIFY COLUMN

From 37e4c531ba5ff9b639830f32c6a4b8cd9a42d9d6 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Fri, 12 May 2023 15:02:37 +0000
Subject: [PATCH 472/689] Don't update contrib

---
 contrib/boost | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/boost b/contrib/boost
index 1035c8bfcc9..8fe7b3326ef 160000
--- a/contrib/boost
+++ b/contrib/boost
@@ -1 +1 @@
-Subproject commit 1035c8bfcc9a3c1cfa7f6e827db94dae1ce1a43a
+Subproject commit 8fe7b3326ef482ee6ecdf5a4f698f2b8c2780f98

From 2960a0420b936ef5c7c7d12eb5b4f768bfd23cd2 Mon Sep 17 00:00:00 2001
From: nellicus <tony.bonuccelli@clickhouse.com>
Date: Fri, 12 May 2023 17:58:41 +0200
Subject: [PATCH 473/689] strongly recommend ded keeper in prod

---
 docs/en/getting-started/install.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md
index 308e3197ad7..484ccf364f1 100644
--- a/docs/en/getting-started/install.md
+++ b/docs/en/getting-started/install.md
@@ -143,8 +143,9 @@ You can also download and install packages manually from [here](https://packages
 #### Install standalone ClickHouse Keeper
 
 :::tip
-If you are going to run ClickHouse Keeper on the same server as ClickHouse server you
-do not need to install ClickHouse Keeper as it is included with ClickHouse server.  This command is only needed on standalone ClickHouse Keeper servers.
+In production environment we [strongly recommend](https://github.com/nellicus/clickhouse-docs/blob/main/docs/en/operations/tips.md#L143-L144) running ClickHouse Keeper on dedicated nodes.
+In test environments, if you decide to run ClickHouse Server and ClickHouse Keeper on the same server,  you do not need to install ClickHouse Keeper as it is included with ClickHouse server.
+This command is only needed on standalone ClickHouse Keeper servers.
 :::
 
 ```bash
@@ -211,8 +212,9 @@ clickhouse-client # or "clickhouse-client --password" if you set up a password.
 #### Install standalone ClickHouse Keeper
 
 :::tip
-If you are going to run ClickHouse Keeper on the same server as ClickHouse server you
-do not need to install ClickHouse Keeper as it is included with ClickHouse server.  This command is only needed on standalone ClickHouse Keeper servers.
+In production environment we [strongly recommend](https://github.com/nellicus/clickhouse-docs/blob/main/docs/en/operations/tips.md#L143-L144) running ClickHouse Keeper on dedicated nodes.
+In test environments, if you decide to run ClickHouse Server and ClickHouse Keeper on the same server,  you do not need to install ClickHouse Keeper as it is included with ClickHouse server.
+This command is only needed on standalone ClickHouse Keeper servers.
 :::
 
 ```bash

From 5366da8bcf8be16801bbfc473524882a6f8e2e1f Mon Sep 17 00:00:00 2001
From: nellicus <tony.bonuccelli@clickhouse.com>
Date: Fri, 12 May 2023 18:07:38 +0200
Subject: [PATCH 474/689] fix links

---
 docs/en/getting-started/install.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md
index 484ccf364f1..3f6c2577c94 100644
--- a/docs/en/getting-started/install.md
+++ b/docs/en/getting-started/install.md
@@ -143,7 +143,7 @@ You can also download and install packages manually from [here](https://packages
 #### Install standalone ClickHouse Keeper
 
 :::tip
-In production environment we [strongly recommend](https://github.com/nellicus/clickhouse-docs/blob/main/docs/en/operations/tips.md#L143-L144) running ClickHouse Keeper on dedicated nodes.
+In production environment we [strongly recommend](/docs/en/operations/tips.md#L143-L144) running ClickHouse Keeper on dedicated nodes.
 In test environments, if you decide to run ClickHouse Server and ClickHouse Keeper on the same server,  you do not need to install ClickHouse Keeper as it is included with ClickHouse server.
 This command is only needed on standalone ClickHouse Keeper servers.
 :::
@@ -212,7 +212,7 @@ clickhouse-client # or "clickhouse-client --password" if you set up a password.
 #### Install standalone ClickHouse Keeper
 
 :::tip
-In production environment we [strongly recommend](https://github.com/nellicus/clickhouse-docs/blob/main/docs/en/operations/tips.md#L143-L144) running ClickHouse Keeper on dedicated nodes.
+In production environment we [strongly recommend](/docs/en/operations/tips.md#L143-L144) running ClickHouse Keeper on dedicated nodes.
 In test environments, if you decide to run ClickHouse Server and ClickHouse Keeper on the same server,  you do not need to install ClickHouse Keeper as it is included with ClickHouse server.
 This command is only needed on standalone ClickHouse Keeper servers.
 :::

From d3c408aaf16356f59f91e95dac0b628e858f4640 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 12 May 2023 16:32:23 +0000
Subject: [PATCH 475/689] ClearableHashSet w/o zero storage for StringRef

---
 src/Common/HashTable/ClearableHashSet.h   | 38 -----------------------
 src/Common/HashTable/HashTableKeyHolder.h |  2 +-
 2 files changed, 1 insertion(+), 39 deletions(-)

diff --git a/src/Common/HashTable/ClearableHashSet.h b/src/Common/HashTable/ClearableHashSet.h
index 006d45df7cd..13067c39d1b 100644
--- a/src/Common/HashTable/ClearableHashSet.h
+++ b/src/Common/HashTable/ClearableHashSet.h
@@ -48,30 +48,6 @@ struct ClearableHashTableCell : public BaseCell
     ClearableHashTableCell(const Key & key_, const State & state) : BaseCell(key_, state), version(state.version) {}
 };
 
-using StringRefBaseCell = HashSetCellWithSavedHash<StringRef, DefaultHash<StringRef>, ClearableHashSetState>;
-
-/// specialization for StringRef to allow zero size key (empty string)
-template <>
-struct ClearableHashTableCell<StringRef, StringRefBaseCell> : public StringRefBaseCell
-{
-    using State = ClearableHashSetState;
-    using value_type = typename StringRefBaseCell::value_type;
-
-    UInt32 version;
-
-    bool isZero(const State & state) const { return version != state.version; }
-    static bool isZero(const StringRef & key_, const State & state_) { return StringRefBaseCell::isZero(key_, state_); }
-
-    /// Set the key value to zero.
-    void setZero() { version = 0; }
-
-    /// Do I need to store the zero key separately (that is, can a zero key be inserted into the hash table).
-    static constexpr bool need_zero_value_storage = true;
-
-    ClearableHashTableCell() { } /// NOLINT
-    ClearableHashTableCell(const StringRef & key_, const State & state) : StringRefBaseCell(key_, state), version(state.version) { }
-};
-
 template <
     typename Key,
     typename Hash = DefaultHash<Key>,
@@ -90,13 +66,6 @@ public:
     {
         ++this->version;
         this->m_size = 0;
-
-        if constexpr (Cell::need_zero_value_storage)
-        {
-            /// clear ZeroValueStorage
-            if (this->hasZero())
-                this->clearHasZero();
-        }
     }
 };
 
@@ -119,13 +88,6 @@ public:
     {
         ++this->version;
         this->m_size = 0;
-
-        if constexpr (Cell::need_zero_value_storage)
-        {
-            /// clear ZeroValueStorage
-            if (this->hasZero())
-                this->clearHasZero();
-        }
     }
 };
 
diff --git a/src/Common/HashTable/HashTableKeyHolder.h b/src/Common/HashTable/HashTableKeyHolder.h
index d10ad093af6..994587e67ec 100644
--- a/src/Common/HashTable/HashTableKeyHolder.h
+++ b/src/Common/HashTable/HashTableKeyHolder.h
@@ -89,7 +89,7 @@ inline StringRef & ALWAYS_INLINE keyHolderGetKey(DB::ArenaKeyHolder & holder)
 inline void ALWAYS_INLINE keyHolderPersistKey(DB::ArenaKeyHolder & holder)
 {
     // Hash table shouldn't ask us to persist a zero key
-    assert(holder.key.size > 0);
+    // assert(holder.key.size > 0);
     holder.key.data = holder.pool.insert(holder.key.data, holder.key.size);
 }
 

From 8c7b63f1e760b0297469bdcfd8b19eb195d943dc Mon Sep 17 00:00:00 2001
From: MikhailBurdukov <burdukvmikhail@gmail.com>
Date: Fri, 12 May 2023 16:42:01 +0000
Subject: [PATCH 476/689] Fix

---
 src/Storages/MergeTree/KeyCondition.cpp                |  2 +-
 .../02731_in_operator_with_one_size_tuple.reference    |  2 ++
 .../02731_in_operator_with_one_size_tuple.sql          | 10 ++++++++++
 3 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/02731_in_operator_with_one_size_tuple.reference
 create mode 100644 tests/queries/0_stateless/02731_in_operator_with_one_size_tuple.sql

diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp
index ed686a0b38b..b8ef2152a99 100644
--- a/src/Storages/MergeTree/KeyCondition.cpp
+++ b/src/Storages/MergeTree/KeyCondition.cpp
@@ -1183,7 +1183,7 @@ bool KeyCondition::tryPrepareSetIndex(
         /// Note: in case of ActionsDAG, tuple may be a constant.
         /// In this case, there is no keys in tuple. So, we don't have to check it.
         auto left_arg_tuple = left_arg.toFunctionNode();
-        if (left_arg_tuple.getFunctionName() == "tuple")
+        if (left_arg_tuple.getFunctionName() == "tuple" && left_arg_tuple.getArgumentsSize() > 1)
         {
             left_args_count = left_arg_tuple.getArgumentsSize();
             for (size_t i = 0; i < left_args_count; ++i)
diff --git a/tests/queries/0_stateless/02731_in_operator_with_one_size_tuple.reference b/tests/queries/0_stateless/02731_in_operator_with_one_size_tuple.reference
new file mode 100644
index 00000000000..d7ede115bc2
--- /dev/null
+++ b/tests/queries/0_stateless/02731_in_operator_with_one_size_tuple.reference
@@ -0,0 +1,2 @@
+2023-04-17	1
+2023-04-17	1
diff --git a/tests/queries/0_stateless/02731_in_operator_with_one_size_tuple.sql b/tests/queries/0_stateless/02731_in_operator_with_one_size_tuple.sql
new file mode 100644
index 00000000000..eab7d24a92e
--- /dev/null
+++ b/tests/queries/0_stateless/02731_in_operator_with_one_size_tuple.sql
@@ -0,0 +1,10 @@
+CREATE TABLE test(`report_date` Date, `sspid` UInt64) ENGINE MergeTree PARTITION BY report_date ORDER BY report_date;
+
+INSERT INTO test SELECT toDate('2023-04-20'), 0;
+INSERT INTO test SELECT toDate('2023-04-19'), 0;
+INSERT INTO test SELECT toDate('2023-04-17'), 1;
+INSERT INTO test SELECT toDate('2023-04-17'), 1;
+
+
+SELECT * FROM test  WHERE tuple(report_date) IN tuple(toDate('2023-04-17'));
+DROP TABLE test;
\ No newline at end of file

From 18e1e508898f61668afd8e63422ce9ae070ed4a9 Mon Sep 17 00:00:00 2001
From: Eridanus <maodingxuan@foxmail.com>
Date: Sat, 13 May 2023 00:47:41 +0800
Subject: [PATCH 477/689] test(parser): remove confusing quota overflow test

---
 .../0_stateless/01297_create_quota.reference      |  3 ---
 tests/queries/0_stateless/01297_create_quota.sql  | 15 ---------------
 2 files changed, 18 deletions(-)

diff --git a/tests/queries/0_stateless/01297_create_quota.reference b/tests/queries/0_stateless/01297_create_quota.reference
index 3ff2d9bfec6..6b844e0d2f6 100644
--- a/tests/queries/0_stateless/01297_create_quota.reference
+++ b/tests/queries/0_stateless/01297_create_quota.reference
@@ -85,9 +85,6 @@ CREATE QUOTA q16_01297 FOR INTERVAL 1 minute MAX execution_time = 1.5
 -- overflow test
 CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 18446744073709551615
 CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 18446744073
--- FIXME
--- the overflow happens somewhere else before reaching the up bound of Quota Value , not in the parser
--- the overflow have different behavior in different environment
 -- zero test
 CREATE QUOTA q1_01297 FOR INTERVAL 1 minute MAX query_selects = 0
 CREATE QUOTA q2_01297 FOR INTERVAL 1 minute MAX execution_time = 0
diff --git a/tests/queries/0_stateless/01297_create_quota.sql b/tests/queries/0_stateless/01297_create_quota.sql
index 1f4c3c729b2..a0ecb6bd2d0 100644
--- a/tests/queries/0_stateless/01297_create_quota.sql
+++ b/tests/queries/0_stateless/01297_create_quota.sql
@@ -197,21 +197,6 @@ SHOW CREATE QUOTA q1_01297;
 SHOW CREATE QUOTA q2_01297;
 DROP QUOTA IF EXISTS q1_01297;
 DROP QUOTA IF EXISTS q2_01297;
-SELECT '-- FIXME';
-SELECT '-- the overflow happens somewhere else before reaching the up bound of Quota Value , not in the parser';
-SELECT '-- the overflow have different behavior in different environment';
--- CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '18446744073709551616';
--- CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1T';
--- CREATE QUOTA q3_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '1Ti';
--- CREATE QUOTA q4_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '18446744074';
--- SHOW CREATE QUOTA q1_01297;
--- SHOW CREATE QUOTA q2_01297;
--- SHOW CREATE QUOTA q3_01297;
--- SHOW CREATE QUOTA q4_01297;
--- DROP QUOTA IF EXISTS q1_01297;
--- DROP QUOTA IF EXISTS q2_01297;
--- DROP QUOTA IF EXISTS q3_01297;
--- DROP QUOTA IF EXISTS q4_01297;
 SELECT '-- zero test';
 CREATE QUOTA q1_01297 FOR INTERVAL 1 MINUTE MAX query_selects = '0';
 CREATE QUOTA q2_01297 FOR INTERVAL 1 MINUTE MAX execution_time = '0';

From fba7120d3d2ea92ca4d8924183e0b63be50e745c Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Tue, 9 May 2023 17:57:23 +0200
Subject: [PATCH 478/689] Fix s3_helper logging and pool setup

---
 tests/ci/s3_helper.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py
index 2af02d572c8..0b182ecb09e 100644
--- a/tests/ci/s3_helper.py
+++ b/tests/ci/s3_helper.py
@@ -8,6 +8,7 @@ import time
 from multiprocessing.dummy import Pool
 
 import boto3  # type: ignore
+import botocore  # type: ignore
 
 from env_helper import (
     S3_TEST_REPORTS_BUCKET,
@@ -40,9 +41,12 @@ def _flatten_list(lst):
 
 
 class S3Helper:
+    max_pool_size = 100
+
     def __init__(self):
+        config = botocore.config.Config(max_pool_connections=self.max_pool_size)
         self.session = boto3.session.Session(region_name="us-east-1")
-        self.client = self.session.client("s3", endpoint_url=S3_URL)
+        self.client = self.session.client("s3", endpoint_url=S3_URL, config=config)
         self.host = S3_URL
         self.download_host = S3_DOWNLOAD
 
@@ -167,25 +171,20 @@ class S3Helper:
                 if counter % 1000 == 0:
                     sum_time += int(time.time() - t)
                     print(
-                        "Uploaded",
-                        counter,
-                        "-",
-                        int(time.time() - t),
-                        "s",
-                        "sum time",
-                        sum_time,
-                        "s",
+                        f"Uploaded {counter}, {int(time.time()-t)}s, "
+                        f"sum time {sum_time}s",
                     )
                     t = time.time()
             except Exception as ex:
                 logging.critical("Failed to upload file, expcetion %s", ex)
             return f"{self.download_host}/{bucket_name}/{s3_path}"
 
-        p = Pool(256)
+        p = Pool(self.max_pool_size)
 
+        original_level = logging.root.level
         logging.basicConfig(level=logging.CRITICAL)
         result = sorted(_flatten_list(p.map(upload_task, all_files)))
-        logging.basicConfig(level=logging.INFO)
+        logging.basicConfig(level=original_level)
         return result
 
     def _upload_folder_to_s3(

From 7614c1c848d5c3a99843252fd14c1e982ed16d42 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Tue, 9 May 2023 19:38:19 +0200
Subject: [PATCH 479/689] Move the command for codebrowser into a script

---
 docker/test/codebrowser/Dockerfile | 19 ++-----------------
 docker/test/codebrowser/build.sh   | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 17 deletions(-)
 create mode 100755 docker/test/codebrowser/build.sh

diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile
index 20c39646af5..db30397ab2e 100644
--- a/docker/test/codebrowser/Dockerfile
+++ b/docker/test/codebrowser/Dockerfile
@@ -26,20 +26,5 @@ RUN git clone --depth=1 https://github.com/ClickHouse/woboq_codebrowser /woboq_c
   && cmake . -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-${LLVM_VERSION} -DCMAKE_C_COMPILER=clang-${LLVM_VERSION} -DCLANG_BUILTIN_HEADERS_DIR=/usr/lib/llvm-${LLVM_VERSION}/lib/clang/${LLVM_VERSION}/include \
   && ninja
 
-ENV CODEGEN=/woboq_codebrowser/generator/codebrowser_generator
-ENV CODEINDEX=/woboq_codebrowser/indexgenerator/codebrowser_indexgenerator
-ENV STATIC_DATA=/woboq_codebrowser/data
-
-ENV SOURCE_DIRECTORY=/repo_folder
-ENV BUILD_DIRECTORY=/build
-ENV HTML_RESULT_DIRECTORY=$BUILD_DIRECTORY/html_report
-ENV SHA=nosha
-ENV DATA="https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data"
-
-CMD mkdir -p $BUILD_DIRECTORY && cd $BUILD_DIRECTORY && \
-    cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-${LLVM_VERSION} -DCMAKE_C_COMPILER=/usr/bin/clang-${LLVM_VERSION} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \
-    mkdir -p $HTML_RESULT_DIRECTORY && \
-    $CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA -d $DATA | ts '%Y-%m-%d %H:%M:%S' && \
-    cp -r $STATIC_DATA $HTML_RESULT_DIRECTORY/ &&\
-    $CODEINDEX $HTML_RESULT_DIRECTORY -d "$DATA" | ts '%Y-%m-%d %H:%M:%S' && \
-    mv $HTML_RESULT_DIRECTORY /test_output
+COPY build.sh /
+CMD ["bash", "-c", "/build.sh 2>&1"]
diff --git a/docker/test/codebrowser/build.sh b/docker/test/codebrowser/build.sh
new file mode 100755
index 00000000000..e673cba95c7
--- /dev/null
+++ b/docker/test/codebrowser/build.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -x -e
+
+
+STATIC_DATA=${STATIC_DATA:-/woboq_codebrowser/data}
+SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-/repo_folder}
+BUILD_DIRECTORY=${BUILD_DIRECTORY:-/build}
+HTML_RESULT_DIRECTORY=${HTML_RESULT_DIRECTORY:-$BUILD_DIRECTORY/html_report}
+SHA=${SHA:-nosha}
+DATA=${DATA:-https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data}
+
+mkdir -p "$BUILD_DIRECTORY" && cd "$BUILD_DIRECTORY"
+cmake "$SOURCE_DIRECTORY" -DCMAKE_CXX_COMPILER="/usr/bin/clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="/usr/bin/clang-${LLVM_VERSION}" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0
+mkdir -p "$HTML_RESULT_DIRECTORY"
+/woboq_codebrowser/generator/codebrowser_generator -b "$BUILD_DIRECTORY" -a \
+  -o "$HTML_RESULT_DIRECTORY" --execute-concurrency=0 -p "ClickHouse:$SOURCE_DIRECTORY:$SHA" \
+  -d "$DATA" | ts '%Y-%m-%d %H:%M:%S'
+cp -r "$STATIC_DATA" "$HTML_RESULT_DIRECTORY/"
+/woboq_codebrowser/indexgenerator/codebrowser_indexgenerator "$HTML_RESULT_DIRECTORY" \
+  -d "$DATA" | ts '%Y-%m-%d %H:%M:%S'
+mv "$HTML_RESULT_DIRECTORY" /test_output

From b7b48b0d0933ff17f8a88fba6616577c3ae26ae3 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Tue, 9 May 2023 21:59:49 +0200
Subject: [PATCH 480/689] Use rebuilt images in woboq, call it from other
 workflows

---
 .github/workflows/woboq.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/woboq.yml b/.github/workflows/woboq.yml
index 363652c9f33..30ce6e4875a 100644
--- a/.github/workflows/woboq.yml
+++ b/.github/workflows/woboq.yml
@@ -9,6 +9,7 @@ on: # yamllint disable-line rule:truthy
   schedule:
     - cron: '0 */18 * * *'
   workflow_dispatch:
+  workflow_call:
 jobs:
   # don't use dockerhub push because this image updates so rarely
   WoboqCodebrowser:
@@ -26,6 +27,10 @@ jobs:
         with:
           clear-repository: true
           submodules: 'true'
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.IMAGES_PATH }}
       - name: Codebrowser
         run: |
           sudo rm -fr "$TEMP_PATH"

From ec701d96c76c70e33aa79e321f225b61adf847a1 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Tue, 9 May 2023 22:01:09 +0200
Subject: [PATCH 481/689] Rework the codebrowser build directories structure

---
 docker/test/codebrowser/build.sh | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/docker/test/codebrowser/build.sh b/docker/test/codebrowser/build.sh
index e673cba95c7..643781a96fc 100755
--- a/docker/test/codebrowser/build.sh
+++ b/docker/test/codebrowser/build.sh
@@ -4,19 +4,20 @@ set -x -e
 
 
 STATIC_DATA=${STATIC_DATA:-/woboq_codebrowser/data}
-SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-/repo_folder}
-BUILD_DIRECTORY=${BUILD_DIRECTORY:-/build}
-HTML_RESULT_DIRECTORY=${HTML_RESULT_DIRECTORY:-$BUILD_DIRECTORY/html_report}
+SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-/build}
+BUILD_DIRECTORY=${BUILD_DIRECTORY:-/workdir/build}
+HTML_RESULT_DIRECTORY=${HTML_RESULT_DIRECTORY:-/workdir/output/html_report}
 SHA=${SHA:-nosha}
 DATA=${DATA:-https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data}
 
+read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
+
 mkdir -p "$BUILD_DIRECTORY" && cd "$BUILD_DIRECTORY"
-cmake "$SOURCE_DIRECTORY" -DCMAKE_CXX_COMPILER="/usr/bin/clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="/usr/bin/clang-${LLVM_VERSION}" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0
+cmake "$SOURCE_DIRECTORY" -DCMAKE_CXX_COMPILER="/usr/bin/clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="/usr/bin/clang-${LLVM_VERSION}" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 "${CMAKE_FLAGS[@]}"
 mkdir -p "$HTML_RESULT_DIRECTORY"
 /woboq_codebrowser/generator/codebrowser_generator -b "$BUILD_DIRECTORY" -a \
   -o "$HTML_RESULT_DIRECTORY" --execute-concurrency=0 -p "ClickHouse:$SOURCE_DIRECTORY:$SHA" \
-  -d "$DATA" | ts '%Y-%m-%d %H:%M:%S'
+  -d "$DATA" |& ts '%Y-%m-%d %H:%M:%S'
 cp -r "$STATIC_DATA" "$HTML_RESULT_DIRECTORY/"
 /woboq_codebrowser/indexgenerator/codebrowser_indexgenerator "$HTML_RESULT_DIRECTORY" \
-  -d "$DATA" | ts '%Y-%m-%d %H:%M:%S'
-mv "$HTML_RESULT_DIRECTORY" /test_output
+  -d "$DATA" |& ts '%Y-%m-%d %H:%M:%S'

From 6a56662f60f77769b4a1134ff667b04f2b02d392 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Tue, 9 May 2023 22:03:20 +0200
Subject: [PATCH 482/689] Improve codebrowser check

- Add sccache
- Report a proper status in report and exit code
- Launch it as a current user
- Fix report link
---
 tests/ci/codebrowser_check.py | 49 +++++++++++++++++++++++++----------
 1 file changed, 35 insertions(+), 14 deletions(-)

diff --git a/tests/ci/codebrowser_check.py b/tests/ci/codebrowser_check.py
index 0eb4921e3fe..67c54fa26be 100644
--- a/tests/ci/codebrowser_check.py
+++ b/tests/ci/codebrowser_check.py
@@ -1,18 +1,20 @@
 #!/usr/bin/env python3
 
 
+import logging
 import os
 import subprocess
-import logging
+import sys
 
 from github import Github
 
 from commit_status_helper import get_commit, post_commit_status
-from docker_pull_helper import get_image_with_version
+from docker_pull_helper import get_image_with_version, DockerImage
 from env_helper import (
     IMAGES_PATH,
     REPO_COPY,
     S3_DOWNLOAD,
+    S3_BUILDS_BUCKET,
     S3_TEST_REPORTS_BUCKET,
     TEMP_PATH,
 )
@@ -27,16 +29,24 @@ from upload_result_helper import upload_results
 NAME = "Woboq Build"
 
 
-def get_run_command(repo_path, output_path, image):
+def get_run_command(
+    repo_path: str, output_path: str, image: DockerImage, sha: str
+) -> str:
+    user = f"{os.geteuid()}:{os.getegid()}"
     cmd = (
-        "docker run " + f"--volume={repo_path}:/repo_folder "
-        f"--volume={output_path}:/test_output "
-        f"-e 'DATA={S3_DOWNLOAD}/{S3_TEST_REPORTS_BUCKET}/codebrowser/data' {image}"
+        f"docker run --rm --user={user} --volume={repo_path}:/build "
+        f"--volume={output_path}:/workdir/output --network=host "
+        # use sccache, https://github.com/KDAB/codebrowser/issues/111
+        f"-e SCCACHE_BUCKET='{S3_BUILDS_BUCKET}' "
+        "-e SCCACHE_S3_KEY_PREFIX=ccache/sccache "
+        '-e CMAKE_FLAGS="$CMAKE_FLAGS -DCOMPILER_CACHE=sccache" '
+        f"-e 'DATA={S3_DOWNLOAD}/{S3_TEST_REPORTS_BUCKET}/codebrowser/data' "
+        f"-e SHA={sha} {image}"
     )
     return cmd
 
 
-if __name__ == "__main__":
+def main():
     logging.basicConfig(level=logging.INFO)
 
     stopwatch = Stopwatch()
@@ -55,37 +65,48 @@ if __name__ == "__main__":
     if not os.path.exists(result_path):
         os.makedirs(result_path)
 
-    run_command = get_run_command(REPO_COPY, result_path, docker_image)
+    run_command = get_run_command(
+        REPO_COPY, result_path, docker_image, pr_info.sha[:12]
+    )
 
     logging.info("Going to run codebrowser: %s", run_command)
 
     run_log_path = os.path.join(TEMP_PATH, "run.log")
 
+    state = "success"
     with TeePopen(run_command, run_log_path) as process:
         retcode = process.wait()
         if retcode == 0:
             logging.info("Run successfully")
         else:
             logging.info("Run failed")
+            state = "failure"
 
     subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {TEMP_PATH}", shell=True)
 
     report_path = os.path.join(result_path, "html_report")
     logging.info("Report path %s", report_path)
     s3_path_prefix = "codebrowser"
-    html_urls = s3_helper.fast_parallel_upload_dir(
-        report_path, s3_path_prefix, "clickhouse-test-reports"
+    _ = s3_helper.fast_parallel_upload_dir(
+        report_path, s3_path_prefix, S3_TEST_REPORTS_BUCKET
     )
 
     index_html = (
-        '<a href="{S3_DOWNLOAD}/{S3_TEST_REPORTS_BUCKET}/codebrowser/index.html">'
-        "HTML report</a>"
+        f'<a href="{S3_DOWNLOAD}/{S3_TEST_REPORTS_BUCKET}/codebrowser/index.html">'
+        "Generate codebrowser site</a>"
     )
 
-    test_result = TestResult(index_html, "Look at the report")
+    test_result = TestResult(index_html, state, stopwatch.duration_seconds)
 
     report_url = upload_results(s3_helper, 0, pr_info.sha, [test_result], [], NAME)
 
     print(f"::notice ::Report url: {report_url}")
 
-    post_commit_status(commit, "success", report_url, "Report built", NAME, pr_info)
+    post_commit_status(commit, state, report_url, "Report built", NAME, pr_info)
+
+    if state != "success":
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

From eb122487ad2f6d6db9ba459db462f101092a72e2 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 10 May 2023 13:02:21 +0200
Subject: [PATCH 483/689] Add logs to the report, do not chown unnecessary

---
 docker/test/codebrowser/build.sh |  9 +++++++--
 tests/ci/codebrowser_check.py    | 28 ++++++++++++++++------------
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/docker/test/codebrowser/build.sh b/docker/test/codebrowser/build.sh
index 643781a96fc..f88f07ef72b 100755
--- a/docker/test/codebrowser/build.sh
+++ b/docker/test/codebrowser/build.sh
@@ -6,7 +6,8 @@ set -x -e
 STATIC_DATA=${STATIC_DATA:-/woboq_codebrowser/data}
 SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-/build}
 BUILD_DIRECTORY=${BUILD_DIRECTORY:-/workdir/build}
-HTML_RESULT_DIRECTORY=${HTML_RESULT_DIRECTORY:-/workdir/output/html_report}
+OUTPUT_DIRECTORY=${OUTPUT_DIRECTORY:-/workdir/output}
+HTML_RESULT_DIRECTORY=${HTML_RESULT_DIRECTORY:-$OUTPUT_DIRECTORY/html_report}
 SHA=${SHA:-nosha}
 DATA=${DATA:-https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data}
 
@@ -15,9 +16,13 @@ read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
 mkdir -p "$BUILD_DIRECTORY" && cd "$BUILD_DIRECTORY"
 cmake "$SOURCE_DIRECTORY" -DCMAKE_CXX_COMPILER="/usr/bin/clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="/usr/bin/clang-${LLVM_VERSION}" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 "${CMAKE_FLAGS[@]}"
 mkdir -p "$HTML_RESULT_DIRECTORY"
+echo 'Filter out too noisy "Error: filename" lines and keep them in full codebrowser_generator.log'
 /woboq_codebrowser/generator/codebrowser_generator -b "$BUILD_DIRECTORY" -a \
   -o "$HTML_RESULT_DIRECTORY" --execute-concurrency=0 -p "ClickHouse:$SOURCE_DIRECTORY:$SHA" \
-  -d "$DATA" |& ts '%Y-%m-%d %H:%M:%S'
+  -d "$DATA" \
+    |& ts '%Y-%m-%d %H:%M:%S' \
+    | tee "$OUTPUT_DIRECTORY/codebrowser_generator.log" \
+    | grep --line-buffered -v ':[0-9]* Error: '
 cp -r "$STATIC_DATA" "$HTML_RESULT_DIRECTORY/"
 /woboq_codebrowser/indexgenerator/codebrowser_indexgenerator "$HTML_RESULT_DIRECTORY" \
   -d "$DATA" |& ts '%Y-%m-%d %H:%M:%S'
diff --git a/tests/ci/codebrowser_check.py b/tests/ci/codebrowser_check.py
index 67c54fa26be..99bc7e2f29a 100644
--- a/tests/ci/codebrowser_check.py
+++ b/tests/ci/codebrowser_check.py
@@ -3,9 +3,10 @@
 
 import logging
 import os
-import subprocess
 import sys
 
+from pathlib import Path
+
 from github import Github
 
 from commit_status_helper import get_commit, post_commit_status
@@ -30,7 +31,7 @@ NAME = "Woboq Build"
 
 
 def get_run_command(
-    repo_path: str, output_path: str, image: DockerImage, sha: str
+    repo_path: Path, output_path: Path, image: DockerImage, sha: str
 ) -> str:
     user = f"{os.geteuid()}:{os.getegid()}"
     cmd = (
@@ -54,24 +55,25 @@ def main():
     gh = Github(get_best_robot_token(), per_page=100)
     pr_info = PRInfo()
     commit = get_commit(gh, pr_info.sha)
+    temp_path = Path(TEMP_PATH)
 
-    if not os.path.exists(TEMP_PATH):
-        os.makedirs(TEMP_PATH)
+    if not temp_path.exists():
+        os.makedirs(temp_path)
 
     docker_image = get_image_with_version(IMAGES_PATH, "clickhouse/codebrowser")
     s3_helper = S3Helper()
 
-    result_path = os.path.join(TEMP_PATH, "result_path")
-    if not os.path.exists(result_path):
+    result_path = temp_path / "result_path"
+    if not result_path.exists():
         os.makedirs(result_path)
 
     run_command = get_run_command(
-        REPO_COPY, result_path, docker_image, pr_info.sha[:12]
+        Path(REPO_COPY), result_path, docker_image, pr_info.sha[:12]
     )
 
     logging.info("Going to run codebrowser: %s", run_command)
 
-    run_log_path = os.path.join(TEMP_PATH, "run.log")
+    run_log_path = result_path / "run.log"
 
     state = "success"
     with TeePopen(run_command, run_log_path) as process:
@@ -82,9 +84,7 @@ def main():
             logging.info("Run failed")
             state = "failure"
 
-    subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {TEMP_PATH}", shell=True)
-
-    report_path = os.path.join(result_path, "html_report")
+    report_path = result_path / "html_report"
     logging.info("Report path %s", report_path)
     s3_path_prefix = "codebrowser"
     _ = s3_helper.fast_parallel_upload_dir(
@@ -96,7 +96,11 @@ def main():
         "Generate codebrowser site</a>"
     )
 
-    test_result = TestResult(index_html, state, stopwatch.duration_seconds)
+    additional_logs = [path.absolute() for path in result_path.glob("*.log")]
+
+    test_result = TestResult(
+        index_html, state, stopwatch.duration_seconds, additional_logs
+    )
 
     report_url = upload_results(s3_helper, 0, pr_info.sha, [test_result], [], NAME)
 

From 3129eda247d15f0293bf192d276d47cb773cb7b7 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 10 May 2023 13:30:39 +0200
Subject: [PATCH 484/689] Launch WoboqBuilder in scope of NightlyBuilds

---
 .github/workflows/nightly.yml | 3 +++
 .github/workflows/woboq.yml   | 2 --
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 8b54433359e..acf6bbe8f6a 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -72,6 +72,9 @@ jobs:
         with:
           name: changed_images
           path: ${{ runner.temp }}/changed_images.json
+  Codebrowser:
+    needs: [DockerHubPush]
+    uses: ./.github/workflows/woboq.yml
   BuilderCoverity:
     needs: DockerHubPush
     runs-on: [self-hosted, builder]
diff --git a/.github/workflows/woboq.yml b/.github/workflows/woboq.yml
index 30ce6e4875a..bdfbc8fef9c 100644
--- a/.github/workflows/woboq.yml
+++ b/.github/workflows/woboq.yml
@@ -6,8 +6,6 @@ env:
 concurrency:
   group: woboq
 on: # yamllint disable-line rule:truthy
-  schedule:
-    - cron: '0 */18 * * *'
   workflow_dispatch:
   workflow_call:
 jobs:

From 65d1ae77eec8f6eac778403ab8c6416ed751a525 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 10 May 2023 16:26:50 +0200
Subject: [PATCH 485/689] Use updated upstream woboq

---
 docker/test/codebrowser/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile
index db30397ab2e..8136fd1fbbc 100644
--- a/docker/test/codebrowser/Dockerfile
+++ b/docker/test/codebrowser/Dockerfile
@@ -20,8 +20,8 @@ RUN arch=${TARGETARCH:-amd64} \
 
 # repo versions doesn't work correctly with C++17
 # also we push reports to s3, so we add index.html to subfolder urls
-# https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b
-RUN git clone --depth=1 https://github.com/ClickHouse/woboq_codebrowser /woboq_codebrowser \
+# https://github.com/ClickHouse/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b
+RUN git clone --branch=master --depth=1 https://github.com/ClickHouse/woboq_codebrowser /woboq_codebrowser \
   && cd /woboq_codebrowser \
   && cmake . -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-${LLVM_VERSION} -DCMAKE_C_COMPILER=clang-${LLVM_VERSION} -DCLANG_BUILTIN_HEADERS_DIR=/usr/lib/llvm-${LLVM_VERSION}/lib/clang/${LLVM_VERSION}/include \
   && ninja

From 0f70ad56a922a7c77700cf23a848f2f5b54ca7cb Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 10 May 2023 20:14:30 +0200
Subject: [PATCH 486/689] Do not upload failed site to the S3

---
 tests/ci/codebrowser_check.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/ci/codebrowser_check.py b/tests/ci/codebrowser_check.py
index 99bc7e2f29a..28e3bbdb4fa 100644
--- a/tests/ci/codebrowser_check.py
+++ b/tests/ci/codebrowser_check.py
@@ -87,9 +87,10 @@ def main():
     report_path = result_path / "html_report"
     logging.info("Report path %s", report_path)
     s3_path_prefix = "codebrowser"
-    _ = s3_helper.fast_parallel_upload_dir(
-        report_path, s3_path_prefix, S3_TEST_REPORTS_BUCKET
-    )
+    if state == "success":
+        _ = s3_helper.fast_parallel_upload_dir(
+            report_path, s3_path_prefix, S3_TEST_REPORTS_BUCKET
+        )
 
     index_html = (
         f'<a href="{S3_DOWNLOAD}/{S3_TEST_REPORTS_BUCKET}/codebrowser/index.html">'

From 338435a52d55c466d19c2b2db1b233e540ab103b Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 10 May 2023 22:16:28 +0200
Subject: [PATCH 487/689] Additional check for FATAL in logs, do not exit with
 code 1 on errors

---
 tests/ci/codebrowser_check.py | 36 ++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/tests/ci/codebrowser_check.py b/tests/ci/codebrowser_check.py
index 28e3bbdb4fa..f9883f0f975 100644
--- a/tests/ci/codebrowser_check.py
+++ b/tests/ci/codebrowser_check.py
@@ -3,8 +3,6 @@
 
 import logging
 import os
-import sys
-
 from pathlib import Path
 
 from github import Github
@@ -99,19 +97,39 @@ def main():
 
     additional_logs = [path.absolute() for path in result_path.glob("*.log")]
 
-    test_result = TestResult(
-        index_html, state, stopwatch.duration_seconds, additional_logs
-    )
+    test_results = [
+        TestResult(index_html, state, stopwatch.duration_seconds, additional_logs)
+    ]
 
-    report_url = upload_results(s3_helper, 0, pr_info.sha, [test_result], [], NAME)
+    # Check if the run log contains `FATAL Error:`, that means the code problem
+    stopwatch = Stopwatch()
+    fatal_error = "FATAL Error:"
+    logging.info("Search for '%s' in %s", fatal_error, run_log_path)
+    with open(run_log_path, "r", encoding="utf-8") as rlfd:
+        for line in rlfd.readlines():
+            if "FATAL Error:" in line:
+                logging.warning(
+                    "The line '%s' found, mark the run as failure", fatal_error
+                )
+                state = "failure"
+                test_results.append(
+                    TestResult(
+                        "Indexing error",
+                        state,
+                        stopwatch.duration_seconds,
+                        additional_logs,
+                    )
+                )
+                break
+
+    report_url = upload_results(
+        s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME
+    )
 
     print(f"::notice ::Report url: {report_url}")
 
     post_commit_status(commit, state, report_url, "Report built", NAME, pr_info)
 
-    if state != "success":
-        sys.exit(1)
-
 
 if __name__ == "__main__":
     main()

From e79c1cdb1a2076a2c452514d080117e9b162a5fe Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 10 May 2023 22:23:27 +0200
Subject: [PATCH 488/689] Attempt to increas parallelism for
 codebrowser_generator

---
 docker/test/codebrowser/build.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/test/codebrowser/build.sh b/docker/test/codebrowser/build.sh
index f88f07ef72b..5ab9de5a453 100755
--- a/docker/test/codebrowser/build.sh
+++ b/docker/test/codebrowser/build.sh
@@ -10,6 +10,7 @@ OUTPUT_DIRECTORY=${OUTPUT_DIRECTORY:-/workdir/output}
 HTML_RESULT_DIRECTORY=${HTML_RESULT_DIRECTORY:-$OUTPUT_DIRECTORY/html_report}
 SHA=${SHA:-nosha}
 DATA=${DATA:-https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data}
+nproc=$(($(nproc) + 2)) # increase parallelism
 
 read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
 
@@ -18,7 +19,7 @@ cmake "$SOURCE_DIRECTORY" -DCMAKE_CXX_COMPILER="/usr/bin/clang++-${LLVM_VERSION}
 mkdir -p "$HTML_RESULT_DIRECTORY"
 echo 'Filter out too noisy "Error: filename" lines and keep them in full codebrowser_generator.log'
 /woboq_codebrowser/generator/codebrowser_generator -b "$BUILD_DIRECTORY" -a \
-  -o "$HTML_RESULT_DIRECTORY" --execute-concurrency=0 -p "ClickHouse:$SOURCE_DIRECTORY:$SHA" \
+  -o "$HTML_RESULT_DIRECTORY" --execute-concurrency="$nproc" -p "ClickHouse:$SOURCE_DIRECTORY:$SHA" \
   -d "$DATA" \
     |& ts '%Y-%m-%d %H:%M:%S' \
     | tee "$OUTPUT_DIRECTORY/codebrowser_generator.log" \

From c375f948d07d871405286574fc25df76fd0deffe Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 10 May 2023 22:56:13 +0200
Subject: [PATCH 489/689] Fix typing in S3Helper.fast_parallel_upload_dir

---
 tests/ci/s3_helper.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py
index 0b182ecb09e..9ee0a431294 100644
--- a/tests/ci/s3_helper.py
+++ b/tests/ci/s3_helper.py
@@ -6,6 +6,8 @@ import re
 import shutil
 import time
 from multiprocessing.dummy import Pool
+from pathlib import Path
+from typing import List, Union
 
 import boto3  # type: ignore
 import botocore  # type: ignore
@@ -128,7 +130,9 @@ class S3Helper:
         else:
             return S3Helper.copy_file_to_local(S3_BUILDS_BUCKET, file_path, s3_path)
 
-    def fast_parallel_upload_dir(self, dir_path, s3_dir_path, bucket_name):
+    def fast_parallel_upload_dir(
+        self, dir_path: Union[str, Path], s3_dir_path: str, bucket_name: str
+    ) -> List[str]:
         all_files = []
 
         for root, _, files in os.walk(dir_path):
@@ -141,12 +145,12 @@ class S3Helper:
         t = time.time()
         sum_time = 0
 
-        def upload_task(file_path):
+        def upload_task(file_path: str) -> str:
             nonlocal counter
             nonlocal t
             nonlocal sum_time
             try:
-                s3_path = file_path.replace(dir_path, s3_dir_path)
+                s3_path = file_path.replace(str(dir_path), s3_dir_path)
                 metadata = {}
                 if s3_path.endswith("html"):
                     metadata["ContentType"] = "text/html; charset=utf-8"

From 4b7aa300175730f3c08e169cc4fd048724714bc8 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Fri, 12 May 2023 12:17:37 +0200
Subject: [PATCH 490/689] The best effor to get the current job ID and URL

This partially addresses https://github.com/actions/runner/issues/2577
---
 tests/ci/env_helper.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py
index 5c2139ae0bc..9303f9ae293 100644
--- a/tests/ci/env_helper.py
+++ b/tests/ci/env_helper.py
@@ -1,3 +1,4 @@
+import logging
 import os
 from os import path as p
 
@@ -65,6 +66,32 @@ def GITHUB_JOB_ID() -> str:
         ):
             _GITHUB_JOB_ID = "0"
 
+    # FIXME: until it's here, we can't move to reusable workflows
+    if not _GITHUB_JOB_URL:
+        # This is a terrible workaround for the case of another broken part of
+        # GitHub actions. For nested workflows it doesn't provide a proper GITHUB_JOB
+        # value, but only the final one. So, for `OriginalJob / NestedJob / FinalJob`
+        # full name, GITHUB_JOB contains only FinalJob
+        matched_jobs = []
+        for job in jobs:
+            nested_parts = job["name"].split(" / ")
+            if len(nested_parts) <= 1:
+                continue
+            if nested_parts[-1] == GITHUB_JOB:
+                matched_jobs.append(job)
+        if len(matched_jobs) == 1:
+            # The best case scenario
+            _GITHUB_JOB_ID = matched_jobs[0]["id"]
+            _GITHUB_JOB_URL = matched_jobs[0]["html_url"]
+            return _GITHUB_JOB_ID
+        if matched_jobs:
+            logging.error(
+                "We could not get the ID and URL for the current job name %s, there "
+                "are more than one jobs match it for the nested workflows. Please, "
+                "refer to https://github.com/actions/runner/issues/2577",
+                GITHUB_JOB,
+            )
+
     return _GITHUB_JOB_ID
 
 

From 51c69b6fbdefa577a5e8ee10819fc5ec91d6aa9b Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 12 May 2023 17:36:35 +0000
Subject: [PATCH 491/689] Comments regarding zero values in clearable hash
 table

---
 src/Common/HashTable/ClearableHashSet.h   | 4 ++++
 src/Common/HashTable/HashTableKeyHolder.h | 6 +++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/Common/HashTable/ClearableHashSet.h b/src/Common/HashTable/ClearableHashSet.h
index 13067c39d1b..657d388cbfb 100644
--- a/src/Common/HashTable/ClearableHashSet.h
+++ b/src/Common/HashTable/ClearableHashSet.h
@@ -10,6 +10,10 @@
   * Instead of this class, you could just use the pair (version, key) in the HashSet as the key
   * but then the table would accumulate all the keys that it ever stored, and it was unreasonably growing.
   * This class goes a step further and considers the keys with the old version empty in the hash table.
+  *
+  * Zero values note:
+  * A cell in ClearableHashSet can store a zero values as normal value
+  * If its version is equal to the version of the set itself, then it's not considered as empty even key's value is zero value of the corresponding type
   */
 
 
diff --git a/src/Common/HashTable/HashTableKeyHolder.h b/src/Common/HashTable/HashTableKeyHolder.h
index 994587e67ec..d1b38744255 100644
--- a/src/Common/HashTable/HashTableKeyHolder.h
+++ b/src/Common/HashTable/HashTableKeyHolder.h
@@ -88,7 +88,11 @@ inline StringRef & ALWAYS_INLINE keyHolderGetKey(DB::ArenaKeyHolder & holder)
 
 inline void ALWAYS_INLINE keyHolderPersistKey(DB::ArenaKeyHolder & holder)
 {
-    // Hash table shouldn't ask us to persist a zero key
+    // Normally, our hash table shouldn't ask us to persist a zero key,
+    // but it can happened in the case of clearable hash table (ClearableHashSet, for example).
+    // The clearable hash table doesn't use zero storage and
+    // distinguishes empty keys by using cell version, not the value itself.
+    // So, when an empty StringRef is inserted in ClearableHashSet we'll get here key of zero size.
     // assert(holder.key.size > 0);
     holder.key.data = holder.pool.insert(holder.key.data, holder.key.size);
 }

From e468b92b9c1d06e916ef2f6d3031febfd63a7b9e Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 12 May 2023 17:47:46 +0000
Subject: [PATCH 492/689] Remove vectorscan

---
 .gitmodules        | 6 ------
 contrib/vectorscan | 1 -
 2 files changed, 7 deletions(-)
 delete mode 160000 contrib/vectorscan

diff --git a/.gitmodules b/.gitmodules
index 67a7cb55726..d04c235c9e2 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -265,12 +265,6 @@
 [submodule "contrib/nats-io"]
 	path = contrib/nats-io
 	url = https://github.com/ClickHouse/nats.c
-[submodule "contrib/vectorscan"]
-	path = contrib/vectorscan
-	# FIXME: update once upstream fixes will be merged:
-	# - https://github.com/VectorCamp/vectorscan/pull/148
-	# - https://github.com/VectorCamp/vectorscan/pull/149
-	url = https://github.com/azat-ch/vectorscan
 [submodule "contrib/c-ares"]
 	path = contrib/c-ares
 	url = https://github.com/ClickHouse/c-ares
diff --git a/contrib/vectorscan b/contrib/vectorscan
deleted file mode 160000
index aaca65aa210..00000000000
--- a/contrib/vectorscan
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit aaca65aa210ce3ec91bd2b249c4d59e55e80a869

From 6eeeea56cf538ad81fbee11c3ec6d3c33bde2275 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 12 May 2023 20:49:57 +0300
Subject: [PATCH 493/689] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a33e62be231..4a619eb4fd3 100644
--- a/README.md
+++ b/README.md
@@ -43,4 +43,4 @@ We are a globally diverse and distributed team, united behind a common goal of c
   
 Check out our **current openings** here: https://clickhouse.com/company/careers 
   
-Cant find what you are looking for, but want to let us know you are interested in joining ClickHouse? Email careers@clickhouse.com ! 
+Cant find what you are looking for, but want to let us know you are interested in joining ClickHouse? Email careers@clickhouse.com! 

From a6d4c17afb19d586774866fb2bed49969f5c4dd1 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 12 May 2023 17:52:45 +0000
Subject: [PATCH 494/689] Add vectorscan submodule

---
 .gitmodules        | 3 +++
 contrib/vectorscan | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 contrib/vectorscan

diff --git a/.gitmodules b/.gitmodules
index d04c235c9e2..b82099fbb71 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -341,3 +341,6 @@
 [submodule "contrib/isa-l"]
 	path = contrib/isa-l
 	url = https://github.com/ClickHouse/isa-l.git
+[submodule "contrib/vectorscan"]
+	path = contrib/vectorscan
+	url = https://github.com/ClickHouse/vectorscan.git
diff --git a/contrib/vectorscan b/contrib/vectorscan
new file mode 160000
index 00000000000..b4bba94b1a2
--- /dev/null
+++ b/contrib/vectorscan
@@ -0,0 +1 @@
+Subproject commit b4bba94b1a250603b0b198e0394946e32f6c3f30

From ef3e4ddc2a0ca7fd21097c23f7faa4c5c64b7f43 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 12 May 2023 17:54:39 +0000
Subject: [PATCH 495/689] Move vectorscan entry

---
 .gitmodules | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index b82099fbb71..dbca3f3f6bc 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -265,6 +265,9 @@
 [submodule "contrib/nats-io"]
 	path = contrib/nats-io
 	url = https://github.com/ClickHouse/nats.c
+[submodule "contrib/vectorscan"]
+	path = contrib/vectorscan
+	url = https://github.com/ClickHouse/vectorscan.git
 [submodule "contrib/c-ares"]
 	path = contrib/c-ares
 	url = https://github.com/ClickHouse/c-ares
@@ -341,6 +344,3 @@
 [submodule "contrib/isa-l"]
 	path = contrib/isa-l
 	url = https://github.com/ClickHouse/isa-l.git
-[submodule "contrib/vectorscan"]
-	path = contrib/vectorscan
-	url = https://github.com/ClickHouse/vectorscan.git

From 13494ffe9be984a58bb2d7f807d0bb89edf2de84 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 12 May 2023 17:56:24 +0000
Subject: [PATCH 496/689] Include msan fixes in vectorscan

---
 contrib/vectorscan | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/vectorscan b/contrib/vectorscan
index b4bba94b1a2..1f4d448314e 160000
--- a/contrib/vectorscan
+++ b/contrib/vectorscan
@@ -1 +1 @@
-Subproject commit b4bba94b1a250603b0b198e0394946e32f6c3f30
+Subproject commit 1f4d448314e581473103187765e4c949d01b4259

From f6144ee32bd824367c64c987a094a4f9ddaab0ae Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 13 May 2023 02:45:07 +0300
Subject: [PATCH 497/689] Revert "Make `Pretty` formats even prettier."

---
 src/Core/Settings.h                           |  1 -
 src/Formats/FormatFactory.cpp                 |  1 -
 src/Formats/FormatSettings.h                  |  1 -
 .../Formats/Impl/PrettyBlockOutputFormat.cpp  | 26 +++++++------------
 .../Formats/Impl/PrettyBlockOutputFormat.h    | 10 ++++---
 .../Impl/PrettySpaceBlockOutputFormat.cpp     |  2 +-
 .../queries/0_stateless/00098_k_union_all.sql |  1 -
 .../0_stateless/00405_pretty_formats.sql      |  2 --
 .../00730_unicode_terminal_format.sql         |  2 --
 ...065_window_view_event_hop_watch_bounded.py |  8 ++----
 .../01082_window_view_watch_limit.py          |  8 ++----
 ...01509_output_format_pretty_row_numbers.sql |  2 --
 ...02122_parallel_formatting_Pretty.reference |  2 ++
 .../02122_parallel_formatting_Pretty.sh       |  5 ++++
 ...arallel_formatting_PrettyCompact.reference |  2 ++
 ...02122_parallel_formatting_PrettyCompact.sh |  5 ++++
 ...ormatting_PrettyCompactNoEscapes.reference |  2 ++
 ...allel_formatting_PrettyCompactNoEscapes.sh |  5 ++++
 ...allel_formatting_PrettyNoEscapes.reference |  2 ++
 ...122_parallel_formatting_PrettyNoEscapes.sh |  5 ++++
 ..._parallel_formatting_PrettySpace.reference |  2 ++
 .../02122_parallel_formatting_PrettySpace.sh  |  5 ++++
 ..._formatting_PrettySpaceNoEscapes.reference |  2 ++
 ...arallel_formatting_PrettySpaceNoEscapes.sh |  5 ++++
 .../0_stateless/02375_pretty_formats.sql.j2   |  1 -
 ...2731_pretty_time_based_squashing.reference | 14 ----------
 .../02731_pretty_time_based_squashing.sql     |  2 --
 27 files changed, 63 insertions(+), 60 deletions(-)
 create mode 100644 tests/queries/0_stateless/02122_parallel_formatting_Pretty.reference
 create mode 100755 tests/queries/0_stateless/02122_parallel_formatting_Pretty.sh
 create mode 100644 tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.reference
 create mode 100755 tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.sh
 create mode 100644 tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.reference
 create mode 100755 tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.sh
 create mode 100644 tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.reference
 create mode 100755 tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.sh
 create mode 100644 tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.reference
 create mode 100755 tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.sh
 create mode 100644 tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.reference
 create mode 100755 tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.sh
 delete mode 100644 tests/queries/0_stateless/02731_pretty_time_based_squashing.reference
 delete mode 100644 tests/queries/0_stateless/02731_pretty_time_based_squashing.sql

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 215e5b38056..6e345e7edb1 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -905,7 +905,6 @@ class IColumn;
     M(UInt64, output_format_pretty_max_value_width, 10000, "Maximum width of value to display in Pretty formats. If greater - it will be cut.", 0) \
     M(Bool, output_format_pretty_color, true, "Use ANSI escape sequences to paint colors in Pretty formats", 0) \
     M(String, output_format_pretty_grid_charset, "UTF-8", "Charset for printing grid borders. Available charsets: ASCII, UTF-8 (default one).", 0) \
-    M(Milliseconds, output_format_pretty_squash_ms, 100, "Squash blocks in Pretty formats if the time passed after the previous block is not greater than the specified threshold in milliseconds. This avoids printing miltiple small blocks.", 0) \
     M(UInt64, output_format_parquet_row_group_size, 1000000, "Target row group size in rows.", 0) \
     M(UInt64, output_format_parquet_row_group_size_bytes, 512 * 1024 * 1024, "Target row group size in bytes, before compression.", 0) \
     M(Bool, output_format_parquet_string_as_string, false, "Use Parquet String type instead of Binary for String columns.", 0) \
diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
index d7e9c33c822..082ccb85970 100644
--- a/src/Formats/FormatFactory.cpp
+++ b/src/Formats/FormatFactory.cpp
@@ -128,7 +128,6 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings)
     format_settings.pretty.max_rows = settings.output_format_pretty_max_rows;
     format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width;
     format_settings.pretty.output_format_pretty_row_numbers = settings.output_format_pretty_row_numbers;
-    format_settings.pretty.squash_milliseconds = static_cast<UInt64>(settings.output_format_pretty_squash_ms);
     format_settings.protobuf.input_flatten_google_wrappers = settings.input_format_protobuf_flatten_google_wrappers;
     format_settings.protobuf.output_nullables_with_google_wrappers = settings.output_format_protobuf_nullables_with_google_wrappers;
     format_settings.protobuf.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference;
diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h
index 3a2e818d540..b9a55ffaa83 100644
--- a/src/Formats/FormatSettings.h
+++ b/src/Formats/FormatSettings.h
@@ -227,7 +227,6 @@ struct FormatSettings
         UInt64 max_rows = 10000;
         UInt64 max_column_pad_width = 250;
         UInt64 max_value_width = 10000;
-        UInt64 squash_milliseconds = 100;
         bool color = true;
 
         bool output_format_pretty_row_numbers = false;
diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp
index bfcc4c101d9..14648e68f94 100644
--- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp
+++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp
@@ -137,25 +137,21 @@ void PrettyBlockOutputFormat::write(Chunk chunk, PortKind port_kind)
         total_rows += chunk.getNumRows();
         return;
     }
-
-    if (mono_block
-        || (format_settings.pretty.squash_milliseconds
-            && time_after_previous_chunk.elapsedMilliseconds() <= format_settings.pretty.squash_milliseconds))
+    if (mono_block)
     {
         if (port_kind == PortKind::Main)
         {
-            if (squashed_chunk)
-                squashed_chunk.append(chunk);
+            if (mono_chunk)
+                mono_chunk.append(chunk);
             else
-                squashed_chunk = std::move(chunk);
+                mono_chunk = std::move(chunk);
             return;
         }
 
         /// Should be written from writeSuffix()
-        assert(!squashed_chunk);
+        assert(!mono_chunk);
     }
 
-    writeSquashedChunkIfNeeded();
     writeChunk(chunk, port_kind);
 }
 
@@ -393,20 +389,18 @@ void PrettyBlockOutputFormat::consumeExtremes(Chunk chunk)
 }
 
 
-void PrettyBlockOutputFormat::writeSquashedChunkIfNeeded()
+void PrettyBlockOutputFormat::writeMonoChunkIfNeeded()
 {
-    if (squashed_chunk)
+    if (mono_chunk)
     {
-        writeChunk(squashed_chunk, PortKind::Main);
-        squashed_chunk.clear();
-        if (format_settings.pretty.squash_milliseconds)
-            time_after_previous_chunk.restart();
+        writeChunk(mono_chunk, PortKind::Main);
+        mono_chunk.clear();
     }
 }
 
 void PrettyBlockOutputFormat::writeSuffix()
 {
-    writeSquashedChunkIfNeeded();
+    writeMonoChunkIfNeeded();
 
     if (total_rows >= format_settings.pretty.max_rows)
     {
diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h
index 621f1387196..dfb23ac63f9 100644
--- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h
+++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h
@@ -39,7 +39,7 @@ protected:
 
     void write(Chunk chunk, PortKind port_kind);
     virtual void writeChunk(const Chunk & chunk, PortKind port_kind);
-    void writeSquashedChunkIfNeeded();
+    void writeMonoChunkIfNeeded();
     void writeSuffix() override;
 
     void onRowsReadBeforeUpdate() override { total_rows = getRowsReadBefore(); }
@@ -59,11 +59,11 @@ protected:
 
 private:
     bool mono_block;
-    Chunk squashed_chunk;
-    Stopwatch time_after_previous_chunk; /// For squashing.
+    /// For mono_block == true only
+    Chunk mono_chunk;
 };
 
-template <typename OutputFormat>
+template <class OutputFormat>
 void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, const String & base_name)
 {
     auto creator = [&](FormatFactory & fact, const String & name, bool no_escapes, bool mono_block)
@@ -81,6 +81,8 @@ void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, cons
             }
             return std::make_shared<OutputFormat>(buf, sample, format_settings, mono_block);
         });
+        if (!mono_block)
+            factory.markOutputFormatSupportsParallelFormatting(name);
     };
     creator(factory, base_name, false, false);
     creator(factory, base_name + "NoEscapes", true, false);
diff --git a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp
index e54d88a11b2..0fb1a413a6c 100644
--- a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp
+++ b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp
@@ -100,7 +100,7 @@ void PrettySpaceBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port
 
 void PrettySpaceBlockOutputFormat::writeSuffix()
 {
-    writeSquashedChunkIfNeeded();
+    writeMonoChunkIfNeeded();
 
     if (total_rows >= format_settings.pretty.max_rows)
     {
diff --git a/tests/queries/0_stateless/00098_k_union_all.sql b/tests/queries/0_stateless/00098_k_union_all.sql
index 8f396d3479a..311e5bb19c4 100644
--- a/tests/queries/0_stateless/00098_k_union_all.sql
+++ b/tests/queries/0_stateless/00098_k_union_all.sql
@@ -1,4 +1,3 @@
-SET output_format_pretty_squash_ms = 0;
 SELECT 1 FORMAT PrettySpace;
 SELECT 1 UNION ALL SELECT 1 FORMAT PrettySpace;
 SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 FORMAT PrettySpace;
diff --git a/tests/queries/0_stateless/00405_pretty_formats.sql b/tests/queries/0_stateless/00405_pretty_formats.sql
index a7dc5aa0600..3c8af776278 100644
--- a/tests/queries/0_stateless/00405_pretty_formats.sql
+++ b/tests/queries/0_stateless/00405_pretty_formats.sql
@@ -1,5 +1,3 @@
-SET output_format_pretty_squash_ms = 0;
-
 SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT Pretty;
 SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompact;
 SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettySpace;
diff --git a/tests/queries/0_stateless/00730_unicode_terminal_format.sql b/tests/queries/0_stateless/00730_unicode_terminal_format.sql
index 6327b41a05a..b7a8084c406 100644
--- a/tests/queries/0_stateless/00730_unicode_terminal_format.sql
+++ b/tests/queries/0_stateless/00730_unicode_terminal_format.sql
@@ -1,7 +1,5 @@
 DROP TABLE IF EXISTS unicode;
 
-SET output_format_pretty_squash_ms = 0;
-
 CREATE TABLE unicode(c1 String, c2 String) ENGINE = Memory;
 INSERT INTO unicode VALUES ('Здравствуйте', 'Этот код можно отредактировать и запустить!');
 INSERT INTO unicode VALUES ('你好', '这段代码是可以编辑并且能够运行的！');
diff --git a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py
index 52d0c2c87d4..7f00130b184 100755
--- a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py
+++ b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py
@@ -19,17 +19,13 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.expect(prompt)
 
-    client1.send(
-        "SET allow_experimental_analyzer = 0, output_format_pretty_squash_ms = 0;"
-    )
+    client1.send("SET allow_experimental_analyzer = 0")
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
     client1.send("SET window_view_heartbeat_interval = 1")
     client1.expect(prompt)
-    client2.send(
-        "SET allow_experimental_window_view = 1, output_format_pretty_squash_ms = 0;"
-    )
+    client2.send("SET allow_experimental_window_view = 1")
     client2.expect(prompt)
 
     client1.send(
diff --git a/tests/queries/0_stateless/01082_window_view_watch_limit.py b/tests/queries/0_stateless/01082_window_view_watch_limit.py
index 07dbc232a63..12c8d295591 100755
--- a/tests/queries/0_stateless/01082_window_view_watch_limit.py
+++ b/tests/queries/0_stateless/01082_window_view_watch_limit.py
@@ -19,17 +19,13 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.expect(prompt)
 
-    client1.send(
-        "SET allow_experimental_analyzer = 0, output_format_pretty_squash_ms = 0;"
-    )
+    client1.send("SET allow_experimental_analyzer = 0")
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
     client1.send("SET window_view_heartbeat_interval = 1")
     client1.expect(prompt)
-    client2.send(
-        "SET allow_experimental_window_view = 1, output_format_pretty_squash_ms = 0;"
-    )
+    client2.send("SET allow_experimental_window_view = 1")
     client2.expect(prompt)
 
     client1.send("CREATE DATABASE IF NOT EXISTS 01082_window_view_watch_limit")
diff --git a/tests/queries/0_stateless/01509_output_format_pretty_row_numbers.sql b/tests/queries/0_stateless/01509_output_format_pretty_row_numbers.sql
index fb4c743d54b..f8ec0be74d7 100644
--- a/tests/queries/0_stateless/01509_output_format_pretty_row_numbers.sql
+++ b/tests/queries/0_stateless/01509_output_format_pretty_row_numbers.sql
@@ -1,5 +1,3 @@
-SET output_format_pretty_squash_ms = 0;
-
 SELECT * FROM numbers(10) FORMAT Pretty;
 SELECT * FROM numbers(10) FORMAT PrettyCompact;
 SELECT * FROM numbers(10) FORMAT PrettyCompactMonoBlock;
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_Pretty.reference b/tests/queries/0_stateless/02122_parallel_formatting_Pretty.reference
new file mode 100644
index 00000000000..72d346f82f7
--- /dev/null
+++ b/tests/queries/0_stateless/02122_parallel_formatting_Pretty.reference
@@ -0,0 +1,2 @@
+Pretty-1
+Pretty-2
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_Pretty.sh b/tests/queries/0_stateless/02122_parallel_formatting_Pretty.sh
new file mode 100755
index 00000000000..a72510b0614
--- /dev/null
+++ b/tests/queries/0_stateless/02122_parallel_formatting_Pretty.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+${CUR_DIR}/02122_parallel_formatting.lib Pretty
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.reference b/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.reference
new file mode 100644
index 00000000000..35761b294db
--- /dev/null
+++ b/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.reference
@@ -0,0 +1,2 @@
+PrettyCompact-1
+PrettyCompact-2
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.sh b/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.sh
new file mode 100755
index 00000000000..351f7ce0cd6
--- /dev/null
+++ b/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompact.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+${CUR_DIR}/02122_parallel_formatting.lib PrettyCompact
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.reference b/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.reference
new file mode 100644
index 00000000000..8b70c25b168
--- /dev/null
+++ b/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.reference
@@ -0,0 +1,2 @@
+PrettyCompactNoEscapes-1
+PrettyCompactNoEscapes-2
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.sh b/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.sh
new file mode 100755
index 00000000000..fd1e7a83ead
--- /dev/null
+++ b/tests/queries/0_stateless/02122_parallel_formatting_PrettyCompactNoEscapes.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+${CUR_DIR}/02122_parallel_formatting.lib PrettyCompactNoEscapes
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.reference b/tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.reference
new file mode 100644
index 00000000000..e2bfec67b7c
--- /dev/null
+++ b/tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.reference
@@ -0,0 +1,2 @@
+PrettyNoEscapes-1
+PrettyNoEscapes-2
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.sh b/tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.sh
new file mode 100755
index 00000000000..20e6d9c6973
--- /dev/null
+++ b/tests/queries/0_stateless/02122_parallel_formatting_PrettyNoEscapes.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+${CUR_DIR}/02122_parallel_formatting.lib PrettyNoEscapes
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.reference b/tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.reference
new file mode 100644
index 00000000000..fab8c8bde76
--- /dev/null
+++ b/tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.reference
@@ -0,0 +1,2 @@
+PrettySpace-1
+PrettySpace-2
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.sh b/tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.sh
new file mode 100755
index 00000000000..8af990d9113
--- /dev/null
+++ b/tests/queries/0_stateless/02122_parallel_formatting_PrettySpace.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+${CUR_DIR}/02122_parallel_formatting.lib PrettySpace
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.reference b/tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.reference
new file mode 100644
index 00000000000..154ee87e82a
--- /dev/null
+++ b/tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.reference
@@ -0,0 +1,2 @@
+PrettySpaceNoEscapes-1
+PrettySpaceNoEscapes-2
diff --git a/tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.sh b/tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.sh
new file mode 100755
index 00000000000..8fcbfbc16cf
--- /dev/null
+++ b/tests/queries/0_stateless/02122_parallel_formatting_PrettySpaceNoEscapes.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+${CUR_DIR}/02122_parallel_formatting.lib PrettySpaceNoEscapes
diff --git a/tests/queries/0_stateless/02375_pretty_formats.sql.j2 b/tests/queries/0_stateless/02375_pretty_formats.sql.j2
index 92e0a357138..cc61346d267 100644
--- a/tests/queries/0_stateless/02375_pretty_formats.sql.j2
+++ b/tests/queries/0_stateless/02375_pretty_formats.sql.j2
@@ -3,7 +3,6 @@
                   'PrettySpaceNoEscapesMonoBlock'] -%}
 
 select '{{ format }}';
-SET output_format_pretty_squash_ms = 0;
 select number as x, number + 1 as y from numbers(4) settings max_block_size=2 format {{ format }};
 
 {% endfor -%}
diff --git a/tests/queries/0_stateless/02731_pretty_time_based_squashing.reference b/tests/queries/0_stateless/02731_pretty_time_based_squashing.reference
deleted file mode 100644
index 653faf33a46..00000000000
--- a/tests/queries/0_stateless/02731_pretty_time_based_squashing.reference
+++ /dev/null
@@ -1,14 +0,0 @@
-┌─number─┐
-│      0 │
-└────────┘
-┌─number─┐
-│      1 │
-└────────┘
-┌─number─┐
-│      2 │
-└────────┘
-┌─number─┐
-│      0 │
-│      1 │
-│      2 │
-└────────┘
diff --git a/tests/queries/0_stateless/02731_pretty_time_based_squashing.sql b/tests/queries/0_stateless/02731_pretty_time_based_squashing.sql
deleted file mode 100644
index e5656c8e1ca..00000000000
--- a/tests/queries/0_stateless/02731_pretty_time_based_squashing.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-SELECT * FROM numbers(3) FORMAT PrettyCompact SETTINGS max_block_size = 1, max_threads = 1, output_format_pretty_squash_ms = 0;
-SELECT * FROM numbers(3) FORMAT PrettyCompact SETTINGS max_block_size = 1, max_threads = 1, output_format_pretty_squash_ms = 600_000;

From 5a44dc26e7048e7e2752cc87897157301fb1e278 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 13 May 2023 02:57:31 +0200
Subject: [PATCH 498/689] Fixes for clang-17

---
 programs/copier/ZooKeeperStaff.h              |  6 ++--
 programs/keeper/Keeper.cpp                    |  8 ++---
 src/Access/AccessChangesNotifier.cpp          | 12 +++----
 src/Access/DiskAccessStorage.cpp              |  4 +--
 src/Access/EnabledRoles.cpp                   | 12 +++----
 src/Access/MemoryAccessStorage.cpp            |  4 +--
 src/Access/ReplicatedAccessStorage.cpp        | 16 +++++-----
 src/Backups/BackupCoordinationRemote.cpp      |  6 ++--
 src/Backups/BackupEntriesCollector.cpp        |  8 ++---
 src/Backups/BackupImpl.cpp                    |  2 +-
 src/Backups/RestoreCoordinationRemote.cpp     |  6 ++--
 src/Client/Suggest.cpp                        |  4 +--
 src/Columns/ColumnArray.cpp                   |  4 +--
 src/Columns/ColumnCompressed.h                |  2 +-
 src/Columns/ColumnDecimal.cpp                 |  6 ++--
 src/Columns/ColumnFixedString.cpp             |  8 ++---
 src/Columns/ColumnMap.cpp                     |  4 +--
 src/Columns/ColumnNullable.cpp                |  4 +--
 src/Columns/ColumnSparse.cpp                  |  4 +--
 src/Columns/ColumnString.cpp                  |  8 ++---
 src/Columns/ColumnTuple.cpp                   |  6 ++--
 src/Columns/ColumnVector.cpp                  |  4 +--
 src/Common/ThreadPool.h                       | 16 +++++-----
 .../tests/gtest_concurrency_control.cpp       |  4 +--
 src/Coordination/KeeperDispatcher.cpp         |  4 +--
 src/Coordination/KeeperStorage.cpp            | 30 ++++++++---------
 src/Coordination/ZooKeeperDataReader.cpp      |  2 +-
 src/Dictionaries/DictionaryStructure.h        |  4 +--
 src/Formats/ProtobufSerializer.cpp            |  4 +--
 .../UserDefinedSQLObjectsBackup.cpp           |  8 ++---
 ...erDefinedSQLObjectsLoaderFromZooKeeper.cpp |  8 ++---
 src/IO/ParallelReadBuffer.cpp                 |  2 +-
 src/Interpreters/AsynchronousInsertQueue.cpp  |  4 +--
 src/Interpreters/ClusterDiscovery.cpp         |  2 +-
 src/Interpreters/executeQuery.cpp             | 10 +++---
 src/Interpreters/threadPoolCallbackRunner.h   | 10 +++---
 src/Processors/Executors/PipelineExecutor.cpp |  2 +-
 .../QueryPlan/ReadFromMergeTree.cpp           |  4 +--
 src/Processors/QueryPlan/ReadFromRemote.cpp   | 32 +++++++++----------
 .../Transforms/AggregatingTransform.h         |  2 +-
 src/QueryPipeline/RemoteQueryExecutor.cpp     | 12 +++----
 src/Server/HTTPHandler.cpp                    |  6 ++--
 src/Server/HTTPHandlerFactory.h               |  4 +--
 src/Server/MySQLHandler.cpp                   |  6 ++--
 src/Storages/HDFS/StorageHDFS.cpp             |  4 +--
 src/Storages/MergeTree/AsyncBlockIDsCache.cpp | 12 +++----
 src/Storages/MergeTree/MergeTreeData.cpp      |  8 ++---
 .../MergeTree/MergeTreePrefetchedReadPool.cpp |  6 ++--
 .../MergeTree/ReplicatedMergeTreeSink.cpp     |  6 ++--
 src/Storages/StorageDistributed.cpp           |  4 +--
 src/Storages/StorageReplicatedMergeTree.cpp   | 10 +++---
 src/Storages/System/StorageSystemReplicas.cpp |  6 ++--
 52 files changed, 186 insertions(+), 184 deletions(-)

diff --git a/programs/copier/ZooKeeperStaff.h b/programs/copier/ZooKeeperStaff.h
index 3d4a11186e3..36dcfa50842 100644
--- a/programs/copier/ZooKeeperStaff.h
+++ b/programs/copier/ZooKeeperStaff.h
@@ -175,7 +175,7 @@ public:
         Coordination::Stat stat{};
         String _some_data;
         auto watch_callback =
-                [stale = stale] (const Coordination::WatchResponse & rsp)
+                [my_stale = stale] (const Coordination::WatchResponse & rsp)
                 {
                     auto logger = &Poco::Logger::get("ClusterCopier");
                     if (rsp.error == Coordination::Error::ZOK)
@@ -184,11 +184,11 @@ public:
                         {
                             case Coordination::CREATED:
                                 LOG_DEBUG(logger, "CleanStateClock change: CREATED, at {}", rsp.path);
-                                stale->store(true);
+                                my_stale->store(true);
                                 break;
                             case Coordination::CHANGED:
                                 LOG_DEBUG(logger, "CleanStateClock change: CHANGED, at {}", rsp.path);
-                                stale->store(true);
+                                my_stale->store(true);
                         }
                     }
                 };
diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp
index 0eb3df58f30..3d1773260f5 100644
--- a/programs/keeper/Keeper.cpp
+++ b/programs/keeper/Keeper.cpp
@@ -498,18 +498,18 @@ try
 
         /// Prometheus (if defined and not setup yet with http_port)
         port_name = "prometheus.port";
-        createServer(listen_host, port_name, listen_try, [&, http_context = std::move(http_context)](UInt16 port) mutable
+        createServer(listen_host, port_name, listen_try, [&, my_http_context = std::move(http_context)](UInt16 port) mutable
         {
             Poco::Net::ServerSocket socket;
             auto address = socketBindListen(socket, listen_host, port);
-            socket.setReceiveTimeout(http_context->getReceiveTimeout());
-            socket.setSendTimeout(http_context->getSendTimeout());
+            socket.setReceiveTimeout(my_http_context->getReceiveTimeout());
+            socket.setSendTimeout(my_http_context->getSendTimeout());
             servers->emplace_back(
                 listen_host,
                 port_name,
                 "Prometheus: http://" + address.toString(),
                 std::make_unique<HTTPServer>(
-                    std::move(http_context), createPrometheusMainHandlerFactory(*this, config_getter(), async_metrics, "PrometheusHandler-factory"), server_pool, socket, http_params));
+                    std::move(my_http_context), createPrometheusMainHandlerFactory(*this, config_getter(), async_metrics, "PrometheusHandler-factory"), server_pool, socket, http_params));
         });
     }
 
diff --git a/src/Access/AccessChangesNotifier.cpp b/src/Access/AccessChangesNotifier.cpp
index 05516285efb..b27dda82142 100644
--- a/src/Access/AccessChangesNotifier.cpp
+++ b/src/Access/AccessChangesNotifier.cpp
@@ -47,10 +47,10 @@ scope_guard AccessChangesNotifier::subscribeForChanges(AccessEntityType type, co
     list.push_back(handler);
     auto handler_it = std::prev(list.end());
 
-    return [handlers=handlers, type, handler_it]
+    return [my_handlers = handlers, type, handler_it]
     {
-        std::lock_guard lock2{handlers->mutex};
-        auto & list2 = handlers->by_type[static_cast<size_t>(type)];
+        std::lock_guard lock2{my_handlers->mutex};
+        auto & list2 = my_handlers->by_type[static_cast<size_t>(type)];
         list2.erase(handler_it);
     };
 }
@@ -63,13 +63,13 @@ scope_guard AccessChangesNotifier::subscribeForChanges(const UUID & id, const On
     list.push_back(handler);
     auto handler_it = std::prev(list.end());
 
-    return [handlers=handlers, it, handler_it]
+    return [my_handlers = handlers, it, handler_it]
     {
-        std::lock_guard lock2{handlers->mutex};
+        std::lock_guard lock2{my_handlers->mutex};
         auto & list2 = it->second;
         list2.erase(handler_it);
         if (list2.empty())
-            handlers->by_id.erase(it);
+            my_handlers->by_id.erase(it);
     };
 }
 
diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp
index 710cf257b95..938881fafff 100644
--- a/src/Access/DiskAccessStorage.cpp
+++ b/src/Access/DiskAccessStorage.cpp
@@ -742,9 +742,9 @@ void DiskAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
     bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace);
     bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate);
 
-    restorer.addDataRestoreTask([this, entities = std::move(entities), replace_if_exists, throw_if_exists]
+    restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists]
     {
-        for (const auto & [id, entity] : entities)
+        for (const auto & [id, entity] : my_entities)
             insertWithID(id, entity, replace_if_exists, throw_if_exists, /* write_on_disk= */ true);
     });
 }
diff --git a/src/Access/EnabledRoles.cpp b/src/Access/EnabledRoles.cpp
index 456529da942..cebc075a6e3 100644
--- a/src/Access/EnabledRoles.cpp
+++ b/src/Access/EnabledRoles.cpp
@@ -26,10 +26,10 @@ scope_guard EnabledRoles::subscribeForChanges(const OnChangeHandler & handler) c
     handlers->list.push_back(handler);
     auto it = std::prev(handlers->list.end());
 
-    return [handlers=handlers, it]
+    return [my_handlers = handlers, it]
     {
-        std::lock_guard lock2{handlers->mutex};
-        handlers->list.erase(it);
+        std::lock_guard lock2{my_handlers->mutex};
+        my_handlers->list.erase(it);
     };
 }
 
@@ -53,10 +53,10 @@ void EnabledRoles::setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> &
         }
 
         notifications->join(scope_guard(
-            [info = info, handlers_to_notify = std::move(handlers_to_notify)]
+            [my_info = info, my_handlers_to_notify = std::move(handlers_to_notify)]
             {
-                for (const auto & handler : handlers_to_notify)
-                    handler(info);
+                for (const auto & handler : my_handlers_to_notify)
+                    handler(my_info);
             }));
     }
 }
diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp
index 8fcca235ee8..c4192ee4552 100644
--- a/src/Access/MemoryAccessStorage.cpp
+++ b/src/Access/MemoryAccessStorage.cpp
@@ -297,9 +297,9 @@ void MemoryAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
     bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace);
     bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate);
 
-    restorer.addDataRestoreTask([this, entities = std::move(entities), replace_if_exists, throw_if_exists]
+    restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists]
     {
-        for (const auto & [id, entity] : entities)
+        for (const auto & [id, entity] : my_entities)
             insertWithID(id, entity, replace_if_exists, throw_if_exists);
     });
 }
diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp
index 7a608c298b1..56d68be9268 100644
--- a/src/Access/ReplicatedAccessStorage.cpp
+++ b/src/Access/ReplicatedAccessStorage.cpp
@@ -525,9 +525,9 @@ void ReplicatedAccessStorage::refreshEntities(const zkutil::ZooKeeperPtr & zooke
     }
 
     const String zookeeper_uuids_path = zookeeper_path + "/uuid";
-    auto watch_entities_list = [watched_queue = watched_queue](const Coordination::WatchResponse &)
+    auto watch_entities_list = [my_watched_queue = watched_queue](const Coordination::WatchResponse &)
     {
-        [[maybe_unused]] bool push_result = watched_queue->push(UUIDHelpers::Nil);
+        [[maybe_unused]] bool push_result = my_watched_queue->push(UUIDHelpers::Nil);
     };
     Coordination::Stat stat;
     const auto entity_uuid_strs = zookeeper->getChildrenWatch(zookeeper_uuids_path, &stat, watch_entities_list);
@@ -592,10 +592,10 @@ void ReplicatedAccessStorage::refreshEntityNoLock(const zkutil::ZooKeeperPtr & z
 
 AccessEntityPtr ReplicatedAccessStorage::tryReadEntityFromZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) const
 {
-    const auto watch_entity = [watched_queue = watched_queue, id](const Coordination::WatchResponse & response)
+    const auto watch_entity = [my_watched_queue = watched_queue, id](const Coordination::WatchResponse & response)
     {
         if (response.type == Coordination::Event::CHANGED)
-            [[maybe_unused]] bool push_result = watched_queue->push(id);
+            [[maybe_unused]] bool push_result = my_watched_queue->push(id);
     };
 
     Coordination::Stat entity_stat;
@@ -680,12 +680,12 @@ void ReplicatedAccessStorage::backup(BackupEntriesCollector & backup_entries_col
 
     backup_entries_collector.addPostTask(
         [backup_entry = backup_entry_with_path.second,
-         zookeeper_path = zookeeper_path,
+         my_zookeeper_path = zookeeper_path,
          type,
          &backup_entries_collector,
          backup_coordination]
         {
-            for (const String & path : backup_coordination->getReplicatedAccessFilePaths(zookeeper_path, type))
+            for (const String & path : backup_coordination->getReplicatedAccessFilePaths(my_zookeeper_path, type))
                 backup_entries_collector.addBackupEntry(path, backup_entry);
         });
 }
@@ -708,9 +708,9 @@ void ReplicatedAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
     bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace);
     bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate);
 
-    restorer.addDataRestoreTask([this, entities = std::move(entities), replace_if_exists, throw_if_exists]
+    restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists]
     {
-        for (const auto & [id, entity] : entities)
+        for (const auto & [id, entity] : my_entities)
             insertWithID(id, entity, replace_if_exists, throw_if_exists);
     });
 }
diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp
index 91a7444c746..c1ef353c4eb 100644
--- a/src/Backups/BackupCoordinationRemote.cpp
+++ b/src/Backups/BackupCoordinationRemote.cpp
@@ -173,13 +173,13 @@ BackupCoordinationRemote::BackupCoordinationRemote(
         log,
         get_zookeeper_,
         keeper_settings,
-        [zookeeper_path = zookeeper_path, current_host = current_host, is_internal = is_internal]
+        [my_zookeeper_path = zookeeper_path, my_current_host = current_host, my_is_internal = is_internal]
         (WithRetries::FaultyKeeper & zk)
         {
             /// Recreate this ephemeral node to signal that we are alive.
-            if (is_internal)
+            if (my_is_internal)
             {
-                String alive_node_path = zookeeper_path + "/stage/alive|" + current_host;
+                String alive_node_path = my_zookeeper_path + "/stage/alive|" + my_current_host;
                 auto code = zk->tryCreate(alive_node_path, "", zkutil::CreateMode::Ephemeral);
 
                 if (code == Coordination::Error::ZNODEEXISTS)
diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp
index 8d2b39f11fa..485d59eff38 100644
--- a/src/Backups/BackupEntriesCollector.cpp
+++ b/src/Backups/BackupEntriesCollector.cpp
@@ -470,17 +470,17 @@ std::vector<std::pair<ASTPtr, StoragePtr>> BackupEntriesCollector::findTablesInD
     const auto & database_info = database_infos.at(database_name);
     const auto & database = database_info.database;
 
-    auto filter_by_table_name = [database_info = &database_info](const String & table_name)
+    auto filter_by_table_name = [my_database_info = &database_info](const String & table_name)
     {
         /// We skip inner tables of materialized views.
         if (table_name.starts_with(".inner_id."))
             return false;
 
-        if (database_info->tables.contains(table_name))
+        if (my_database_info->tables.contains(table_name))
             return true;
 
-        if (database_info->all_tables)
-            return !database_info->except_table_names.contains(table_name);
+        if (my_database_info->all_tables)
+            return !my_database_info->except_table_names.contains(table_name);
 
         return false;
     };
diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp
index 0ab1bf7f997..a8c6e16b268 100644
--- a/src/Backups/BackupImpl.cpp
+++ b/src/Backups/BackupImpl.cpp
@@ -208,7 +208,7 @@ void BackupImpl::openArchive()
         if (!reader->fileExists(archive_name))
             throw Exception(ErrorCodes::BACKUP_NOT_FOUND, "Backup {} not found", backup_name_for_logging);
         size_t archive_size = reader->getFileSize(archive_name);
-        archive_reader = createArchiveReader(archive_name, [reader=reader, archive_name]{ return reader->readFile(archive_name); }, archive_size);
+        archive_reader = createArchiveReader(archive_name, [my_reader = reader, archive_name]{ return my_reader->readFile(archive_name); }, archive_size);
         archive_reader->setPassword(archive_params.password);
     }
     else
diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp
index b2da013220d..860443665cf 100644
--- a/src/Backups/RestoreCoordinationRemote.cpp
+++ b/src/Backups/RestoreCoordinationRemote.cpp
@@ -34,13 +34,13 @@ RestoreCoordinationRemote::RestoreCoordinationRemote(
         log,
         get_zookeeper_,
         keeper_settings,
-        [zookeeper_path = zookeeper_path, current_host = current_host, is_internal = is_internal]
+        [my_zookeeper_path = zookeeper_path, my_current_host = current_host, my_is_internal = is_internal]
         (WithRetries::FaultyKeeper & zk)
         {
             /// Recreate this ephemeral node to signal that we are alive.
-            if (is_internal)
+            if (my_is_internal)
             {
-                String alive_node_path = zookeeper_path + "/stage/alive|" + current_host;
+                String alive_node_path = my_zookeeper_path + "/stage/alive|" + my_current_host;
                 auto code = zk->tryCreate(alive_node_path, "", zkutil::CreateMode::Ephemeral);
 
                 if (code == Coordination::Error::ZNODEEXISTS)
diff --git a/src/Client/Suggest.cpp b/src/Client/Suggest.cpp
index 7027f35d21a..4ffa828dd40 100644
--- a/src/Client/Suggest.cpp
+++ b/src/Client/Suggest.cpp
@@ -108,14 +108,14 @@ static String getLoadSuggestionQuery(Int32 suggestion_limit, bool basic_suggesti
 template <typename ConnectionType>
 void Suggest::load(ContextPtr context, const ConnectionParameters & connection_parameters, Int32 suggestion_limit)
 {
-    loading_thread = std::thread([context=Context::createCopy(context), connection_parameters, suggestion_limit, this]
+    loading_thread = std::thread([my_context = Context::createCopy(context), connection_parameters, suggestion_limit, this]
     {
         ThreadStatus thread_status;
         for (size_t retry = 0; retry < 10; ++retry)
         {
             try
             {
-                auto connection = ConnectionType::createConnection(connection_parameters, context);
+                auto connection = ConnectionType::createConnection(connection_parameters, my_context);
                 fetch(*connection, connection_parameters.timeouts, getLoadSuggestionQuery(suggestion_limit, std::is_same_v<ConnectionType, LocalConnection>));
             }
             catch (const Exception & e)
diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp
index cbeebe52141..74512d1669b 100644
--- a/src/Columns/ColumnArray.cpp
+++ b/src/Columns/ColumnArray.cpp
@@ -942,9 +942,9 @@ ColumnPtr ColumnArray::compress() const
     size_t byte_size = data_compressed->byteSize() + offsets_compressed->byteSize();
 
     return ColumnCompressed::create(size(), byte_size,
-        [data_compressed = std::move(data_compressed), offsets_compressed = std::move(offsets_compressed)]
+        [my_data_compressed = std::move(data_compressed), my_offsets_compressed = std::move(offsets_compressed)]
         {
-            return ColumnArray::create(data_compressed->decompress(), offsets_compressed->decompress());
+            return ColumnArray::create(my_data_compressed->decompress(), my_offsets_compressed->decompress());
         });
 }
 
diff --git a/src/Columns/ColumnCompressed.h b/src/Columns/ColumnCompressed.h
index d747f8ef5f4..bfe7cdb4924 100644
--- a/src/Columns/ColumnCompressed.h
+++ b/src/Columns/ColumnCompressed.h
@@ -64,7 +64,7 @@ public:
         return ColumnCompressed::create(
             size,
             bytes,
-            [column = std::move(column)]{ return column; });
+            [my_column = std::move(column)]{ return my_column; });
     }
 
     /// Helper methods for compression.
diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp
index d9fed8c87e5..8e5792934cf 100644
--- a/src/Columns/ColumnDecimal.cpp
+++ b/src/Columns/ColumnDecimal.cpp
@@ -386,11 +386,11 @@ ColumnPtr ColumnDecimal<T>::compress() const
 
     const size_t compressed_size = compressed->size();
     return ColumnCompressed::create(data_size, compressed_size,
-        [compressed = std::move(compressed), column_size = data_size, scale = this->scale]
+        [my_compressed = std::move(compressed), column_size = data_size, my_scale = this->scale]
         {
-            auto res = ColumnDecimal<T>::create(column_size, scale);
+            auto res = ColumnDecimal<T>::create(column_size, my_scale);
             ColumnCompressed::decompressBuffer(
-                compressed->data(), res->getData().data(), compressed->size(), column_size * sizeof(T));
+                my_compressed->data(), res->getData().data(), my_compressed->size(), column_size * sizeof(T));
             return res;
         });
 }
diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp
index f708405667b..24b5c435ecd 100644
--- a/src/Columns/ColumnFixedString.cpp
+++ b/src/Columns/ColumnFixedString.cpp
@@ -393,13 +393,13 @@ ColumnPtr ColumnFixedString::compress() const
     const size_t column_size = size();
     const size_t compressed_size = compressed->size();
     return ColumnCompressed::create(column_size, compressed_size,
-        [compressed = std::move(compressed), column_size, n = n]
+        [my_compressed = std::move(compressed), column_size, my_n = n]
         {
-            size_t chars_size = n * column_size;
-            auto res = ColumnFixedString::create(n);
+            size_t chars_size = my_n * column_size;
+            auto res = ColumnFixedString::create(my_n);
             res->getChars().resize(chars_size);
             ColumnCompressed::decompressBuffer(
-                compressed->data(), res->getChars().data(), compressed->size(), chars_size);
+                my_compressed->data(), res->getChars().data(), my_compressed->size(), chars_size);
             return res;
         });
 }
diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp
index b533b68b98d..7f5693bf470 100644
--- a/src/Columns/ColumnMap.cpp
+++ b/src/Columns/ColumnMap.cpp
@@ -312,9 +312,9 @@ ColumnPtr ColumnMap::compress() const
     const auto byte_size = compressed->byteSize();
     /// The order of evaluation of function arguments is unspecified
     /// and could cause interacting with object in moved-from state
-    return ColumnCompressed::create(size(), byte_size, [compressed = std::move(compressed)]
+    return ColumnCompressed::create(size(), byte_size, [my_compressed = std::move(compressed)]
     {
-        return ColumnMap::create(compressed->decompress());
+        return ColumnMap::create(my_compressed->decompress());
     });
 }
 
diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp
index 5ac7d07364b..aa04fc910a5 100644
--- a/src/Columns/ColumnNullable.cpp
+++ b/src/Columns/ColumnNullable.cpp
@@ -644,9 +644,9 @@ ColumnPtr ColumnNullable::compress() const
     size_t byte_size = nested_column->byteSize() + null_map->byteSize();
 
     return ColumnCompressed::create(size(), byte_size,
-        [nested_column = std::move(nested_compressed), null_map = std::move(null_map_compressed)]
+        [my_nested_column = std::move(nested_compressed), my_null_map = std::move(null_map_compressed)]
         {
-            return ColumnNullable::create(nested_column->decompress(), null_map->decompress());
+            return ColumnNullable::create(my_nested_column->decompress(), my_null_map->decompress());
         });
 }
 
diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp
index fbab61c879d..9e4609ab4b5 100644
--- a/src/Columns/ColumnSparse.cpp
+++ b/src/Columns/ColumnSparse.cpp
@@ -738,9 +738,9 @@ ColumnPtr ColumnSparse::compress() const
     size_t byte_size = values_compressed->byteSize() + offsets_compressed->byteSize();
 
     return ColumnCompressed::create(size(), byte_size,
-        [values_compressed = std::move(values_compressed), offsets_compressed = std::move(offsets_compressed), size = size()]
+        [my_values_compressed = std::move(values_compressed), my_offsets_compressed = std::move(offsets_compressed), size = size()]
         {
-            return ColumnSparse::create(values_compressed->decompress(), offsets_compressed->decompress(), size);
+            return ColumnSparse::create(my_values_compressed->decompress(), my_offsets_compressed->decompress(), size);
         });
 }
 
diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp
index 9d0d39e9fde..de96c57700b 100644
--- a/src/Columns/ColumnString.cpp
+++ b/src/Columns/ColumnString.cpp
@@ -532,8 +532,8 @@ ColumnPtr ColumnString::compress() const
     const size_t offsets_compressed_size = offsets_compressed->size();
     return ColumnCompressed::create(source_offsets_elements, chars_compressed_size + offsets_compressed_size,
         [
-            chars_compressed = std::move(chars_compressed),
-            offsets_compressed = std::move(offsets_compressed),
+            my_chars_compressed = std::move(chars_compressed),
+            my_offsets_compressed = std::move(offsets_compressed),
             source_chars_size,
             source_offsets_elements
         ]
@@ -544,10 +544,10 @@ ColumnPtr ColumnString::compress() const
             res->getOffsets().resize(source_offsets_elements);
 
             ColumnCompressed::decompressBuffer(
-                chars_compressed->data(), res->getChars().data(), chars_compressed->size(), source_chars_size);
+                my_chars_compressed->data(), res->getChars().data(), my_chars_compressed->size(), source_chars_size);
 
             ColumnCompressed::decompressBuffer(
-                offsets_compressed->data(), res->getOffsets().data(), offsets_compressed->size(), source_offsets_elements * sizeof(Offset));
+                my_offsets_compressed->data(), res->getOffsets().data(), my_offsets_compressed->size(), source_offsets_elements * sizeof(Offset));
 
             return res;
         });
diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp
index 369842c7281..9f659aa7a7c 100644
--- a/src/Columns/ColumnTuple.cpp
+++ b/src/Columns/ColumnTuple.cpp
@@ -552,11 +552,11 @@ ColumnPtr ColumnTuple::compress() const
     }
 
     return ColumnCompressed::create(size(), byte_size,
-        [compressed = std::move(compressed)]() mutable
+        [my_compressed = std::move(compressed)]() mutable
         {
-            for (auto & column : compressed)
+            for (auto & column : my_compressed)
                 column = column->decompress();
-            return ColumnTuple::create(compressed);
+            return ColumnTuple::create(my_compressed);
         });
 }
 
diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp
index d78f97426b7..f2fe343a371 100644
--- a/src/Columns/ColumnVector.cpp
+++ b/src/Columns/ColumnVector.cpp
@@ -927,11 +927,11 @@ ColumnPtr ColumnVector<T>::compress() const
 
     const size_t compressed_size = compressed->size();
     return ColumnCompressed::create(data_size, compressed_size,
-        [compressed = std::move(compressed), column_size = data_size]
+        [my_compressed = std::move(compressed), column_size = data_size]
         {
             auto res = ColumnVector<T>::create(column_size);
             ColumnCompressed::decompressBuffer(
-                compressed->data(), res->getData().data(), compressed->size(), column_size * sizeof(T));
+                my_compressed->data(), res->getData().data(), my_compressed->size(), column_size * sizeof(T));
             return res;
         });
 }
diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h
index 12bc7815784..b92cb256b13 100644
--- a/src/Common/ThreadPool.h
+++ b/src/Common/ThreadPool.h
@@ -206,21 +206,21 @@ public:
         /// - If this will throw an exception, the destructor won't be called
         /// - this pointer cannot be passed in the lambda, since after detach() it will not be valid
         GlobalThreadPool::instance().scheduleOrThrow([
-            state = state,
-            func = std::forward<Function>(func),
-            args = std::make_tuple(std::forward<Args>(args)...)]() mutable /// mutable is needed to destroy capture
+            my_state = state,
+            my_func = std::forward<Function>(func),
+            my_args = std::make_tuple(std::forward<Args>(args)...)]() mutable /// mutable is needed to destroy capture
         {
             SCOPE_EXIT(
-                state->thread_id = std::thread::id();
-                state->event.set();
+                my_state->thread_id = std::thread::id();
+                my_state->event.set();
             );
 
-            state->thread_id = std::this_thread::get_id();
+            my_state->thread_id = std::this_thread::get_id();
 
             /// This moves are needed to destroy function and arguments before exit.
             /// It will guarantee that after ThreadFromGlobalPool::join all captured params are destroyed.
-            auto function = std::move(func);
-            auto arguments = std::move(args);
+            auto function = std::move(my_func);
+            auto arguments = std::move(my_args);
 
             /// Thread status holds raw pointer on query context, thus it always must be destroyed
             /// before sending signal that permits to join this thread.
diff --git a/src/Common/tests/gtest_concurrency_control.cpp b/src/Common/tests/gtest_concurrency_control.cpp
index 852f4e72cf7..8e5b89a72a0 100644
--- a/src/Common/tests/gtest_concurrency_control.cpp
+++ b/src/Common/tests/gtest_concurrency_control.cpp
@@ -234,12 +234,12 @@ TEST(ConcurrencyControl, MultipleThreads)
             while (auto slot = slots->tryAcquire())
             {
                 std::unique_lock lock{threads_mutex};
-                threads.emplace_back([&, slot = std::move(slot)]
+                threads.emplace_back([&, my_slot = std::move(slot)]
                 {
                     pcg64 rng(randomSeed());
                     std::uniform_int_distribution<size_t> distribution(1, cfg_work_us);
                     size_t steps = distribution(rng);
-                    for (size_t step = 0; step < steps; step++)
+                    for (size_t step = 0; step < steps; ++step)
                     {
                         sleepForMicroseconds(distribution(rng)); // emulate work
                         spawn_threads(); // upscale
diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp
index 2cde7c2465e..d64134f3024 100644
--- a/src/Coordination/KeeperDispatcher.cpp
+++ b/src/Coordination/KeeperDispatcher.cpp
@@ -471,9 +471,9 @@ void KeeperDispatcher::shutdown()
             const auto raft_result = server->putRequestBatch(close_requests);
             auto sessions_closing_done_promise = std::make_shared<std::promise<void>>();
             auto sessions_closing_done = sessions_closing_done_promise->get_future();
-            raft_result->when_ready([sessions_closing_done_promise = std::move(sessions_closing_done_promise)](
+            raft_result->when_ready([my_sessions_closing_done_promise = std::move(sessions_closing_done_promise)](
                                         nuraft::cmd_result<nuraft::ptr<nuraft::buffer>> & /*result*/,
-                                        nuraft::ptr<std::exception> & /*exception*/) { sessions_closing_done_promise->set_value(); });
+                                        nuraft::ptr<std::exception> & /*exception*/) { my_sessions_closing_done_promise->set_value(); });
 
             auto session_shutdown_timeout = configuration_and_settings->coordination_settings->session_shutdown_timeout.totalMilliseconds();
             if (sessions_closing_done.wait_for(std::chrono::milliseconds(session_shutdown_timeout)) != std::future_status::ready)
diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp
index a838de07ecb..58fccc83e6a 100644
--- a/src/Coordination/KeeperStorage.cpp
+++ b/src/Coordination/KeeperStorage.cpp
@@ -342,34 +342,34 @@ void KeeperStorage::UncommittedState::applyDelta(const Delta & delta)
     auto & [node, acls, last_applied_zxid] = nodes.at(delta.path);
 
     std::visit(
-        [&, &node = node, &acls = acls, &last_applied_zxid = last_applied_zxid]<typename DeltaType>(const DeltaType & operation)
+        [&, &my_node = node, &my_acls = acls, &my_last_applied_zxid = last_applied_zxid]<typename DeltaType>(const DeltaType & operation)
         {
             if constexpr (std::same_as<DeltaType, CreateNodeDelta>)
             {
-                assert(!node);
-                node = std::make_shared<Node>();
-                node->stat = operation.stat;
-                node->setData(operation.data);
-                acls = operation.acls;
-                last_applied_zxid = delta.zxid;
+                assert(!my_node);
+                my_node = std::make_shared<Node>();
+                my_node->stat = operation.stat;
+                my_node->setData(operation.data);
+                my_acls = operation.acls;
+                my_last_applied_zxid = delta.zxid;
             }
             else if constexpr (std::same_as<DeltaType, RemoveNodeDelta>)
             {
-                assert(node);
-                node = nullptr;
-                last_applied_zxid = delta.zxid;
+                assert(my_node);
+                my_node = nullptr;
+                my_last_applied_zxid = delta.zxid;
             }
             else if constexpr (std::same_as<DeltaType, UpdateNodeDelta>)
             {
-                assert(node);
-                node->invalidateDigestCache();
+                assert(my_node);
+                my_node->invalidateDigestCache();
                 operation.update_fn(*node);
-                last_applied_zxid = delta.zxid;
+                my_last_applied_zxid = delta.zxid;
             }
             else if constexpr (std::same_as<DeltaType, SetACLDelta>)
             {
-                acls = operation.acls;
-                last_applied_zxid = delta.zxid;
+                my_acls = operation.acls;
+                my_last_applied_zxid = delta.zxid;
             }
         },
         delta.operation);
diff --git a/src/Coordination/ZooKeeperDataReader.cpp b/src/Coordination/ZooKeeperDataReader.cpp
index 2d4f64e033f..bdc462f3ea0 100644
--- a/src/Coordination/ZooKeeperDataReader.cpp
+++ b/src/Coordination/ZooKeeperDataReader.cpp
@@ -140,7 +140,7 @@ int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in, Poco::L
         if (itr.key != "/")
         {
             auto parent_path = parentPath(itr.key);
-            storage.container.updateValue(parent_path, [path = itr.key] (KeeperStorage::Node & value) { value.addChild(getBaseName(path)); value.stat.numChildren++; });
+            storage.container.updateValue(parent_path, [my_path = itr.key] (KeeperStorage::Node & value) { value.addChild(getBaseName(my_path)); ++value.stat.numChildren; });
         }
     }
 
diff --git a/src/Dictionaries/DictionaryStructure.h b/src/Dictionaries/DictionaryStructure.h
index c77ae291c34..f726a8a2a46 100644
--- a/src/Dictionaries/DictionaryStructure.h
+++ b/src/Dictionaries/DictionaryStructure.h
@@ -80,10 +80,10 @@ struct DictionaryAttributeType
 template <typename F>
 constexpr void callOnDictionaryAttributeType(AttributeUnderlyingType type, F && func)
 {
-    static_for<AttributeUnderlyingType>([type, func = std::forward<F>(func)](auto other)
+    static_for<AttributeUnderlyingType>([type, my_func = std::forward<F>(func)](auto other)
     {
         if (type == other)
-            func(DictionaryAttributeType<other>{});
+            my_func(DictionaryAttributeType<other>{});
     });
 }
 
diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp
index cdcd42713ec..8bdef0e7d3f 100644
--- a/src/Formats/ProtobufSerializer.cpp
+++ b/src/Formats/ProtobufSerializer.cpp
@@ -3005,10 +3005,10 @@ namespace
             bool google_wrappers_special_treatment)
         {
             root_serializer_ptr = std::make_shared<ProtobufSerializer *>();
-            get_root_desc_function = [root_serializer_ptr = root_serializer_ptr](size_t indent) -> String
+            get_root_desc_function = [my_root_serializer_ptr = root_serializer_ptr](size_t indent) -> String
             {
                 WriteBufferFromOwnString buf;
-                (*root_serializer_ptr)->describeTree(buf, indent);
+                (*my_root_serializer_ptr)->describeTree(buf, indent);
                 return buf.str();
             };
 
diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp
index 53d8ebc7b61..6920e8ce2c2 100644
--- a/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp
+++ b/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp
@@ -56,18 +56,18 @@ void backupUserDefinedSQLObjects(
     // They will only be returned for one of the hosts below, for the rest an empty list.
     // See also BackupCoordinationReplicatedSQLObjects class.
     backup_entries_collector.addPostTask(
-        [backup_entries = std::move(backup_entries),
-         replication_id = std::move(replication_id),
+        [my_backup_entries = std::move(backup_entries),
+         my_replication_id = std::move(replication_id),
          object_type,
          &backup_entries_collector,
          backup_coordination]
         {
-            auto dirs = backup_coordination->getReplicatedSQLObjectsDirs(replication_id, object_type);
+            auto dirs = backup_coordination->getReplicatedSQLObjectsDirs(my_replication_id, object_type);
 
             for (const auto & dir : dirs)
             {
                 fs::path dir_fs{dir};
-                for (const auto & [file_name, entry] : backup_entries)
+                for (const auto & [file_name, entry] : my_backup_entries)
                 {
                     backup_entries_collector.addBackupEntry(dir_fs / file_name, entry);
                 }
diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsLoaderFromZooKeeper.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsLoaderFromZooKeeper.cpp
index 284adeb2c2d..76ebcf769d5 100644
--- a/src/Functions/UserDefined/UserDefinedSQLObjectsLoaderFromZooKeeper.cpp
+++ b/src/Functions/UserDefined/UserDefinedSQLObjectsLoaderFromZooKeeper.cpp
@@ -283,11 +283,11 @@ bool UserDefinedSQLObjectsLoaderFromZooKeeper::getObjectDataAndSetWatch(
     UserDefinedSQLObjectType object_type,
     const String & object_name)
 {
-    const auto object_watcher = [watch_queue = watch_queue, object_type, object_name](const Coordination::WatchResponse & response)
+    const auto object_watcher = [my_watch_queue = watch_queue, object_type, object_name](const Coordination::WatchResponse & response)
     {
         if (response.type == Coordination::Event::CHANGED)
         {
-            [[maybe_unused]] bool inserted = watch_queue->emplace(object_type, object_name);
+            [[maybe_unused]] bool inserted = my_watch_queue->emplace(object_type, object_name);
             /// `inserted` can be false if `watch_queue` was already finalized (which happens when stopWatching() is called).
         }
         /// Event::DELETED is processed as child event by getChildren watch
@@ -346,9 +346,9 @@ ASTPtr UserDefinedSQLObjectsLoaderFromZooKeeper::tryLoadObject(
 Strings UserDefinedSQLObjectsLoaderFromZooKeeper::getObjectNamesAndSetWatch(
     const zkutil::ZooKeeperPtr & zookeeper, UserDefinedSQLObjectType object_type)
 {
-    auto object_list_watcher = [watch_queue = watch_queue, object_type](const Coordination::WatchResponse &)
+    auto object_list_watcher = [my_watch_queue = watch_queue, object_type](const Coordination::WatchResponse &)
     {
-        [[maybe_unused]] bool inserted = watch_queue->emplace(object_type, "");
+        [[maybe_unused]] bool inserted = my_watch_queue->emplace(object_type, "");
         /// `inserted` can be false if `watch_queue` was already finalized (which happens when stopWatching() is called).
     };
 
diff --git a/src/IO/ParallelReadBuffer.cpp b/src/IO/ParallelReadBuffer.cpp
index b2ba670ca44..6188fc210be 100644
--- a/src/IO/ParallelReadBuffer.cpp
+++ b/src/IO/ParallelReadBuffer.cpp
@@ -87,7 +87,7 @@ bool ParallelReadBuffer::addReaderToPool()
     auto worker = read_workers.emplace_back(std::make_shared<ReadWorker>(std::move(reader), range_start, size));
 
     ++active_working_reader;
-    schedule([this, worker = std::move(worker)]() mutable { readerThreadFunction(std::move(worker)); }, 0);
+    schedule([this, my_worker = std::move(worker)]() mutable { readerThreadFunction(std::move(my_worker)); }, 0);
 
     return true;
 }
diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp
index 142d0cc5373..dc2310cfebf 100644
--- a/src/Interpreters/AsynchronousInsertQueue.cpp
+++ b/src/Interpreters/AsynchronousInsertQueue.cpp
@@ -190,9 +190,9 @@ void AsynchronousInsertQueue::scheduleDataProcessingJob(const InsertQuery & key,
 {
     /// Wrap 'unique_ptr' with 'shared_ptr' to make this
     /// lambda copyable and allow to save it to the thread pool.
-    pool.scheduleOrThrowOnError([key, global_context, data = std::make_shared<InsertDataPtr>(std::move(data))]() mutable
+    pool.scheduleOrThrowOnError([key, global_context, my_data = std::make_shared<InsertDataPtr>(std::move(data))]() mutable
     {
-        processData(key, std::move(*data), std::move(global_context));
+        processData(key, std::move(*my_data), std::move(global_context));
     });
 }
 
diff --git a/src/Interpreters/ClusterDiscovery.cpp b/src/Interpreters/ClusterDiscovery.cpp
index a55e0e2a639..02046dec066 100644
--- a/src/Interpreters/ClusterDiscovery.cpp
+++ b/src/Interpreters/ClusterDiscovery.cpp
@@ -149,7 +149,7 @@ Strings ClusterDiscovery::getNodeNames(zkutil::ZooKeeperPtr & zk,
                                        int * version,
                                        bool set_callback)
 {
-    auto watch_callback = [cluster_name, clusters_to_update=clusters_to_update](auto) { clusters_to_update->set(cluster_name); };
+    auto watch_callback = [cluster_name, my_clusters_to_update = clusters_to_update](auto) { my_clusters_to_update->set(cluster_name); };
 
     Coordination::Stat stat;
     Strings nodes = zk->getChildrenWatch(getShardsListPath(zk_root), &stat, set_callback ? watch_callback : Coordination::WatchCallback{});
diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index 3a6b1f2124f..9a2750f399c 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -922,7 +922,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
             auto finish_callback = [elem,
                                     context,
                                     ast,
-                                    can_use_query_cache = can_use_query_cache,
+                                    my_can_use_query_cache = can_use_query_cache,
                                     enable_writes_to_query_cache = settings.enable_writes_to_query_cache,
                                     query_cache_store_results_of_queries_with_nondeterministic_functions = settings.query_cache_store_results_of_queries_with_nondeterministic_functions,
                                     log_queries,
@@ -940,7 +940,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
                 auto query_cache = context->getQueryCache();
                 if (query_cache != nullptr
                     && pulling_pipeline
-                    && can_use_query_cache && enable_writes_to_query_cache
+                    && my_can_use_query_cache && enable_writes_to_query_cache
                     && (!astContainsNonDeterministicFunctions(ast, context) || query_cache_store_results_of_queries_with_nondeterministic_functions))
                 {
                     query_pipeline.finalizeWriteInQueryCache();
@@ -1071,7 +1071,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
                                        log_queries,
                                        log_queries_min_type = settings.log_queries_min_type,
                                        log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(),
-                                       quota(quota),
+                                       my_quota(quota),
                                        status_info_to_query_log,
                                        implicit_txn_control,
                                        execute_implicit_tcl_query,
@@ -1082,8 +1082,8 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
                 else if (auto txn = context->getCurrentTransaction())
                     txn->onException();
 
-                if (quota)
-                    quota->used(QuotaType::ERRORS, 1, /* check_exceeded = */ false);
+                if (my_quota)
+                    my_quota->used(QuotaType::ERRORS, 1, /* check_exceeded = */ false);
 
                 elem.type = QueryLogElementType::EXCEPTION_WHILE_PROCESSING;
                 elem.exception_code = getCurrentExceptionCode();
diff --git a/src/Interpreters/threadPoolCallbackRunner.h b/src/Interpreters/threadPoolCallbackRunner.h
index 6151f7b1de6..b8805722346 100644
--- a/src/Interpreters/threadPoolCallbackRunner.h
+++ b/src/Interpreters/threadPoolCallbackRunner.h
@@ -17,9 +17,9 @@ using ThreadPoolCallbackRunner = std::function<std::future<Result>(Callback &&,
 template <typename Result, typename Callback = std::function<Result()>>
 ThreadPoolCallbackRunner<Result, Callback> threadPoolCallbackRunner(ThreadPool & pool, const std::string & thread_name)
 {
-    return [pool = &pool, thread_group = CurrentThread::getGroup(), thread_name](Callback && callback, int64_t priority) mutable -> std::future<Result>
+    return [my_pool = &pool, thread_group = CurrentThread::getGroup(), thread_name](Callback && callback, int64_t priority) mutable -> std::future<Result>
     {
-        auto task = std::make_shared<std::packaged_task<Result()>>([thread_group, thread_name, callback = std::move(callback)]() mutable -> Result
+        auto task = std::make_shared<std::packaged_task<Result()>>([thread_group, thread_name, my_callback = std::move(callback)]() mutable -> Result
         {
             if (thread_group)
                 CurrentThread::attachToGroup(thread_group);
@@ -29,7 +29,7 @@ ThreadPoolCallbackRunner<Result, Callback> threadPoolCallbackRunner(ThreadPool &
                     /// Release all captutred resources before detaching thread group
                     /// Releasing has to use proper memory tracker which has been set here before callback
 
-                    [[maybe_unused]] auto tmp = std::move(callback);
+                    [[maybe_unused]] auto tmp = std::move(my_callback);
                 }
 
                 if (thread_group)
@@ -39,13 +39,13 @@ ThreadPoolCallbackRunner<Result, Callback> threadPoolCallbackRunner(ThreadPool &
 
             setThreadName(thread_name.data());
 
-            return callback();
+            return my_callback();
         });
 
         auto future = task->get_future();
 
         /// ThreadPool is using "bigger is higher priority" instead of "smaller is more priority".
-        pool->scheduleOrThrow([task = std::move(task)]{ (*task)(); }, -priority);
+        my_pool->scheduleOrThrow([my_task = std::move(task)]{ (*my_task)(); }, -priority);
 
         return future;
     };
diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp
index abb98412b12..f523e7b7cf9 100644
--- a/src/Processors/Executors/PipelineExecutor.cpp
+++ b/src/Processors/Executors/PipelineExecutor.cpp
@@ -327,7 +327,7 @@ void PipelineExecutor::spawnThreads()
         tasks.upscale(thread_num + 1);
 
         /// Start new thread
-        pool->scheduleOrThrowOnError([this, thread_num, thread_group = CurrentThread::getGroup(), slot = std::move(slot)]
+        pool->scheduleOrThrowOnError([this, thread_num, thread_group = CurrentThread::getGroup(), my_slot = std::move(slot)]
         {
             SCOPE_EXIT_SAFE(
                 if (thread_group)
diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
index be6a5d33b74..1c5b625656c 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
@@ -669,10 +669,10 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder(
 
     /// Let's split ranges to avoid reading much data.
     auto split_ranges
-        = [rows_granularity = data_settings->index_granularity, max_block_size = max_block_size](const auto & ranges, int direction)
+        = [rows_granularity = data_settings->index_granularity, my_max_block_size = max_block_size](const auto & ranges, int direction)
     {
         MarkRanges new_ranges;
-        const size_t max_marks_in_range = (max_block_size + rows_granularity - 1) / rows_granularity;
+        const size_t max_marks_in_range = (my_max_block_size + rows_granularity - 1) / rows_granularity;
         size_t marks_in_range = 1;
 
         if (direction == 1)
diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp
index ca603c53707..ed740e3e242 100644
--- a/src/Processors/QueryPlan/ReadFromRemote.cpp
+++ b/src/Processors/QueryPlan/ReadFromRemote.cpp
@@ -146,31 +146,31 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream
     }
 
     auto lazily_create_stream = [
-            shard = shard, shard_count = shard_count, query = shard.query, header = shard.header,
-            context = context, throttler = throttler,
-            main_table = main_table, table_func_ptr = table_func_ptr,
-            scalars = scalars, external_tables = external_tables,
-            stage = stage, local_delay = shard.local_delay,
+            my_shard = shard, my_shard_count = shard_count, query = shard.query, header = shard.header,
+            my_context = context, my_throttler = throttler,
+            my_main_table = main_table, my_table_func_ptr = table_func_ptr,
+            my_scalars = scalars, my_external_tables = external_tables,
+            my_stage = stage, local_delay = shard.local_delay,
             add_agg_info, add_totals, add_extremes, async_read, async_query_sending]() mutable
         -> QueryPipelineBuilder
     {
-        auto current_settings = context->getSettingsRef();
+        auto current_settings = my_context->getSettingsRef();
         auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(
             current_settings).getSaturated(
                 current_settings.max_execution_time);
         std::vector<ConnectionPoolWithFailover::TryResult> try_results;
         try
         {
-            if (table_func_ptr)
-                try_results = shard.shard_info.pool->getManyForTableFunction(timeouts, &current_settings, PoolMode::GET_MANY);
+            if (my_table_func_ptr)
+                try_results = my_shard.shard_info.pool->getManyForTableFunction(timeouts, &current_settings, PoolMode::GET_MANY);
             else
-                try_results = shard.shard_info.pool->getManyChecked(timeouts, &current_settings, PoolMode::GET_MANY, main_table.getQualifiedName());
+                try_results = my_shard.shard_info.pool->getManyChecked(timeouts, &current_settings, PoolMode::GET_MANY, my_main_table.getQualifiedName());
         }
         catch (const Exception & ex)
         {
             if (ex.code() == ErrorCodes::ALL_CONNECTION_TRIES_FAILED)
                 LOG_WARNING(&Poco::Logger::get("ClusterProxy::SelectStreamFactory"),
-                    "Connections to remote replicas of local shard {} failed, will use stale local replica", shard.shard_info.shard_num);
+                    "Connections to remote replicas of local shard {} failed, will use stale local replica", my_shard.shard_info.shard_num);
             else
                 throw;
         }
@@ -185,11 +185,11 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream
         if (try_results.empty() || local_delay < max_remote_delay)
         {
             auto plan = createLocalPlan(
-                query, header, context, stage, shard.shard_info.shard_num, shard_count, 0, 0, /*coordinator=*/nullptr);
+                query, header, my_context, my_stage, my_shard.shard_info.shard_num, my_shard_count, 0, 0, /*coordinator=*/nullptr);
 
             return std::move(*plan->buildQueryPipeline(
-                QueryPlanOptimizationSettings::fromContext(context),
-                BuildQueryPipelineSettings::fromContext(context)));
+                QueryPlanOptimizationSettings::fromContext(my_context),
+                BuildQueryPipelineSettings::fromContext(my_context)));
         }
         else
         {
@@ -200,10 +200,10 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream
 
             String query_string = formattedAST(query);
 
-            scalars["_shard_num"]
-                = Block{{DataTypeUInt32().createColumnConst(1, shard.shard_info.shard_num), std::make_shared<DataTypeUInt32>(), "_shard_num"}};
+            my_scalars["_shard_num"]
+                = Block{{DataTypeUInt32().createColumnConst(1, my_shard.shard_info.shard_num), std::make_shared<DataTypeUInt32>(), "_shard_num"}};
             auto remote_query_executor = std::make_shared<RemoteQueryExecutor>(
-                std::move(connections), query_string, header, context, throttler, scalars, external_tables, stage);
+                std::move(connections), query_string, header, my_context, my_throttler, my_scalars, my_external_tables, my_stage);
 
             auto pipe = createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending);
             QueryPipelineBuilder builder;
diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h
index beb23a91c88..38baa4d0394 100644
--- a/src/Processors/Transforms/AggregatingTransform.h
+++ b/src/Processors/Transforms/AggregatingTransform.h
@@ -107,7 +107,7 @@ struct ManyAggregatedData
                 {
                     // variant is moved here and will be destroyed in the destructor of the lambda function.
                     pool->trySchedule(
-                        [variant = std::move(variant), thread_group = CurrentThread::getGroup()]()
+                        [my_variant = std::move(variant), thread_group = CurrentThread::getGroup()]()
                         {
                             SCOPE_EXIT_SAFE(
                                 if (thread_group)
diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp
index 8ca561bd309..991876eccbd 100644
--- a/src/QueryPipeline/RemoteQueryExecutor.cpp
+++ b/src/QueryPipeline/RemoteQueryExecutor.cpp
@@ -665,24 +665,24 @@ void RemoteQueryExecutor::sendExternalTables()
 
                 auto data = std::make_unique<ExternalTableData>();
                 data->table_name = table.first;
-                data->creating_pipe_callback = [cur, limits, context = this->context]()
+                data->creating_pipe_callback = [cur, limits, my_context = this->context]()
                 {
                     SelectQueryInfo query_info;
                     auto metadata_snapshot = cur->getInMemoryMetadataPtr();
-                    auto storage_snapshot = cur->getStorageSnapshot(metadata_snapshot, context);
+                    auto storage_snapshot = cur->getStorageSnapshot(metadata_snapshot, my_context);
                     QueryProcessingStage::Enum read_from_table_stage = cur->getQueryProcessingStage(
-                        context, QueryProcessingStage::Complete, storage_snapshot, query_info);
+                        my_context, QueryProcessingStage::Complete, storage_snapshot, query_info);
 
                     QueryPlan plan;
                     cur->read(
                         plan,
                         metadata_snapshot->getColumns().getNamesOfPhysical(),
-                        storage_snapshot, query_info, context,
+                        storage_snapshot, query_info, my_context,
                         read_from_table_stage, DEFAULT_BLOCK_SIZE, 1);
 
                     auto builder = plan.buildQueryPipeline(
-                        QueryPlanOptimizationSettings::fromContext(context),
-                        BuildQueryPipelineSettings::fromContext(context));
+                        QueryPlanOptimizationSettings::fromContext(my_context),
+                        BuildQueryPipelineSettings::fromContext(my_context));
 
                     builder->resize(1);
                     builder->addTransform(std::make_shared<LimitsCheckingTransform>(builder->getHeader(), limits));
diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp
index c280ff06c69..5a2bf0bad6c 100644
--- a/src/Server/HTTPHandler.cpp
+++ b/src/Server/HTTPHandler.cpp
@@ -800,11 +800,11 @@ void HTTPHandler::processQuery(
     if (settings.add_http_cors_header && !request.get("Origin", "").empty() && !config.has("http_options_response"))
         used_output.out->addHeaderCORS(true);
 
-    auto append_callback = [context = context] (ProgressCallback callback)
+    auto append_callback = [my_context = context] (ProgressCallback callback)
     {
-        auto prev = context->getProgressCallback();
+        auto prev = my_context->getProgressCallback();
 
-        context->setProgressCallback([prev, callback] (const Progress & progress)
+        my_context->setProgressCallback([prev, callback] (const Progress & progress)
         {
             if (prev)
                 prev(progress);
diff --git a/src/Server/HTTPHandlerFactory.h b/src/Server/HTTPHandlerFactory.h
index 76460dcc4a8..fe11833dc31 100644
--- a/src/Server/HTTPHandlerFactory.h
+++ b/src/Server/HTTPHandlerFactory.h
@@ -28,12 +28,12 @@ public:
     template <typename... TArgs>
     explicit HandlingRuleHTTPHandlerFactory(TArgs &&... args)
     {
-        creator = [args = std::tuple<TArgs...>(std::forward<TArgs>(args) ...)]()
+        creator = [my_args = std::tuple<TArgs...>(std::forward<TArgs>(args) ...)]()
         {
             return std::apply([&](auto && ... endpoint_args)
             {
                 return std::make_unique<TEndpoint>(std::forward<decltype(endpoint_args)>(endpoint_args)...);
-            }, std::move(args));
+            }, std::move(my_args));
         };
     }
 
diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp
index 96566a7d706..7318b0ad89b 100644
--- a/src/Server/MySQLHandler.cpp
+++ b/src/Server/MySQLHandler.cpp
@@ -340,10 +340,10 @@ void MySQLHandler::comQuery(ReadBuffer & payload)
 
         std::atomic<size_t> affected_rows {0};
         auto prev = query_context->getProgressCallback();
-        query_context->setProgressCallback([&, prev = prev](const Progress & progress)
+        query_context->setProgressCallback([&, my_prev = prev](const Progress & progress)
         {
-            if (prev)
-                prev(progress);
+            if (my_prev)
+                my_prev(progress);
 
             affected_rows += progress.written_rows;
         });
diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp
index c915213f4ac..19c0840149b 100644
--- a/src/Storages/HDFS/StorageHDFS.cpp
+++ b/src/Storages/HDFS/StorageHDFS.cpp
@@ -211,12 +211,12 @@ ColumnsDescription StorageHDFS::getTableStructureFromData(
     if (ctx->getSettingsRef().schema_inference_use_cache_for_hdfs)
         columns_from_cache = tryGetColumnsFromCache(paths, path_from_uri, last_mod_time, format, ctx);
 
-    ReadBufferIterator read_buffer_iterator = [&, uri_without_path = uri_without_path, it = paths.begin()](ColumnsDescription &) mutable -> std::unique_ptr<ReadBuffer>
+    ReadBufferIterator read_buffer_iterator = [&, my_uri_without_path = uri_without_path, it = paths.begin()](ColumnsDescription &) mutable -> std::unique_ptr<ReadBuffer>
     {
         if (it == paths.end())
             return nullptr;
         auto compression = chooseCompressionMethod(*it, compression_method);
-        auto impl = std::make_unique<ReadBufferFromHDFS>(uri_without_path, *it++, ctx->getGlobalContext()->getConfigRef(), ctx->getReadSettings());
+        auto impl = std::make_unique<ReadBufferFromHDFS>(my_uri_without_path, *it++, ctx->getGlobalContext()->getConfigRef(), ctx->getReadSettings());
         const Int64 zstd_window_log_max = ctx->getSettingsRef().zstd_window_log_max;
         return wrapReadBufferWithCompressionMethod(std::move(impl), compression, static_cast<int>(zstd_window_log_max));
     };
diff --git a/src/Storages/MergeTree/AsyncBlockIDsCache.cpp b/src/Storages/MergeTree/AsyncBlockIDsCache.cpp
index 7a8a4cd4347..23b6668c8d8 100644
--- a/src/Storages/MergeTree/AsyncBlockIDsCache.cpp
+++ b/src/Storages/MergeTree/AsyncBlockIDsCache.cpp
@@ -32,17 +32,17 @@ std::vector<String> AsyncBlockIDsCache::getChildren()
     auto zookeeper = storage.getZooKeeper();
 
     auto watch_callback = [last_time = this->last_updatetime.load()
-                           , update_min_interval = this->update_min_interval
-                           , task = task->shared_from_this()](const Coordination::WatchResponse &)
+                           , my_update_min_interval = this->update_min_interval
+                           , my_task = task->shared_from_this()](const Coordination::WatchResponse &)
     {
         auto now = std::chrono::steady_clock::now();
-        if (now - last_time < update_min_interval)
+        if (now - last_time < my_update_min_interval)
         {
-            std::chrono::milliseconds sleep_time = std::chrono::duration_cast<std::chrono::milliseconds>(update_min_interval - (now - last_time));
-            task->scheduleAfter(sleep_time.count());
+            std::chrono::milliseconds sleep_time = std::chrono::duration_cast<std::chrono::milliseconds>(my_update_min_interval - (now - last_time));
+            my_task->scheduleAfter(sleep_time.count());
         }
         else
-            task->schedule();
+            my_task->schedule();
     };
     std::vector<String> children;
     Coordination::Stat stat;
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index c8b1b70e356..47d2bc67ddd 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1955,10 +1955,10 @@ try
             outdated_unloaded_data_parts.pop_back();
         }
 
-        parts_futures.push_back(runner([&, part = part]()
+        parts_futures.push_back(runner([&, my_part = part]()
         {
             auto res = loadDataPartWithRetries(
-            part->info, part->name, part->disk,
+            my_part->info, my_part->name, part->disk,
             DataPartState::Outdated, data_parts_mutex, loading_parts_initial_backoff_ms,
             loading_parts_max_backoff_ms, loading_parts_max_tries);
 
@@ -5226,9 +5226,9 @@ void MergeTreeData::restorePartsFromBackup(RestorerFromBackup & restorer, const
             [storage = std::static_pointer_cast<MergeTreeData>(shared_from_this()),
              backup,
              part_path_in_backup = data_path_in_backup_fs / part_name,
-             part_info=*part_info,
+             my_part_info = *part_info,
              restored_parts_holder]
-            { storage->restorePartFromBackup(restored_parts_holder, part_info, part_path_in_backup); });
+            { storage->restorePartFromBackup(restored_parts_holder, my_part_info, part_path_in_backup); });
 
         ++num_parts;
     }
diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp
index 7428fd5c056..30f31910bee 100644
--- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp
+++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp
@@ -104,13 +104,13 @@ std::future<MergeTreeReaderPtr> MergeTreePrefetchedReadPool::createPrefetchedRea
     /// and we cannot block either, therefore make prefetch inside the pool and put the future
     /// into the read task (MergeTreeReadTask). When a thread calls getTask(), it will wait for
     /// it (if not yet ready) after getting the task.
-    auto task = [=, reader = std::move(reader), context = getContext()]() mutable -> MergeTreeReaderPtr &&
+    auto task = [=, my_reader = std::move(reader), context = getContext()]() mutable -> MergeTreeReaderPtr &&
     {
         /// For async read metrics in system.query_log.
         PrefetchIncrement watch(context->getAsyncReadCounters());
 
-        reader->prefetchBeginOfRange(priority);
-        return std::move(reader);
+        my_reader->prefetchBeginOfRange(priority);
+        return std::move(my_reader);
     };
     return scheduleFromThreadPool<IMergeTreeDataPart::MergeTreeReaderPtr>(std::move(task), prefetch_threadpool, "ReadPrepare", priority);
 }
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index 9f97835612b..1c39b53dcc2 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -989,8 +989,10 @@ std::vector<String> ReplicatedMergeTreeSinkImpl<async_insert>::commitPart(
             /// here lambda capture part name, it's ok since we'll not generate new one for this insert,
             /// see comments around 'part_committed_locally_but_zookeeper' flag
             retries_ctl.actionAfterLastFailedRetry(
-                [&storage = storage, part_name = part->name]()
-                { storage.enqueuePartForCheck(part_name, MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER); });
+                [&my_storage = storage, part_name = part->name]
+                {
+                    my_storage.enqueuePartForCheck(part_name, MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER);
+                });
 
             /// We do not know whether or not data has been inserted.
             retries_ctl.setUserError(
diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp
index b2091d7305f..86a2599b49c 100644
--- a/src/Storages/StorageDistributed.cpp
+++ b/src/Storages/StorageDistributed.cpp
@@ -1115,10 +1115,10 @@ void StorageDistributed::read(
             }
 
             additional_shard_filter_generator =
-                [&, custom_key_ast = std::move(custom_key_ast), shard_count = query_info.cluster->getShardCount()](uint64_t shard_num) -> ASTPtr
+                [&, my_custom_key_ast = std::move(custom_key_ast), shard_count = query_info.cluster->getShardCount()](uint64_t shard_num) -> ASTPtr
             {
                 return getCustomKeyFilterForParallelReplica(
-                    shard_count, shard_num - 1, custom_key_ast, settings.parallel_replicas_custom_key_filter_type, *this, local_context);
+                    shard_count, shard_num - 1, my_custom_key_ast, settings.parallel_replicas_custom_key_filter_type, *this, local_context);
             };
         }
     }
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 90ada1b708f..1dde057569e 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -9325,9 +9325,9 @@ void StorageReplicatedMergeTree::backupData(
     /// This task will be executed after all replicas have collected their parts and the coordination is ready to
     /// give us the final list of parts to add to the BackupEntriesCollector.
     auto post_collecting_task = [shared_id,
-                                 replica_name = getReplicaName(),
+                                 my_replica_name = getReplicaName(),
                                  coordination,
-                                 backup_entries = std::move(backup_entries),
+                                 my_backup_entries = std::move(backup_entries),
                                  &backup_entries_collector]()
     {
         Strings data_paths = coordination->getReplicatedDataPaths(shared_id);
@@ -9336,10 +9336,10 @@ void StorageReplicatedMergeTree::backupData(
         for (const auto & data_path : data_paths)
             data_paths_fs.push_back(data_path);
 
-        Strings part_names = coordination->getReplicatedPartNames(shared_id, replica_name);
+        Strings part_names = coordination->getReplicatedPartNames(shared_id, my_replica_name);
         std::unordered_set<std::string_view> part_names_set{part_names.begin(), part_names.end()};
 
-        for (const auto & [relative_path, backup_entry] : backup_entries)
+        for (const auto & [relative_path, backup_entry] : my_backup_entries)
         {
             size_t slash_pos = relative_path.find('/');
             String part_name = relative_path.substr(0, slash_pos);
@@ -9349,7 +9349,7 @@ void StorageReplicatedMergeTree::backupData(
                 backup_entries_collector.addBackupEntry(data_path / relative_path, backup_entry);
         }
 
-        auto mutation_infos = coordination->getReplicatedMutations(shared_id, replica_name);
+        auto mutation_infos = coordination->getReplicatedMutations(shared_id, my_replica_name);
         for (const auto & mutation_info : mutation_infos)
         {
             auto backup_entry = ReplicatedMergeTreeMutationEntry::parse(mutation_info.entry, mutation_info.id).backup();
diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp
index 92e04748149..1b87382c222 100644
--- a/src/Storages/System/StorageSystemReplicas.cpp
+++ b/src/Storages/System/StorageSystemReplicas.cpp
@@ -175,12 +175,12 @@ Pipe StorageSystemReplicas::read(
 
     for (size_t i = 0; i < tables_size; ++i)
     {
-        thread_pool.scheduleOrThrowOnError([&, i=i]
+        thread_pool.scheduleOrThrowOnError([&, my_i = i]
         {
             dynamic_cast<StorageReplicatedMergeTree &>(
             *replicated_tables
-                [(*col_database)[i].safeGet<const String &>()]
-                [(*col_table)[i].safeGet<const String &>()]).getStatus(statuses[i], with_zk_fields);
+                [(*col_database)[my_i].safeGet<const String &>()]
+                [(*col_table)[my_i].safeGet<const String &>()]).getStatus(statuses[my_i], with_zk_fields);
         });
     }
 

From a96067987ee85875bdc8766b12d36ccda4216412 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Fri, 12 May 2023 18:35:06 +0200
Subject: [PATCH 499/689] Mark joinGet() as non deterministic (so as dictGet)

joinGet() should not be considered as deterministic function, since
shards could have different data in tables.

Also since now there is allow_nondeterministic_mutations, it could be
used as a workaround for this backward incompatible change.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Functions/FunctionJoinGet.h                                 | 1 +
 ...01017_mutations_with_nondeterministic_functions_zookeeper.sh | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Functions/FunctionJoinGet.h b/src/Functions/FunctionJoinGet.h
index 2dd0cb9fdea..998a892f3df 100644
--- a/src/Functions/FunctionJoinGet.h
+++ b/src/Functions/FunctionJoinGet.h
@@ -88,6 +88,7 @@ public:
 
     explicit JoinGetOverloadResolver(ContextPtr context_) : WithContext(context_) {}
 
+    bool isDeterministic() const override { return false; }
     String getName() const override { return name; }
 
     FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override;
diff --git a/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh b/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh
index e4d091ea0bb..4f35b69da0b 100755
--- a/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh
+++ b/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh
@@ -60,7 +60,7 @@ ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $T1 UPDATE y = y + rand() % 1 WHERE no
 
 # hm... it looks like joinGet condidered determenistic
 ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $R1 UPDATE y = joinGet('${CLICKHOUSE_DATABASE}.lookup_table', 'y_new', y) WHERE x=1" 2>&1 \
-&& echo 'OK' || echo 'FAIL'
+| grep -F -q "must use only deterministic functions" && echo 'OK' || echo 'FAIL'
 
 ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $R1 DELETE WHERE dictHas('${CLICKHOUSE_DATABASE}.dict1', toUInt64(x))" 2>&1 \
 | grep -F -q "must use only deterministic functions" && echo 'OK' || echo 'FAIL'

From 87b5ab2fcda9b8754243a48f51027f11d2e2287d Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 13 May 2023 13:59:49 +0200
Subject: [PATCH 500/689] Enable few slow clang-tidy checks for clangd

In #47424 the readability-identifier-naming had been disabled for
clang-tidy builds, but the code base is already uses this notations, so
at least, let's enable it for clangd, so that developers who are using
editor/IDE with clangd will be warned.

FWIW clangd does not think that this check is slow [1], but I guess this
file hadn't been updated for quite a while.

  [1]: https://github.com/llvm/llvm-project/blob/main/clang-tools-extra/clangd/TidyFastChecks.inc

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .clangd    | 16 ++++++++++++++++
 .gitignore |  1 -
 2 files changed, 16 insertions(+), 1 deletion(-)
 create mode 100644 .clangd

diff --git a/.clangd b/.clangd
new file mode 100644
index 00000000000..ad471db8d8b
--- /dev/null
+++ b/.clangd
@@ -0,0 +1,16 @@
+Diagnostics:
+    # clangd does parse .clang-tidy, but some checks are too slow to run in
+    # clang-tidy build, so let's enable them explicitly for clangd at least.
+    ClangTidy:
+        # The following checks had been disabled due to slowliness with C++23,
+        # for more details see [1].
+        #
+        #   [1]: https://github.com/llvm/llvm-project/issues/61418
+        #
+        # But the code base had been written in a style that had been checked
+        # by this check, so at least, let's enable it for clangd.
+        Add: [
+            # configured in .clang-tidy
+            readability-identifier-naming,
+            bugprone-reserved-identifier,
+        ]
diff --git a/.gitignore b/.gitignore
index 14b860244c2..a04c60d5ca3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -129,7 +129,6 @@ website/package-lock.json
 /.ccls-cache
 
 # clangd cache
-/.clangd
 /.cache
 
 /compile_commands.json

From 2b793e3a14a850197ffbb23b76389b54f565586c Mon Sep 17 00:00:00 2001
From: darkkeks <darkkeks@rambler.ru>
Date: Sat, 13 May 2023 16:22:17 +0300
Subject: [PATCH 501/689] [docs] Remove "example" section from
 date-time-functions page toc

---
 .../functions/date-time-functions.md          | 22 ++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md
index 599b64ef43f..4710a5617c3 100644
--- a/docs/en/sql-reference/functions/date-time-functions.md
+++ b/docs/en/sql-reference/functions/date-time-functions.md
@@ -1218,12 +1218,16 @@ Rounds the time to the half hour.
 
 Converts a date or date with time to a UInt32 number containing the year and month number (YYYY \* 100 + MM). Accepts a second optional timezone argument. If provided, the timezone must be a string constant.
 
-### example
-```sql
+**Example**
+
+``` sql
 SELECT
     toYYYYMM(now(), 'US/Eastern')
 ```
-```response
+
+Result:
+
+``` text
 ┌─toYYYYMM(now(), 'US/Eastern')─┐
 │                        202303 │
 └───────────────────────────────┘
@@ -1233,11 +1237,15 @@ SELECT
 
 Converts a date or date with time to a UInt32 number containing the year and month number (YYYY \* 10000 + MM \* 100 + DD). Accepts a second optional timezone argument. If provided, the timezone must be a string constant.
 
-### example
+**Example**
+
 ```sql
 SELECT
     toYYYYMMDD(now(), 'US/Eastern')
 ```
+
+Result:
+
 ```response
 ┌─toYYYYMMDD(now(), 'US/Eastern')─┐
 │                        20230302 │
@@ -1248,11 +1256,15 @@ SELECT
 
 Converts a date or date with time to a UInt64 number containing the year and month number (YYYY \* 10000000000 + MM \* 100000000 + DD \* 1000000 + hh \* 10000 + mm \* 100 + ss). Accepts a second optional timezone argument. If provided, the timezone must be a string constant.
 
-### example
+**Example**
+
 ```sql
 SELECT
     toYYYYMMDDhhmmss(now(), 'US/Eastern')
 ```
+
+Result:
+
 ```response
 ┌─toYYYYMMDDhhmmss(now(), 'US/Eastern')─┐
 │                        20230302112209 │

From 0d9a6b5545c66a351183ac3ddd2663208e12fbb9 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 13 May 2023 13:26:47 +0000
Subject: [PATCH 502/689] Update QPL docs

---
 .../building_and_benchmarking_deflate_qpl.md  | 58 ++++++++++++++++++-
 .../sql-reference/statements/create/table.md  |  2 +-
 2 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/docs/en/development/building_and_benchmarking_deflate_qpl.md b/docs/en/development/building_and_benchmarking_deflate_qpl.md
index a492a30abae..0501c1cbdcb 100644
--- a/docs/en/development/building_and_benchmarking_deflate_qpl.md
+++ b/docs/en/development/building_and_benchmarking_deflate_qpl.md
@@ -4,16 +4,22 @@ sidebar_position: 73
 sidebar_label: Building and Benchmarking DEFLATE_QPL
 description: How to build Clickhouse and run benchmark with DEFLATE_QPL Codec
 ---
+
 # Build Clickhouse with DEFLATE_QPL
-- Make sure your target machine meet the QPL required [Prerequisites](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#prerequisites)
-- Pass the following flag to CMake when building ClickHouse, depending on the capabilities of your target machine:
+
+- Make sure your target machine meet the QPL required [prerequisites](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#prerequisites)
+- Pass the following flag to CMake when building ClickHouse:
+
 ``` bash
-cmake -DENABLE_SSE42=1 -DENABLE_QPL=1 ..
+cmake -DENABLE_QPL=1 ..
 ```
+
 - For generic requirements, please refer to Clickhouse generic [build instructions](/docs/en/development/build.md)
 
 # Run Benchmark with DEFLATE_QPL
+
 ## Files list
+
 The folders `benchmark_sample` under [qpl-cmake](https://github.com/ClickHouse/ClickHouse/tree/master/contrib/qpl-cmake) give example to run benchmark with python scripts:
 
 `client_scripts` contains python scripts for running typical benchmark, for example:
@@ -24,48 +30,60 @@ The folders `benchmark_sample` under [qpl-cmake](https://github.com/ClickHouse/C
 `database_files` means it will store database files according to lz4/deflate/zstd codec.
 
 ## Run benchmark automatically for Star Schema:
+
 ``` bash
 $ cd ./benchmark_sample/client_scripts
 $ sh run_ssb.sh
 ```
+
 After complete, please check all the results in this folder:`./output/`
 
 In case you run into failure, please manually run benchmark as below sections.
 
 ## Definition
+
 [CLICKHOUSE_EXE] means the path of clickhouse executable program.
 
 ## Environment
+
 - CPU: Sapphire Rapid
 - OS Requirements refer to [System Requirements for QPL](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#system-requirements)
 - IAA Setup refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration)
 - Install python modules:
+
 ``` bash
 pip3 install clickhouse_driver numpy
 ```
+
 [Self-check for IAA]
+
 ``` bash
 $ accel-config list | grep -P 'iax|state'
 ```
+
 Expected output like this:
 ``` bash
     "dev":"iax1",
     "state":"enabled",
             "state":"enabled",
 ```
+
 If you see nothing output, it means IAA is not ready to work. Please check IAA setup again.
 
 ## Generate raw data
+
 ``` bash
 $ cd ./benchmark_sample
 $ mkdir rawdata_dir && cd rawdata_dir
 ```
+
 Use [`dbgen`](https://clickhouse.com/docs/en/getting-started/example-datasets/star-schema) to generate 100 million rows data with the parameters:
 -s 20
 
 The files like `*.tbl` are expected to output under `./benchmark_sample/rawdata_dir/ssb-dbgen`:
 
 ## Database setup
+
 Set up database with LZ4 codec
 
 ``` bash
@@ -73,6 +91,7 @@ $ cd ./database_dir/lz4
 $ [CLICKHOUSE_EXE] server -C config_lz4.xml >&/dev/null&
 $ [CLICKHOUSE_EXE] client
 ```
+
 Here you should see the message `Connected to ClickHouse server` from console which means client successfully setup connection with server.
 
 Complete below three steps mentioned in [Star Schema Benchmark](https://clickhouse.com/docs/en/getting-started/example-datasets/star-schema)
@@ -110,6 +129,7 @@ You are expected to see below output:
 └───────────┘
 ```
 [Self-check for IAA Deflate codec]
+
 At the first time you execute insertion or query from client, clickhouse server console is expected to print this log:
 ```text
 Hardware-assisted DeflateQpl codec is ready!
@@ -121,17 +141,21 @@ Initialization of hardware-assisted DeflateQpl codec failed
 That means IAA devices is not ready, you need check IAA setup again.
 
 ## Benchmark with single instance 
+
 - Before start benchmark, Please disable C6 and set CPU frequency governor to be `performance`
+
 ``` bash
 $ cpupower idle-set -d 3
 $ cpupower frequency-set -g performance
 ```
+
 - To eliminate impact of memory bound on cross sockets, we use `numactl` to bind server on one socket and client on another socket.
 - Single instance means single server connected with single client
 
 Now run benchmark for LZ4/Deflate/ZSTD respectively:
 
 LZ4:
+
 ``` bash
 $ cd ./database_dir/lz4 
 $ numactl -m 0 -N 0 [CLICKHOUSE_EXE] server -C config_lz4.xml >&/dev/null&
@@ -140,13 +164,16 @@ $ numactl -m 1 -N 1 python3 client_stressing_test.py queries_ssb.sql 1 > lz4.log
 ```
 
 IAA deflate:
+
 ``` bash
 $ cd ./database_dir/deflate
 $ numactl -m 0 -N 0 [CLICKHOUSE_EXE] server -C config_deflate.xml >&/dev/null&
 $ cd ./client_scripts
 $ numactl -m 1 -N 1 python3 client_stressing_test.py queries_ssb.sql 1 > deflate.log
 ```
+
 ZSTD:
+
 ``` bash
 $ cd ./database_dir/zstd
 $ numactl -m 0 -N 0 [CLICKHOUSE_EXE] server -C config_zstd.xml >&/dev/null&
@@ -166,6 +193,7 @@ How to check performance metrics:
 We focus on QPS, please search the keyword: `QPS_Final` and collect statistics
 
 ## Benchmark with multi-instances
+
 - To reduce impact of memory bound on too much threads, We recommend run benchmark with multi-instances.
 - Multi-instance means multiple（2 or 4）servers connected with respective client.
 - The cores of one socket need to be divided equally and assigned to the servers respectively.
@@ -178,35 +206,46 @@ There are 2 differences:
 Here we assume there are 60 cores per socket and take 2 instances for example.
 Launch server for first instance
 LZ4:
+
 ``` bash
 $ cd ./database_dir/lz4
 $ numactl -C 0-29,120-149 [CLICKHOUSE_EXE] server -C config_lz4.xml >&/dev/null&
 ```
+
 ZSTD:
+
 ``` bash
 $ cd ./database_dir/zstd
 $ numactl -C 0-29,120-149 [CLICKHOUSE_EXE] server -C config_zstd.xml >&/dev/null&
 ```
+
 IAA Deflate:
+
 ``` bash
 $ cd ./database_dir/deflate
 $ numactl -C 0-29,120-149 [CLICKHOUSE_EXE] server -C config_deflate.xml >&/dev/null&
 ```
+
 [Launch server for second instance]
 
 LZ4:
+
 ``` bash
 $ cd ./database_dir && mkdir lz4_s2 && cd lz4_s2
 $ cp ../../server_config/config_lz4_s2.xml ./
 $ numactl -C 30-59,150-179 [CLICKHOUSE_EXE] server -C config_lz4_s2.xml >&/dev/null&
 ```
+
 ZSTD:
+
 ``` bash
 $ cd ./database_dir && mkdir zstd_s2 && cd zstd_s2
 $ cp ../../server_config/config_zstd_s2.xml ./
 $ numactl -C 30-59,150-179 [CLICKHOUSE_EXE] server -C config_zstd_s2.xml >&/dev/null&
 ```
+
 IAA Deflate:
+
 ``` bash
 $ cd ./database_dir && mkdir deflate_s2 && cd deflate_s2
 $ cp ../../server_config/config_deflate_s2.xml ./
@@ -216,19 +255,24 @@ $ numactl -C 30-59,150-179 [CLICKHOUSE_EXE] server -C config_deflate_s2.xml >&/d
 Creating tables && Inserting data for second instance
 
 Creating tables:
+
 ``` bash
 $ [CLICKHOUSE_EXE] client -m --port=9001 
 ```
+
 Inserting data:
+
 ``` bash
 $ [CLICKHOUSE_EXE] client --query "INSERT INTO [TBL_FILE_NAME] FORMAT CSV" < [TBL_FILE_NAME].tbl  --port=9001
 ```
+
 - [TBL_FILE_NAME] represents the name of a file named with the regular expression: *. tbl under `./benchmark_sample/rawdata_dir/ssb-dbgen`.
 - `--port=9001` stands for the assigned port for server instance which is also defined in config_lz4_s2.xml/config_zstd_s2.xml/config_deflate_s2.xml. For even more instances, you need replace it with the value: 9002/9003 which stand for s3/s4 instance respectively. If you don't assign it, the port is 9000 by default which has been used by first instance.
 
 Benchmarking with 2 instances
 
 LZ4:
+
 ``` bash
 $ cd ./database_dir/lz4
 $ numactl -C 0-29,120-149 [CLICKHOUSE_EXE] server -C config_lz4.xml >&/dev/null&
@@ -237,7 +281,9 @@ $ numactl -C 30-59,150-179 [CLICKHOUSE_EXE] server -C config_lz4_s2.xml >&/dev/n
 $ cd ./client_scripts
 $ numactl -m 1 -N 1 python3 client_stressing_test.py queries_ssb.sql 2  > lz4_2insts.log
 ```
+
 ZSTD:
+
 ``` bash
 $ cd ./database_dir/zstd
 $ numactl -C 0-29,120-149 [CLICKHOUSE_EXE] server -C config_zstd.xml >&/dev/null&
@@ -246,7 +292,9 @@ $ numactl -C 30-59,150-179 [CLICKHOUSE_EXE] server -C config_zstd_s2.xml >&/dev/
 $ cd ./client_scripts
 $ numactl -m 1 -N 1 python3 client_stressing_test.py queries_ssb.sql 2 > zstd_2insts.log
 ```
+
 IAA deflate
+
 ``` bash
 $ cd ./database_dir/deflate
 $ numactl -C 0-29,120-149 [CLICKHOUSE_EXE] server -C config_deflate.xml >&/dev/null&
@@ -255,9 +303,11 @@ $ numactl -C 30-59,150-179 [CLICKHOUSE_EXE] server -C config_deflate_s2.xml >&/d
 $ cd ./client_scripts
 $ numactl -m 1 -N 1 python3 client_stressing_test.py queries_ssb.sql 2 > deflate_2insts.log
 ```
+
 Here the last argument: `2` of client_stressing_test.py stands for the number of instances. For more instances, you need replace it with the value: 3 or 4. This script support up to 4 instances/
 
 Now three logs should be output as expected:
+
 ``` text
 lz4_2insts.log
 deflate_2insts.log
@@ -271,7 +321,9 @@ Benchmark setup for 4 instances is similar with 2 instances above.
 We recommend use 2 instances benchmark data as final report for review.
 
 ## Tips
+
 Each time before launch new clickhouse server, please make sure no background clickhouse process running, please check and kill old one:
+
 ``` bash
 $ ps -aux| grep clickhouse
 $ kill -9 [PID]
diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md
index 2011d12131c..de44a001472 100644
--- a/docs/en/sql-reference/statements/create/table.md
+++ b/docs/en/sql-reference/statements/create/table.md
@@ -381,7 +381,7 @@ High compression levels are useful for asymmetric scenarios, like compress once,
 `DEFLATE_QPL` — [Deflate compression algorithm](https://github.com/intel/qpl) implemented by Intel® Query Processing Library. Some limitations apply:
 
 - DEFLATE_QPL is experimental and can only be used after setting configuration parameter `allow_experimental_codecs=1`.
-- DEFLATE_QPL requires build ClickHouse with SSE 4.2 enabled. Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details.
+- DEFLATE_QPL requires a ClickHouse build compiled with SSE 4.2 instructions (by default, this is the case). Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details.
 - DEFLATE_QPL works best if the system has a Intel® IAA (In-Memory Analytics Accelerator) offloading device. Refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration) and [Benchmark with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Run-Benchmark-with-DEFLATE_QPL) for more details.
 - DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with SSE 4.2 enabled.
 

From 2ffd00df8a6a9fd36113a33710b7051863bb1783 Mon Sep 17 00:00:00 2001
From: flynn <fenglv15@mails.ucas.ac.cn>
Date: Sat, 13 May 2023 13:56:36 +0000
Subject: [PATCH 503/689] bitHammingDistance support String and FixedString
 data type

---
 src/Functions/FunctionBinaryArithmetic.h      | 313 +++++++++++++++---
 src/Functions/bitHammingDistance.cpp          |   2 +-
 .../01017_bithamming_distance.reference       |  15 +
 .../0_stateless/01017_bithamming_distance.sql |  24 ++
 4 files changed, 300 insertions(+), 54 deletions(-)

diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h
index a83bc6382b6..d583ee04373 100644
--- a/src/Functions/FunctionBinaryArithmetic.h
+++ b/src/Functions/FunctionBinaryArithmetic.h
@@ -116,6 +116,16 @@ template <> inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal128>, Da
 template <> inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal128>, DataTypeDecimal<Decimal64>> = true;
 template <> inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal64>, DataTypeDecimal<Decimal32>> = true;
 
+template <typename DataType>
+constexpr bool IsFixedString = false;
+template <>
+inline constexpr bool IsFixedString<DataTypeFixedString> = true;
+
+template <typename DataType>
+constexpr bool IsString = false;
+template <>
+inline constexpr bool IsString<DataTypeString> = true;
+
 template <template <typename, typename> class Operation, typename LeftDataType, typename RightDataType>
 struct BinaryOperationTraits
 {
@@ -144,6 +154,8 @@ public:
         Case<IsOperation<Operation>::allow_decimal && IsDataTypeDecimal<RightDataType> && IsFloatingPoint<LeftDataType>, DataTypeFloat64>,
 
         Case<IsOperation<Operation>::bit_hamming_distance && IsIntegral<LeftDataType> && IsIntegral<RightDataType>, DataTypeUInt8>,
+        Case<IsOperation<Operation>::bit_hamming_distance && IsFixedString<LeftDataType> && IsFixedString<RightDataType>, DataTypeUInt16>,
+        Case<IsOperation<Operation>::bit_hamming_distance && IsString<LeftDataType> && IsString<RightDataType>, DataTypeUInt64>,
 
         /// Decimal <op> Real is not supported (traditional DBs convert Decimal <op> Real to Real)
         Case<IsDataTypeDecimal<LeftDataType> && !IsIntegralOrExtendedOrDecimal<RightDataType>, InvalidType>,
@@ -246,9 +258,6 @@ private:
 template <typename B, typename Op>
 struct StringIntegerOperationImpl
 {
-    static const constexpr bool allow_fixed_string = false;
-    static const constexpr bool allow_string_integer = true;
-
     template <OpCase op_case>
     static void NO_INLINE processFixedString(const UInt8 * __restrict in_vec, const UInt64 n, const B * __restrict b, ColumnFixedString::Chars & out_vec, size_t size)
     {
@@ -381,6 +390,105 @@ private:
     }
 };
 
+template <typename Op>
+struct FixedStringReduceOperationImpl
+{
+    template <OpCase op_case>
+    static void inline process(const UInt8 * __restrict a, const UInt8 * __restrict b, UInt16 * __restrict result, size_t size, size_t N)
+    {
+        if constexpr (op_case == OpCase::Vector)
+            vectorVector(a, b, result, size, N);
+        else if constexpr (op_case == OpCase::LeftConstant)
+            vectorConstant(b, a, result, size, N);
+        else
+            vectorConstant(a, b, result, size, N);
+    }
+
+private:
+    static void vectorVector(const UInt8 * __restrict a, const UInt8 * __restrict b, UInt16 * __restrict result, size_t size, size_t N)
+    {
+        for (size_t i = 0; i < size; ++i)
+        {
+            size_t offset = i * N;
+            for (size_t j = 0; j < N; ++j)
+            {
+                result[i] += Op::template apply<UInt8>(a[offset + j], b[offset + j]);
+            }
+        }
+    }
+
+    static void vectorConstant(const UInt8 * __restrict a, const UInt8 * __restrict b, UInt16 * __restrict result, size_t size, size_t N)
+    {
+        for (size_t i = 0; i < size; ++i)
+        {
+            size_t offset = i * N;
+            for (size_t j = 0; j < N; ++j)
+            {
+                result[i] += Op::template apply<UInt8>(a[offset + j], b[j]);
+            }
+        }
+    }
+};
+
+template <typename Op>
+struct StringReduceOperationImpl
+{
+    static void vectorVector(
+        const ColumnString::Chars & a,
+        const ColumnString::Offsets & offsets_a,
+        const ColumnString::Chars & b,
+        const ColumnString::Offsets & offsets_b,
+        PaddedPODArray<UInt64> & res)
+    {
+        size_t size = res.size();
+        for (size_t i = 0; i < size; ++i)
+        {
+            res[i] = process(
+                a.data() + offsets_a[i - 1],
+                a.data() + offsets_a[i] - 1,
+                b.data() + offsets_b[i - 1],
+                b.data() + offsets_b[i] - 1);
+        }
+    }
+
+    static void
+    vectorConstant(const ColumnString::Chars & a, const ColumnString::Offsets & offsets_a, std::string_view b, PaddedPODArray<UInt64> & res)
+    {
+        size_t size = res.size();
+        for (size_t i = 0; i < size; ++i)
+        {
+            res[i] = process(
+                a.data() + offsets_a[i - 1],
+                a.data() + offsets_a[i] - 1,
+                reinterpret_cast<const UInt8 *>(b.data()),
+                reinterpret_cast<const UInt8 *>(b.data()) + b.size());
+        }
+    }
+
+    static inline UInt64 constConst(std::string_view a, std::string_view b)
+    {
+        return process(
+            reinterpret_cast<const UInt8 *>(a.data()),
+            reinterpret_cast<const UInt8 *>(a.data()) + a.size(),
+            reinterpret_cast<const UInt8 *>(b.data()),
+            reinterpret_cast<const UInt8 *>(b.data()) + b.size());
+    }
+
+private:
+    static UInt64 process(const UInt8 * __restrict start_a, const UInt8 * __restrict end_a, const UInt8 * start_b, const UInt8 * end_b)
+    {
+        UInt64 res = 0;
+        while (start_a < end_a && start_b < end_b)
+            res += Op::template apply<UInt8>(*start_a++, *start_b++);
+
+        while (start_a < end_a)
+            res += Op::template apply<UInt8>(*start_a++, 0);
+        while (start_b < end_b)
+            res += Op::template apply<UInt8>(0, *start_b++);
+        return res;
+    }
+};
+
 template <typename A, typename B, typename Op, typename ResultType = typename Op::ResultType>
 struct BinaryOperationImpl : BinaryOperation<A, B, Op, ResultType> { };
 
@@ -632,10 +740,11 @@ using namespace impl_;
 template <template <typename, typename> class Op, typename Name, bool valid_on_default_arguments = true, bool valid_on_float_arguments = true, bool division_by_nullable = false>
 class FunctionBinaryArithmetic : public IFunction
 {
-    static constexpr const bool is_plus = IsOperation<Op>::plus;
-    static constexpr const bool is_minus = IsOperation<Op>::minus;
-    static constexpr const bool is_multiply = IsOperation<Op>::multiply;
-    static constexpr const bool is_division = IsOperation<Op>::division;
+    static constexpr bool is_plus = IsOperation<Op>::plus;
+    static constexpr bool is_minus = IsOperation<Op>::minus;
+    static constexpr bool is_multiply = IsOperation<Op>::multiply;
+    static constexpr bool is_division = IsOperation<Op>::division;
+    static constexpr bool is_bit_hamming_distance = IsOperation<Op>::bit_hamming_distance;
 
     ContextPtr context;
     bool check_decimal_overflow = true;
@@ -780,8 +889,8 @@ class FunctionBinaryArithmetic : public IFunction
 
         if (tuple_data_type_0)
         {
-            auto & tuple_types = tuple_data_type_0->getElements();
-            for (auto & type : tuple_types)
+            const auto & tuple_types = tuple_data_type_0->getElements();
+            for (const auto & type : tuple_types)
                 if (!isInterval(type))
                     return {};
         }
@@ -1014,11 +1123,9 @@ class FunctionBinaryArithmetic : public IFunction
     ColumnPtr executeIntervalTupleOfIntervalsPlusMinus(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type,
                                                size_t input_rows_count, const FunctionOverloadResolverPtr & function_builder) const
     {
-        ColumnsWithTypeAndName new_arguments = arguments;
+        auto function = function_builder->build(arguments);
 
-        auto function = function_builder->build(new_arguments);
-
-        return function->execute(new_arguments, result_type, input_rows_count);
+        return function->execute(arguments, result_type, input_rows_count);
     }
 
     ColumnPtr executeTupleNumberOperator(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type,
@@ -1316,13 +1423,20 @@ public:
                     {
                         if (left.getN() == right.getN())
                         {
-                            type_res = std::make_shared<LeftDataType>(left.getN());
+                            if constexpr (is_bit_hamming_distance)
+                                type_res = std::make_shared<DataTypeUInt16>();
+                            else
+                                type_res = std::make_shared<LeftDataType>(left.getN());
                             return true;
                         }
                     }
                 }
 
-                if constexpr (!Op<LeftDataType, RightDataType>::allow_string_integer)
+                if constexpr (
+                    is_bit_hamming_distance
+                    && std::is_same_v<DataTypeString, LeftDataType> && std::is_same_v<DataTypeString, RightDataType>)
+                    type_res = std::make_shared<DataTypeUInt64>();
+                else if constexpr (!Op<LeftDataType, RightDataType>::allow_string_integer)
                     return false;
                 else if constexpr (!IsIntegral<RightDataType>)
                     return false;
@@ -1411,6 +1525,7 @@ public:
     ColumnPtr executeFixedString(const ColumnsWithTypeAndName & arguments) const
     {
         using OpImpl = FixedStringOperationImpl<Op<UInt8, UInt8>>;
+        using OpReduceImpl = FixedStringReduceOperationImpl<Op<UInt8, UInt8>>;
 
         const auto * const col_left_raw = arguments[0].column.get();
         const auto * const col_right_raw = arguments[1].column.get();
@@ -1425,18 +1540,30 @@ public:
                 if (col_left->getN() != col_right->getN())
                     return nullptr;
 
-                auto col_res = ColumnFixedString::create(col_left->getN());
-                auto & out_chars = col_res->getChars();
+                if constexpr (is_bit_hamming_distance)
+                {
+                    auto col_res = ColumnUInt16::create();
+                    auto & data = col_res->getData();
+                    data.resize_fill(col_left->size());
 
-                out_chars.resize(col_left->getN());
+                    OpReduceImpl::template process<OpCase::Vector>(
+                        col_left->getChars().data(), col_right->getChars().data(), data.data(), data.size(), col_left->getN());
 
-                OpImpl::template process<OpCase::Vector>(
-                    col_left->getChars().data(),
-                    col_right->getChars().data(),
-                    out_chars.data(),
-                    out_chars.size(), {});
+                    return ColumnConst::create(std::move(col_res), col_left_raw->size());
+                }
+                else
+                {
+                    auto col_res = ColumnFixedString::create(col_left->getN());
+                    auto & out_chars = col_res->getChars();
+
+                    out_chars.resize(col_left->getN());
+
+                    OpImpl::template process<OpCase::Vector>(
+                        col_left->getChars().data(), col_right->getChars().data(), out_chars.data(), out_chars.size(), {});
+
+                    return ColumnConst::create(std::move(col_res), col_left_raw->size());
+                }
 
-                return ColumnConst::create(std::move(col_res), col_left_raw->size());
             }
         }
 
@@ -1457,35 +1584,112 @@ public:
             if (col_left->getN() != col_right->getN())
                 return nullptr;
 
-            auto col_res = ColumnFixedString::create(col_left->getN());
-            auto & out_chars = col_res->getChars();
-            out_chars.resize((is_right_column_const ? col_left->size() : col_right->size()) * col_left->getN());
+            if constexpr (is_bit_hamming_distance)
+            {
+                auto col_res = ColumnUInt16::create();
+                auto & data = col_res->getData();
+                data.resize_fill(is_right_column_const ? col_left->size() : col_right->size());
 
-            if (!is_left_column_const && !is_right_column_const)
-            {
-                OpImpl::template process<OpCase::Vector>(
-                    col_left->getChars().data(),
-                    col_right->getChars().data(),
-                    out_chars.data(),
-                    out_chars.size(), {});
-            }
-            else if (is_left_column_const)
-            {
-                OpImpl::template process<OpCase::LeftConstant>(
-                    col_left->getChars().data(),
-                    col_right->getChars().data(),
-                    out_chars.data(),
-                    out_chars.size(),
-                    col_left->getN());
+                if (!is_left_column_const && !is_right_column_const)
+                {
+                    OpReduceImpl::template process<OpCase::Vector>(
+                        col_left->getChars().data(), col_right->getChars().data(), data.data(), data.size(), col_left->getN());
+                }
+                else if (is_left_column_const)
+                {
+                    OpReduceImpl::template process<OpCase::LeftConstant>(
+                        col_left->getChars().data(), col_right->getChars().data(), data.data(), data.size(), col_left->getN());
+                }
+                else
+                {
+                    OpReduceImpl::template process<OpCase::RightConstant>(
+                        col_left->getChars().data(), col_right->getChars().data(), data.data(), data.size(), col_left->getN());
+                }
+
+                return col_res;
             }
             else
             {
-                OpImpl::template process<OpCase::RightConstant>(
-                    col_left->getChars().data(),
-                    col_right->getChars().data(),
-                    out_chars.data(),
-                    out_chars.size(),
-                    col_left->getN());
+                auto col_res = ColumnFixedString::create(col_left->getN());
+                auto & out_chars = col_res->getChars();
+                out_chars.resize((is_right_column_const ? col_left->size() : col_right->size()) * col_left->getN());
+
+                if (!is_left_column_const && !is_right_column_const)
+                {
+                    OpImpl::template process<OpCase::Vector>(
+                        col_left->getChars().data(), col_right->getChars().data(), out_chars.data(), out_chars.size(), {});
+                }
+                else if (is_left_column_const)
+                {
+                    OpImpl::template process<OpCase::LeftConstant>(
+                        col_left->getChars().data(), col_right->getChars().data(), out_chars.data(), out_chars.size(), col_left->getN());
+                }
+                else
+                {
+                    OpImpl::template process<OpCase::RightConstant>(
+                        col_left->getChars().data(), col_right->getChars().data(), out_chars.data(), out_chars.size(), col_left->getN());
+                }
+
+                return col_res;
+            }
+        }
+        return nullptr;
+    }
+
+    /// Only used for bitHammingDistance
+    ColumnPtr executeString(const ColumnsWithTypeAndName & arguments) const
+    {
+        using OpImpl = StringReduceOperationImpl<Op<UInt8, UInt8>>;
+
+        const auto * const col_left_raw = arguments[0].column.get();
+        const auto * const col_right_raw = arguments[1].column.get();
+
+        if (const auto * col_left_const = checkAndGetColumnConst<ColumnString>(col_left_raw))
+        {
+            if (const auto * col_right_const = checkAndGetColumnConst<ColumnString>(col_right_raw))
+            {
+                const auto * col_left = checkAndGetColumn<ColumnString>(col_left_const->getDataColumn());
+                const auto * col_right = checkAndGetColumn<ColumnString>(col_right_const->getDataColumn());
+
+                std::string_view a = col_left->getDataAt(0).toView();
+                std::string_view b = col_right->getDataAt(0).toView();
+
+                auto res = OpImpl::constConst(a, b);
+
+                return DataTypeUInt64{}.createColumnConst(1, res);
+            }
+        }
+
+        const bool is_left_column_const = checkAndGetColumnConst<ColumnString>(col_left_raw) != nullptr;
+        const bool is_right_column_const = checkAndGetColumnConst<ColumnString>(col_right_raw) != nullptr;
+
+        const auto * col_left = is_left_column_const
+            ? checkAndGetColumn<ColumnString>(checkAndGetColumnConst<ColumnString>(col_left_raw)->getDataColumn())
+            : checkAndGetColumn<ColumnString>(col_left_raw);
+        const auto * col_right = is_right_column_const
+            ? checkAndGetColumn<ColumnString>(checkAndGetColumnConst<ColumnString>(col_right_raw)->getDataColumn())
+            : checkAndGetColumn<ColumnString>(col_right_raw);
+
+        if (col_left && col_right)
+        {
+            auto col_res = ColumnUInt64::create();
+            auto & data = col_res->getData();
+            data.resize(is_right_column_const ? col_left->size() : col_right->size());
+
+            if (!is_left_column_const && !is_right_column_const)
+            {
+                OpImpl::vectorVector(
+                    col_left->getChars(), col_left->getOffsets(), col_right->getChars(), col_right->getOffsets(), data);
+            }
+            else if (is_left_column_const)
+            {
+                std::string_view str_view = col_left->getDataAt(0).toView();
+                OpImpl::vectorConstant(col_right->getChars(), col_right->getOffsets(), str_view, data);
+            }
+            else 
+            {
+                std::string_view str_view = col_right->getDataAt(0).toView();
+                OpImpl::vectorConstant(col_left->getChars(), col_left->getOffsets(), str_view, data);
             }
 
             return col_res;
@@ -1493,9 +1697,8 @@ public:
         return nullptr;
     }
 
-
-    template <typename LeftColumnType, typename A, typename B>
-    ColumnPtr executeStringInteger(const ColumnsWithTypeAndName & arguments, const A & left, const B & right) const
+template <typename LeftColumnType, typename A, typename B>
+ColumnPtr executeStringInteger(const ColumnsWithTypeAndName & arguments, const A & left, const B & right) const
     {
         using LeftDataType = std::decay_t<decltype(left)>;
         using RightDataType = std::decay_t<decltype(right)>;
@@ -1812,7 +2015,11 @@ public:
                         return (res = executeFixedString(arguments)) != nullptr;
                 }
 
-                if constexpr (!Op<LeftDataType, RightDataType>::allow_string_integer)
+                if constexpr (
+                    is_bit_hamming_distance
+                    && std::is_same_v<DataTypeString, LeftDataType> && std::is_same_v<DataTypeString, RightDataType>)
+                    return (res = executeString(arguments)) != nullptr;
+                else if constexpr (!Op<LeftDataType, RightDataType>::allow_string_integer)
                     return false;
                 else if constexpr (!IsIntegral<RightDataType>)
                     return false;
diff --git a/src/Functions/bitHammingDistance.cpp b/src/Functions/bitHammingDistance.cpp
index 040f45eaedd..2eaa397dd04 100644
--- a/src/Functions/bitHammingDistance.cpp
+++ b/src/Functions/bitHammingDistance.cpp
@@ -8,7 +8,7 @@ template <typename A, typename B>
 struct BitHammingDistanceImpl
 {
     using ResultType = UInt8;
-    static constexpr bool allow_fixed_string = false;
+    static constexpr bool allow_fixed_string = true;
     static constexpr bool allow_string_integer = false;
 
     template <typename Result = ResultType>
diff --git a/tests/queries/0_stateless/01017_bithamming_distance.reference b/tests/queries/0_stateless/01017_bithamming_distance.reference
index cc2d4f39154..3e82e0d2864 100644
--- a/tests/queries/0_stateless/01017_bithamming_distance.reference
+++ b/tests/queries/0_stateless/01017_bithamming_distance.reference
@@ -13,3 +13,18 @@
 5
 9
 9
+0
+6
+6
+0
+6
+47
+0
+6
+47
+0
+6
+26
+0
+6
+26
diff --git a/tests/queries/0_stateless/01017_bithamming_distance.sql b/tests/queries/0_stateless/01017_bithamming_distance.sql
index 4b36894b97c..11f4f27d9a8 100644
--- a/tests/queries/0_stateless/01017_bithamming_distance.sql
+++ b/tests/queries/0_stateless/01017_bithamming_distance.sql
@@ -18,3 +18,27 @@ SELECT bitHammingDistance(n2, 100) FROM defaults;
 SELECT bitHammingDistance(n3, n4) FROM defaults;
 
 DROP TABLE defaults;
+
+DROP TABLE IF EXISTS test_string;
+
+CREATE TABLE test_string
+(
+	s1 String,
+	s2 String,
+	s3 FixedString(10),
+	s4 FixedString(10),
+) ENGINE = Memory;
+
+INSERT INTO test_string VALUES ('hello', 'hello', 'hello', 'hello') ('hello', 'hellow', 'hello', 'hellow') ('clickhouse', '012345', 'clickhouse', '012345');
+
+SELECT bitHammingDistance('hello', 'hello');
+SELECT bitHammingDistance('hello', 'hellow');
+SELECT bitHammingDistance(toFixedString('hello', 6), toFixedString('hellow', 6));
+
+SELECT bitHammingDistance(s1, s2) FROM test_string;
+SELECT bitHammingDistance(s3, s4) FROM test_string;
+
+SELECT bitHammingDistance('hello', s2) FROM test_string;
+SELECT bitHammingDistance(s4, toFixedString('hello', 10)) FROM test_string;
+
+DROP TABLE test_string;

From 2f88605c3d24f08f42fe7e1839391b6b25269028 Mon Sep 17 00:00:00 2001
From: flynn <fenglv15@mails.ucas.ac.cn>
Date: Sat, 13 May 2023 14:00:55 +0000
Subject: [PATCH 504/689] remove space

format
---
 src/Functions/FunctionBinaryArithmetic.h | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h
index d583ee04373..7e26602eee4 100644
--- a/src/Functions/FunctionBinaryArithmetic.h
+++ b/src/Functions/FunctionBinaryArithmetic.h
@@ -116,15 +116,11 @@ template <> inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal128>, Da
 template <> inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal128>, DataTypeDecimal<Decimal64>> = true;
 template <> inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal64>, DataTypeDecimal<Decimal32>> = true;
 
-template <typename DataType>
-constexpr bool IsFixedString = false;
-template <>
-inline constexpr bool IsFixedString<DataTypeFixedString> = true;
+template <typename DataType> constexpr bool IsFixedString = false;
+template <> inline constexpr bool IsFixedString<DataTypeFixedString> = true;
 
-template <typename DataType>
-constexpr bool IsString = false;
-template <>
-inline constexpr bool IsString<DataTypeString> = true;
+template <typename DataType> constexpr bool IsString = false;
+template <> inline constexpr bool IsString<DataTypeString> = true;
 
 template <template <typename, typename> class Operation, typename LeftDataType, typename RightDataType>
 struct BinaryOperationTraits
@@ -1686,7 +1682,7 @@ public:
                 std::string_view str_view = col_left->getDataAt(0).toView();
                 OpImpl::vectorConstant(col_right->getChars(), col_right->getOffsets(), str_view, data);
             }
-            else 
+            else
             {
                 std::string_view str_view = col_right->getDataAt(0).toView();
                 OpImpl::vectorConstant(col_left->getChars(), col_left->getOffsets(), str_view, data);

From 7e0bf3b8b22e7d5a3c1e7715c891a67cd45633a8 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 13 May 2023 15:29:45 +0000
Subject: [PATCH 505/689] Small-ish .clang-tidy update

---
 .clang-tidy | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.clang-tidy b/.clang-tidy
index 2c5f8c918ba..7bf48321260 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -23,7 +23,7 @@ Checks: '*,
     -bugprone-implicit-widening-of-multiplication-result,
     -bugprone-narrowing-conversions,
     -bugprone-not-null-terminated-result,
-    -bugprone-reserved-identifier,
+    -bugprone-reserved-identifier, # useful but too slow, TODO retry when https://reviews.llvm.org/rG1c282052624f9d0bd273bde0b47b30c96699c6c7 is merged
     -bugprone-unchecked-optional-access,
 
     -cert-dcl16-c,
@@ -137,7 +137,7 @@ Checks: '*,
     -readability-function-cognitive-complexity,
     -readability-function-size,
     -readability-identifier-length,
-    -readability-identifier-naming,
+    -readability-identifier-naming, # useful but too slow
     -readability-implicit-bool-conversion,
     -readability-isolate-declaration,
     -readability-magic-numbers,

From f775a466f75950c41c522df9e686e89fe4e30fb8 Mon Sep 17 00:00:00 2001
From: flynn <fenglv15@mails.ucas.ac.cn>
Date: Sat, 13 May 2023 16:12:51 +0000
Subject: [PATCH 506/689] update doc

---
 docs/en/sql-reference/data-types/array.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/docs/en/sql-reference/data-types/array.md b/docs/en/sql-reference/data-types/array.md
index 707acbda760..20ce7d2ed52 100644
--- a/docs/en/sql-reference/data-types/array.md
+++ b/docs/en/sql-reference/data-types/array.md
@@ -46,8 +46,6 @@ SELECT [1, 2] AS x, toTypeName(x)
 
 ## Working with Data Types
 
-The maximum size of an array is limited to one million elements.
-
 When creating an array on the fly, ClickHouse automatically defines the argument type as the narrowest data type that can store all the listed arguments. If there are any [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable) or literal [NULL](../../sql-reference/syntax.md#null-literal) values, the type of an array element also becomes [Nullable](../../sql-reference/data-types/nullable.md).
 
 If ClickHouse couldn’t determine the data type, it generates an exception. For instance, this happens when trying to create an array with strings and numbers simultaneously (`SELECT array(1, 'a')`).

From 14d831e3bf2099214a4dcccab258c8f2383e3d15 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 13 May 2023 20:18:24 +0000
Subject: [PATCH 507/689] zirkon --> zircon

---
 .clang-tidy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.clang-tidy b/.clang-tidy
index 7bf48321260..1c1e5373027 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -149,7 +149,7 @@ Checks: '*,
     -readability-uppercase-literal-suffix,
     -readability-use-anyofallof,
 
-    -zirkon-*,
+    -zircon-*,
 '
 
 WarningsAsErrors: '*'

From 804276ffe62735b827ac9a82e881112839c83091 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 13 May 2023 20:26:43 +0000
Subject: [PATCH 508/689] readability-identifier-names: adjust invalid options

The check is disabled, anyways ...
- remove UsingCase (it is probably subsumed by TypedefCase)
- rename TemplateCase to TemplateParameterCase
---
 .clang-tidy | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.clang-tidy b/.clang-tidy
index 1c1e5373027..85989d311a2 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -169,11 +169,10 @@ CheckOptions:
   readability-identifier-naming.ParameterPackCase: lower_case
   readability-identifier-naming.StructCase: CamelCase
   readability-identifier-naming.TemplateTemplateParameterCase: CamelCase
-  readability-identifier-naming.TemplateUsingCase: lower_case
+  readability-identifier-naming.TemplateParameterCase: lower_case
   readability-identifier-naming.TypeTemplateParameterCase: CamelCase
   readability-identifier-naming.TypedefCase: CamelCase
   readability-identifier-naming.UnionCase: CamelCase
-  readability-identifier-naming.UsingCase: CamelCase
   modernize-loop-convert.UseCxx20ReverseRanges: false
   performance-move-const-arg.CheckTriviallyCopyableMove: false
   # Workaround clang-tidy bug: https://github.com/llvm/llvm-project/issues/46097

From f0d21a910078c12fe3a4a038ef283a5ceb6dc219 Mon Sep 17 00:00:00 2001
From: Ilya Golshtein <igolshtein@altinity.com>
Date: Fri, 3 Mar 2023 11:00:12 +0100
Subject: [PATCH 509/689] row policy template: initial, works, restrictive
 rules fix

---
 src/Access/AccessEntityIO.cpp                 |   4 +
 src/Access/EnabledRowPolicies.cpp             |  36 ++++-
 src/Access/EnabledRowPolicies.h               |   8 ++
 src/Access/RowPolicy.h                        |   1 +
 src/Access/RowPolicyCache.cpp                 | 128 ++++++++++++++----
 src/Access/RowPolicyCache.h                   |   7 +-
 .../Access/ParserCreateRowPolicyQuery.cpp     |   1 +
 src/Parsers/Access/ParserRowPolicyName.cpp    |  22 ++-
 src/Parsers/parseQuery.cpp                    |   2 +
 9 files changed, 183 insertions(+), 26 deletions(-)

diff --git a/src/Access/AccessEntityIO.cpp b/src/Access/AccessEntityIO.cpp
index 80bb63b04bf..008ba5e5dfe 100644
--- a/src/Access/AccessEntityIO.cpp
+++ b/src/Access/AccessEntityIO.cpp
@@ -28,6 +28,9 @@
 #include <boost/range/algorithm/copy.hpp>
 #include <boost/range/algorithm_ext/push_back.hpp>
 
+#include <Common/logger_useful.h>
+
+
 namespace DB
 {
 namespace ErrorCodes
@@ -62,6 +65,7 @@ AccessEntityPtr deserializeAccessEntityImpl(const String & definition)
     const char * end = begin + definition.size();
     while (pos < end)
     {
+        LOG_TRACE((&Poco::Logger::get("deserializeAccessEntityImpl")), "{}", std::string(pos, end));
         queries.emplace_back(parseQueryAndMovePosition(parser, pos, end, "", true, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH));
         while (isWhitespaceASCII(*pos) || *pos == ';')
             ++pos;
diff --git a/src/Access/EnabledRowPolicies.cpp b/src/Access/EnabledRowPolicies.cpp
index c00dcf9e3a7..225f211bdd4 100644
--- a/src/Access/EnabledRowPolicies.cpp
+++ b/src/Access/EnabledRowPolicies.cpp
@@ -3,6 +3,8 @@
 #include <boost/range/adaptor/map.hpp>
 #include <boost/range/algorithm/copy.hpp>
 
+#include <Common/logger_useful.h>
+
 
 namespace DB
 {
@@ -18,6 +20,12 @@ size_t EnabledRowPolicies::Hash::operator()(const MixedFiltersKey & key) const
     return std::hash<std::string_view>{}(key.database) - std::hash<std::string_view>{}(key.table_name) + static_cast<size_t>(key.filter_type);
 }
 
+
+// size_t EnabledRowPolicies::Hash::operator()(const MixedFiltersKey & key) const
+// {
+//     return std::hash<std::string_view>{}(key.database) + static_cast<size_t>(key.filter_type);
+// }
+
 EnabledRowPolicies::EnabledRowPolicies() : params()
 {
 }
@@ -32,11 +40,37 @@ EnabledRowPolicies::~EnabledRowPolicies() = default;
 RowPolicyFilterPtr EnabledRowPolicies::getFilter(const String & database, const String & table_name, RowPolicyFilterType filter_type) const
 {
     /// We don't lock `mutex` here.
+
     auto loaded = mixed_filters.load();
+    {
+
+
+        for (auto it = loaded->begin(); it != loaded->end(); ++it)
+        {
+            LOG_TRACE((&Poco::Logger::get("EnabledRowPolicies::getFilter")), "  db: {}, table {}", it->first.database, it->first.table_name);
+
+        }
+
+    }
+
+
+
+
     auto it = loaded->find({database, table_name, filter_type});
     if (it == loaded->end())
-        return {};
+    {
+        it = loaded->find({database, "*", filter_type});
+        if (it == loaded->end())
+        {
+            LOG_TRACE((&Poco::Logger::get("EnabledRowPolicies::getFilter")), "db: {}, table {} - not found ({} records)",
+                database, table_name, loaded->size());
+            return {};
+        }
+    }
 
+
+    LOG_TRACE((&Poco::Logger::get("EnabledRowPolicies::getFilter")), "db: {}, table {} - found ({} records)",
+        database, table_name, loaded->size());
     return it->second;
 }
 
diff --git a/src/Access/EnabledRowPolicies.h b/src/Access/EnabledRowPolicies.h
index b8e6b2e0549..e09d32264f0 100644
--- a/src/Access/EnabledRowPolicies.h
+++ b/src/Access/EnabledRowPolicies.h
@@ -72,6 +72,14 @@ private:
         auto toTuple() const { return std::tie(database, table_name, filter_type); }
         friend bool operator==(const MixedFiltersKey & left, const MixedFiltersKey & right) { return left.toTuple() == right.toTuple(); }
         friend bool operator!=(const MixedFiltersKey & left, const MixedFiltersKey & right) { return left.toTuple() != right.toTuple(); }
+        // friend bool operator==(const MixedFiltersKey & left, const MixedFiltersKey & right)
+        // {
+        //     return left.database == right.database && left.filter_type == right.filter_type;
+        // }
+        // friend bool operator!=(const MixedFiltersKey & left, const MixedFiltersKey & right)
+        // {
+        //     return left.database != right.database || left.filter_type != right.filter_type;
+        // }
     };
 
     struct Hash
diff --git a/src/Access/RowPolicy.h b/src/Access/RowPolicy.h
index 99e6f1992f5..31ee876b47b 100644
--- a/src/Access/RowPolicy.h
+++ b/src/Access/RowPolicy.h
@@ -34,6 +34,7 @@ struct RowPolicy : public IAccessEntity
     /// in addition to all the restrictive policies.
     void setPermissive(bool permissive_ = true) { setRestrictive(!permissive_); }
     bool isPermissive() const { return !isRestrictive(); }
+    bool isDatabase() const { return full_name.table_name == "*"; }
 
     /// Sets that the policy is restrictive.
     /// A row is only accessible if at least one of the permissive policies passes,
diff --git a/src/Access/RowPolicyCache.cpp b/src/Access/RowPolicyCache.cpp
index 1036df92609..81e5acdf3ce 100644
--- a/src/Access/RowPolicyCache.cpp
+++ b/src/Access/RowPolicyCache.cpp
@@ -11,6 +11,8 @@
 #include <boost/smart_ptr/make_shared.hpp>
 #include <Core/Defines.h>
 
+#include <Common/logger_useful.h>
+
 
 namespace DB
 {
@@ -148,9 +150,19 @@ void RowPolicyCache::ensureAllRowPoliciesRead()
 
     for (const UUID & id : access_control.findAll<RowPolicy>())
     {
-        auto quota = access_control.tryRead<RowPolicy>(id);
-        if (quota)
-            all_policies.emplace(id, PolicyInfo(quota));
+        auto policy = access_control.tryRead<RowPolicy>(id);
+        if (policy)
+        {
+            PolicyInfo policy_info(policy);
+            if (policy_info.database_and_table_name->second == "*")
+            {
+                database_policies.emplace(id, std::move(policy_info));
+            }
+            else
+            {
+                table_policies.emplace(id, std::move(policy_info));
+            }
+        }
     }
 }
 
@@ -158,15 +170,23 @@ void RowPolicyCache::ensureAllRowPoliciesRead()
 void RowPolicyCache::rowPolicyAddedOrChanged(const UUID & policy_id, const RowPolicyPtr & new_policy)
 {
     std::lock_guard lock{mutex};
-    auto it = all_policies.find(policy_id);
-    if (it == all_policies.end())
+    bool found = true;
+
+    auto it = table_policies.find(policy_id);
+    if (it == table_policies.end())
     {
-        it = all_policies.emplace(policy_id, PolicyInfo(new_policy)).first;
+        it = database_policies.find(policy_id);
+        if (it == database_policies.end())
+        {
+            PolicyMap & policy_map = new_policy->isDatabase() ? database_policies : table_policies;
+            it = policy_map.emplace(policy_id, PolicyInfo(new_policy)).first;
+            found = false;
+        }
     }
-    else
+
+    if (found && it->second.policy == new_policy)
     {
-        if (it->second.policy == new_policy)
-            return;
+        return;
     }
 
     auto & info = it->second;
@@ -178,7 +198,15 @@ void RowPolicyCache::rowPolicyAddedOrChanged(const UUID & policy_id, const RowPo
 void RowPolicyCache::rowPolicyRemoved(const UUID & policy_id)
 {
     std::lock_guard lock{mutex};
-    all_policies.erase(policy_id);
+    auto it = database_policies.find(policy_id);
+    if (it != database_policies.end())
+    {
+        database_policies.erase(it);
+    }
+    else
+    {
+        table_policies.erase(policy_id);
+    }
     mixFilters();
 }
 
@@ -215,22 +243,71 @@ void RowPolicyCache::mixFiltersFor(EnabledRowPolicies & enabled)
         std::vector<RowPolicyPtr> policies;
     };
 
-    std::unordered_map<MixedFiltersKey, MixerWithNames, Hash> mixers;
+    std::unordered_map<MixedFiltersKey, MixerWithNames, Hash> table_mixers;
+    std::unordered_map<MixedFiltersKey, MixerWithNames, Hash> database_mixers;
 
-    for (const auto & [policy_id, info] : all_policies)
+
+    for (const auto & [policy_id, info] : database_policies)
     {
         const auto & policy = *info.policy;
         bool match = info.roles->match(enabled.params.user_id, enabled.params.enabled_roles);
-        MixedFiltersKey key;
-        key.database = info.database_and_table_name->first;
-        key.table_name = info.database_and_table_name->second;
         for (auto filter_type : collections::range(0, RowPolicyFilterType::MAX))
         {
             auto filter_type_i = static_cast<size_t>(filter_type);
             if (info.parsed_filters[filter_type_i])
             {
-                key.filter_type = filter_type;
-                auto & mixer = mixers[key];
+                MixedFiltersKey key{info.database_and_table_name->first,
+                    info.database_and_table_name->second,
+                    filter_type};
+                LOG_TRACE((&Poco::Logger::get("mixFiltersFor")), "db: {} : {}", key.database, key.table_name);
+
+                auto & mixer = database_mixers[key];  //  getting database level mixer
+                mixer.database_and_table_name = info.database_and_table_name;
+                if (match)
+                {
+                    mixer.mixer.add(info.parsed_filters[filter_type_i], policy.isRestrictive());
+                    mixer.policies.push_back(info.policy);
+                }
+            }
+        }
+    }
+
+
+    for (const auto & [policy_id, info] : table_policies)
+    {
+        const auto & policy = *info.policy;
+        bool match = info.roles->match(enabled.params.user_id, enabled.params.enabled_roles);
+        for (auto filter_type : collections::range(0, RowPolicyFilterType::MAX))
+        {
+            auto filter_type_i = static_cast<size_t>(filter_type);
+            if (info.parsed_filters[filter_type_i])
+            {
+                MixedFiltersKey key{info.database_and_table_name->first,
+                    info.database_and_table_name->second,
+                    filter_type};
+                LOG_TRACE((&Poco::Logger::get("mixFiltersFor")), "table: {} : {}", key.database, key.table_name);
+                auto table_it = table_mixers.find(key);
+                if (table_it == table_mixers.end())
+                {
+                    LOG_TRACE((&Poco::Logger::get("mixFiltersFor")), "table: not found, looking for db");
+                    MixedFiltersKey database_key = key;
+                    database_key.table_name = "*";
+
+                    auto database_it = database_mixers.find(database_key);
+
+                    if (database_it == database_mixers.end())
+                    {
+                        LOG_TRACE((&Poco::Logger::get("mixFiltersFor")), "table: not found, database not found");
+                        table_it = table_mixers.try_emplace(key).first;
+                    }
+                    else
+                    {
+                        LOG_TRACE((&Poco::Logger::get("mixFiltersFor")), "table: not found, database found");
+                        table_it = table_mixers.insert({key, database_it->second}).first;
+                    }
+                }
+
+                auto & mixer = table_it->second; // table_mixers[key];    getting table level mixer
                 mixer.database_and_table_name = info.database_and_table_name;
                 if (match)
                 {
@@ -242,15 +319,20 @@ void RowPolicyCache::mixFiltersFor(EnabledRowPolicies & enabled)
     }
 
     auto mixed_filters = boost::make_shared<MixedFiltersMap>();
-    for (auto & [key, mixer] : mixers)
+
+    for (auto mixer_map_ptr :  { &table_mixers, &database_mixers})
     {
-        auto mixed_filter = std::make_shared<RowPolicyFilter>();
-        mixed_filter->database_and_table_name = std::move(mixer.database_and_table_name);
-        mixed_filter->expression = std::move(mixer.mixer).getResult(access_control.isEnabledUsersWithoutRowPoliciesCanReadRows());
-        mixed_filter->policies = std::move(mixer.policies);
-        mixed_filters->emplace(key, std::move(mixed_filter));
+        for (auto & [key, mixer] : *mixer_map_ptr)
+        {
+            auto mixed_filter = std::make_shared<RowPolicyFilter>();
+            mixed_filter->database_and_table_name = std::move(mixer.database_and_table_name);
+            mixed_filter->expression = std::move(mixer.mixer).getResult(access_control.isEnabledUsersWithoutRowPoliciesCanReadRows());
+            mixed_filter->policies = std::move(mixer.policies);
+            mixed_filters->emplace(key, std::move(mixed_filter));
+        }
     }
 
+
     enabled.mixed_filters.store(mixed_filters);
 }
 
diff --git a/src/Access/RowPolicyCache.h b/src/Access/RowPolicyCache.h
index 4fbf90d1a2d..49e6a2e5df3 100644
--- a/src/Access/RowPolicyCache.h
+++ b/src/Access/RowPolicyCache.h
@@ -12,6 +12,7 @@ namespace DB
 class AccessControl;
 struct RolesOrUsersSet;
 struct RowPolicy;
+
 using RowPolicyPtr = std::shared_ptr<const RowPolicy>;
 
 /// Stores read and parsed row policies.
@@ -35,14 +36,18 @@ private:
         ASTPtr parsed_filters[static_cast<size_t>(RowPolicyFilterType::MAX)];
     };
 
+    using PolicyMap = std::unordered_map<UUID, PolicyInfo>;
+
     void ensureAllRowPoliciesRead();
     void rowPolicyAddedOrChanged(const UUID & policy_id, const RowPolicyPtr & new_policy);
     void rowPolicyRemoved(const UUID & policy_id);
     void mixFilters();
     void mixFiltersFor(EnabledRowPolicies & enabled);
 
+
     const AccessControl & access_control;
-    std::unordered_map<UUID, PolicyInfo> all_policies;
+    PolicyMap database_policies;
+    PolicyMap table_policies;
     bool all_policies_read = false;
     scope_guard subscription;
     std::map<EnabledRowPolicies::Params, std::weak_ptr<EnabledRowPolicies>> enabled_row_policies;
diff --git a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp
index 2c25fc14e7d..03d0754ca1a 100644
--- a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp
+++ b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp
@@ -203,6 +203,7 @@ namespace
 
 bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
+    // poco_assert("ParserCreateRowPolicyQuery::parseImpl" == nullptr);
     bool alter = false;
     if (attach_mode)
     {
diff --git a/src/Parsers/Access/ParserRowPolicyName.cpp b/src/Parsers/Access/ParserRowPolicyName.cpp
index cf5d2ab21b6..e087c45fad9 100644
--- a/src/Parsers/Access/ParserRowPolicyName.cpp
+++ b/src/Parsers/Access/ParserRowPolicyName.cpp
@@ -7,6 +7,8 @@
 #include <Parsers/parseDatabaseAndTableName.h>
 #include <base/insertAtEnd.h>
 
+#include <Common/logger_useful.h>
+
 
 namespace DB
 {
@@ -26,8 +28,19 @@ namespace
         return IParserBase::wrapParseImpl(pos, [&]
         {
             String res_database, res_table_name;
-            if (!parseDatabaseAndTableName(pos, expected, res_database, res_table_name))
+            // if (!parseDatabaseAndTableName(pos, expected, res_database, res_table_name))
+            bool any_database = false;
+            bool any_table = true;
+
+            if (!parseDatabaseAndTableNameOrAsterisks(pos, expected, res_database, any_database, res_table_name, any_table))
+            {
+                // poco_assert("parseDatabaseAndTableName failed" == nullptr);
+                LOG_TRACE((&Poco::Logger::get("ParserRowPolicyName")), "parseDatabaseAndTableName failed");
                 return false;
+            }
+            if (any_table)
+                res_table_name = "*";
+
 
             /// If table is specified without DB it cannot be followed by "ON"
             /// (but can be followed by "ON CLUSTER").
@@ -51,8 +64,10 @@ namespace
     }
 
 
+
     bool parseOnDBAndTableName(IParser::Pos & pos, Expected & expected, String & database, String & table_name)
     {
+        // poco_assert("parseOnDBAndTableNames" == nullptr);
         return IParserBase::wrapParseImpl(pos, [&]
         {
             return ParserKeyword{"ON"}.ignore(pos, expected) && parseDBAndTableName(pos, expected, database, table_name);
@@ -62,6 +77,9 @@ namespace
 
     bool parseOnDBAndTableNames(IParser::Pos & pos, Expected & expected, std::vector<std::pair<String, String>> & database_and_table_names)
     {
+        // poco_assert("parseOnDBAndTableNames" == nullptr);
+
+
         return IParserBase::wrapParseImpl(pos, [&]
         {
             if (!ParserKeyword{"ON"}.ignore(pos, expected))
@@ -146,6 +164,7 @@ namespace
 
 bool ParserRowPolicyName::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
+    // poco_assert("ParserRowPolicyName::parseImpl" == nullptr);
     std::vector<RowPolicyName> full_names;
     String cluster;
     if (!parseRowPolicyNamesAroundON(pos, expected, false, false, allow_on_cluster, full_names, cluster))
@@ -162,6 +181,7 @@ bool ParserRowPolicyName::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte
 
 bool ParserRowPolicyNames::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
+    // poco_assert("ParserRowPolicyName::parseImpl" == nullptr);
     std::vector<RowPolicyName> full_names;
     size_t num_added_names_last_time = 0;
     String cluster;
diff --git a/src/Parsers/parseQuery.cpp b/src/Parsers/parseQuery.cpp
index 8d794409f78..1b1ff439e13 100644
--- a/src/Parsers/parseQuery.cpp
+++ b/src/Parsers/parseQuery.cpp
@@ -139,6 +139,8 @@ void writeCommonErrorMessage(
     if (!query_description.empty())
         out << " (" << query_description << ")";
 
+    // poco_assert("writeCommonErrorMessage" == nullptr);
+
     out << ": failed at position " << (last_token.begin - begin + 1);
 
     if (last_token.type == TokenType::EndOfStream || last_token.type == TokenType::Semicolon)

From 7fa6ea4ccc7c03ea879396572fad24f7ffe93551 Mon Sep 17 00:00:00 2001
From: Ilya Golshtein <igolshtein@altinity.com>
Date: Thu, 16 Mar 2023 17:02:05 +0100
Subject: [PATCH 510/689] row policy template: tests and code cleanup, code
 review changes

---
 .../statements/create/row-policy.md           |  20 ++-
 src/Access/AccessEntityIO.cpp                 |   4 -
 src/Access/EnabledRowPolicies.cpp             |  30 +---
 src/Access/EnabledRowPolicies.h               |   8 -
 src/Access/RolesOrUsersSet.cpp                |  10 +-
 src/Access/RowPolicy.h                        |   4 +-
 src/Access/RowPolicyCache.cpp                 | 152 ++++++++----------
 src/Access/RowPolicyCache.h                   |   8 +-
 .../Access/ParserCreateRowPolicyQuery.cpp     |   1 -
 src/Parsers/Access/ParserRowPolicyName.cpp    |  26 +--
 src/Parsers/parseQuery.cpp                    |   2 -
 .../02131_row_policies_combination.sql        |   8 +
 ...ow_policies_database_combination.reference |  42 +++++
 ...5341_row_policies_database_combination.sql |  88 ++++++++++
 .../25341_row_policy_database.reference       |  22 +++
 .../0_stateless/25341_row_policy_database.sql |  53 ++++++
 16 files changed, 315 insertions(+), 163 deletions(-)
 create mode 100644 tests/queries/0_stateless/25341_row_policies_database_combination.reference
 create mode 100644 tests/queries/0_stateless/25341_row_policies_database_combination.sql
 create mode 100644 tests/queries/0_stateless/25341_row_policy_database.reference
 create mode 100644 tests/queries/0_stateless/25341_row_policy_database.sql

diff --git a/docs/en/sql-reference/statements/create/row-policy.md b/docs/en/sql-reference/statements/create/row-policy.md
index aa0a07747f2..83bb2e6bb9a 100644
--- a/docs/en/sql-reference/statements/create/row-policy.md
+++ b/docs/en/sql-reference/statements/create/row-policy.md
@@ -14,8 +14,8 @@ Row policies makes sense only for users with readonly access. If user can modify
 Syntax:
 
 ``` sql
-CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name1 [ON CLUSTER cluster_name1] ON [db1.]table1
-        [, policy_name2 [ON CLUSTER cluster_name2] ON [db2.]table2 ...]
+CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name1 [ON CLUSTER cluster_name1] ON [db1.]table1|db1.*
+        [, policy_name2 [ON CLUSTER cluster_name2] ON [db2.]table2|db2.* ...]
     [FOR SELECT] USING condition
     [AS {PERMISSIVE | RESTRICTIVE}]
     [TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}]
@@ -76,6 +76,20 @@ CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio
 
 enables the user `peter` to see rows only if both `b=1` AND `c=2`.
 
+Database policies are combined with table policies.
+
+For example, the following policies
+
+``` sql
+CREATE ROW POLICY pol1 ON mydb.* USING b=1 TO mira, peter
+CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio
+```
+
+enables the user `peter` to see table1 rows only if both `b=1` AND `c=2`, although
+any other table in mydb would have only `b=1` policy applied for the user.
+
+
+
 ## ON CLUSTER Clause
 
 Allows creating row policies on a cluster, see [Distributed DDL](../../../sql-reference/distributed-ddl.md).
@@ -88,3 +102,5 @@ Allows creating row policies on a cluster, see [Distributed DDL](../../../sql-re
 `CREATE ROW POLICY filter2 ON mydb.mytable USING a<1000 AND b=5 TO ALL EXCEPT mira`
 
 `CREATE ROW POLICY filter3 ON mydb.mytable USING 1 TO admin`
+
+`CREATE ROW POLICY filter4 ON mydb.* USING 1 TO admin`
diff --git a/src/Access/AccessEntityIO.cpp b/src/Access/AccessEntityIO.cpp
index 008ba5e5dfe..80bb63b04bf 100644
--- a/src/Access/AccessEntityIO.cpp
+++ b/src/Access/AccessEntityIO.cpp
@@ -28,9 +28,6 @@
 #include <boost/range/algorithm/copy.hpp>
 #include <boost/range/algorithm_ext/push_back.hpp>
 
-#include <Common/logger_useful.h>
-
-
 namespace DB
 {
 namespace ErrorCodes
@@ -65,7 +62,6 @@ AccessEntityPtr deserializeAccessEntityImpl(const String & definition)
     const char * end = begin + definition.size();
     while (pos < end)
     {
-        LOG_TRACE((&Poco::Logger::get("deserializeAccessEntityImpl")), "{}", std::string(pos, end));
         queries.emplace_back(parseQueryAndMovePosition(parser, pos, end, "", true, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH));
         while (isWhitespaceASCII(*pos) || *pos == ';')
             ++pos;
diff --git a/src/Access/EnabledRowPolicies.cpp b/src/Access/EnabledRowPolicies.cpp
index 225f211bdd4..9efac6e992e 100644
--- a/src/Access/EnabledRowPolicies.cpp
+++ b/src/Access/EnabledRowPolicies.cpp
@@ -3,8 +3,6 @@
 #include <boost/range/adaptor/map.hpp>
 #include <boost/range/algorithm/copy.hpp>
 
-#include <Common/logger_useful.h>
-
 
 namespace DB
 {
@@ -20,12 +18,6 @@ size_t EnabledRowPolicies::Hash::operator()(const MixedFiltersKey & key) const
     return std::hash<std::string_view>{}(key.database) - std::hash<std::string_view>{}(key.table_name) + static_cast<size_t>(key.filter_type);
 }
 
-
-// size_t EnabledRowPolicies::Hash::operator()(const MixedFiltersKey & key) const
-// {
-//     return std::hash<std::string_view>{}(key.database) + static_cast<size_t>(key.filter_type);
-// }
-
 EnabledRowPolicies::EnabledRowPolicies() : params()
 {
 }
@@ -40,37 +32,17 @@ EnabledRowPolicies::~EnabledRowPolicies() = default;
 RowPolicyFilterPtr EnabledRowPolicies::getFilter(const String & database, const String & table_name, RowPolicyFilterType filter_type) const
 {
     /// We don't lock `mutex` here.
-
     auto loaded = mixed_filters.load();
-    {
-
-
-        for (auto it = loaded->begin(); it != loaded->end(); ++it)
-        {
-            LOG_TRACE((&Poco::Logger::get("EnabledRowPolicies::getFilter")), "  db: {}, table {}", it->first.database, it->first.table_name);
-
-        }
-
-    }
-
-
-
-
     auto it = loaded->find({database, table_name, filter_type});
     if (it == loaded->end())
     {
-        it = loaded->find({database, "*", filter_type});
+        it = loaded->find({database, RowPolicy::ANY_TABLE_MARK, filter_type});
         if (it == loaded->end())
         {
-            LOG_TRACE((&Poco::Logger::get("EnabledRowPolicies::getFilter")), "db: {}, table {} - not found ({} records)",
-                database, table_name, loaded->size());
             return {};
         }
     }
 
-
-    LOG_TRACE((&Poco::Logger::get("EnabledRowPolicies::getFilter")), "db: {}, table {} - found ({} records)",
-        database, table_name, loaded->size());
     return it->second;
 }
 
diff --git a/src/Access/EnabledRowPolicies.h b/src/Access/EnabledRowPolicies.h
index e09d32264f0..b8e6b2e0549 100644
--- a/src/Access/EnabledRowPolicies.h
+++ b/src/Access/EnabledRowPolicies.h
@@ -72,14 +72,6 @@ private:
         auto toTuple() const { return std::tie(database, table_name, filter_type); }
         friend bool operator==(const MixedFiltersKey & left, const MixedFiltersKey & right) { return left.toTuple() == right.toTuple(); }
         friend bool operator!=(const MixedFiltersKey & left, const MixedFiltersKey & right) { return left.toTuple() != right.toTuple(); }
-        // friend bool operator==(const MixedFiltersKey & left, const MixedFiltersKey & right)
-        // {
-        //     return left.database == right.database && left.filter_type == right.filter_type;
-        // }
-        // friend bool operator!=(const MixedFiltersKey & left, const MixedFiltersKey & right)
-        // {
-        //     return left.database != right.database || left.filter_type != right.filter_type;
-        // }
     };
 
     struct Hash
diff --git a/src/Access/RolesOrUsersSet.cpp b/src/Access/RolesOrUsersSet.cpp
index 52374c3739d..c026ae42f76 100644
--- a/src/Access/RolesOrUsersSet.cpp
+++ b/src/Access/RolesOrUsersSet.cpp
@@ -228,25 +228,25 @@ void RolesOrUsersSet::add(const std::vector<UUID> & ids_)
 
 bool RolesOrUsersSet::match(const UUID & id) const
 {
-    return (all || ids.count(id)) && !except_ids.count(id);
+    return (all || ids.contains(id)) && !except_ids.contains(id);
 }
 
 
 bool RolesOrUsersSet::match(const UUID & user_id, const boost::container::flat_set<UUID> & enabled_roles) const
 {
-    if (!all && !ids.count(user_id))
+    if (!all && !ids.contains(user_id))
     {
         bool found_enabled_role = std::any_of(
-            enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return ids.count(enabled_role); });
+            enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return ids.contains(enabled_role); });
         if (!found_enabled_role)
             return false;
     }
 
-    if (except_ids.count(user_id))
+    if (except_ids.contains(user_id))
         return false;
 
     bool in_except_list = std::any_of(
-        enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.count(enabled_role); });
+        enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.contains(enabled_role); });
     return !in_except_list;
 }
 
diff --git a/src/Access/RowPolicy.h b/src/Access/RowPolicy.h
index 31ee876b47b..b9ba528e9bb 100644
--- a/src/Access/RowPolicy.h
+++ b/src/Access/RowPolicy.h
@@ -14,6 +14,8 @@ namespace DB
   */
 struct RowPolicy : public IAccessEntity
 {
+    static constexpr char ANY_TABLE_MARK[] = "*";
+
     void setShortName(const String & short_name);
     void setDatabase(const String & database);
     void setTableName(const String & table_name);
@@ -34,7 +36,7 @@ struct RowPolicy : public IAccessEntity
     /// in addition to all the restrictive policies.
     void setPermissive(bool permissive_ = true) { setRestrictive(!permissive_); }
     bool isPermissive() const { return !isRestrictive(); }
-    bool isDatabase() const { return full_name.table_name == "*"; }
+    bool isDatabase() const { return full_name.table_name == ANY_TABLE_MARK; }
 
     /// Sets that the policy is restrictive.
     /// A row is only accessible if at least one of the permissive policies passes,
diff --git a/src/Access/RowPolicyCache.cpp b/src/Access/RowPolicyCache.cpp
index 81e5acdf3ce..07bec185131 100644
--- a/src/Access/RowPolicyCache.cpp
+++ b/src/Access/RowPolicyCache.cpp
@@ -11,14 +11,13 @@
 #include <boost/smart_ptr/make_shared.hpp>
 #include <Core/Defines.h>
 
-#include <Common/logger_useful.h>
-
 
 namespace DB
 {
 namespace
 {
-    /// Accumulates filters from multiple row policies and joins them using the AND logical operation.
+    /// Helper to accumulate filters from multiple row policies and join them together
+    ///   by AND or OR logical operations.
     class FiltersMixer
     {
     public:
@@ -153,15 +152,7 @@ void RowPolicyCache::ensureAllRowPoliciesRead()
         auto policy = access_control.tryRead<RowPolicy>(id);
         if (policy)
         {
-            PolicyInfo policy_info(policy);
-            if (policy_info.database_and_table_name->second == "*")
-            {
-                database_policies.emplace(id, std::move(policy_info));
-            }
-            else
-            {
-                table_policies.emplace(id, std::move(policy_info));
-            }
+            all_policies.emplace(id, PolicyInfo(policy));
         }
     }
 }
@@ -170,23 +161,15 @@ void RowPolicyCache::ensureAllRowPoliciesRead()
 void RowPolicyCache::rowPolicyAddedOrChanged(const UUID & policy_id, const RowPolicyPtr & new_policy)
 {
     std::lock_guard lock{mutex};
-    bool found = true;
-
-    auto it = table_policies.find(policy_id);
-    if (it == table_policies.end())
+    auto it = all_policies.find(policy_id);
+    if (it == all_policies.end())
     {
-        it = database_policies.find(policy_id);
-        if (it == database_policies.end())
-        {
-            PolicyMap & policy_map = new_policy->isDatabase() ? database_policies : table_policies;
-            it = policy_map.emplace(policy_id, PolicyInfo(new_policy)).first;
-            found = false;
-        }
+        it = all_policies.emplace(policy_id, PolicyInfo(new_policy)).first;
     }
-
-    if (found && it->second.policy == new_policy)
+    else
     {
-        return;
+        if (it->second.policy == new_policy)
+            return;
     }
 
     auto & info = it->second;
@@ -198,15 +181,7 @@ void RowPolicyCache::rowPolicyAddedOrChanged(const UUID & policy_id, const RowPo
 void RowPolicyCache::rowPolicyRemoved(const UUID & policy_id)
 {
     std::lock_guard lock{mutex};
-    auto it = database_policies.find(policy_id);
-    if (it != database_policies.end())
-    {
-        database_policies.erase(it);
-    }
-    else
-    {
-        table_policies.erase(policy_id);
-    }
+    all_policies.erase(policy_id);
     mixFilters();
 }
 
@@ -246,73 +221,76 @@ void RowPolicyCache::mixFiltersFor(EnabledRowPolicies & enabled)
     std::unordered_map<MixedFiltersKey, MixerWithNames, Hash> table_mixers;
     std::unordered_map<MixedFiltersKey, MixerWithNames, Hash> database_mixers;
 
-
-    for (const auto & [policy_id, info] : database_policies)
+    /// populate database_mixers using database-level policies
+    ///  to aggregate (mix) rules per database
+    for (const auto & [policy_id, info] : all_policies)
     {
-        const auto & policy = *info.policy;
-        bool match = info.roles->match(enabled.params.user_id, enabled.params.enabled_roles);
-        for (auto filter_type : collections::range(0, RowPolicyFilterType::MAX))
+        if (info.isDatabase())
         {
-            auto filter_type_i = static_cast<size_t>(filter_type);
-            if (info.parsed_filters[filter_type_i])
+            const auto & policy = *info.policy;
+            bool match = info.roles->match(enabled.params.user_id, enabled.params.enabled_roles);
+            for (auto filter_type : collections::range(0, RowPolicyFilterType::MAX))
             {
-                MixedFiltersKey key{info.database_and_table_name->first,
-                    info.database_and_table_name->second,
-                    filter_type};
-                LOG_TRACE((&Poco::Logger::get("mixFiltersFor")), "db: {} : {}", key.database, key.table_name);
-
-                auto & mixer = database_mixers[key];  //  getting database level mixer
-                mixer.database_and_table_name = info.database_and_table_name;
-                if (match)
+                auto filter_type_i = static_cast<size_t>(filter_type);
+                if (info.parsed_filters[filter_type_i])
                 {
-                    mixer.mixer.add(info.parsed_filters[filter_type_i], policy.isRestrictive());
-                    mixer.policies.push_back(info.policy);
+                    MixedFiltersKey key{info.database_and_table_name->first,
+                        info.database_and_table_name->second,
+                        filter_type};
+
+                    auto & mixer = database_mixers[key];
+                    mixer.database_and_table_name = info.database_and_table_name;
+                    if (match)
+                    {
+                        mixer.mixer.add(info.parsed_filters[filter_type_i], policy.isRestrictive());
+                        mixer.policies.push_back(info.policy);
+                    }
                 }
             }
         }
     }
 
-
-    for (const auto & [policy_id, info] : table_policies)
+    /// populate table_mixers using database_mixers and table-level policies
+    for (const auto & [policy_id, info] : all_policies)
     {
-        const auto & policy = *info.policy;
-        bool match = info.roles->match(enabled.params.user_id, enabled.params.enabled_roles);
-        for (auto filter_type : collections::range(0, RowPolicyFilterType::MAX))
+        if (!info.isDatabase())
         {
-            auto filter_type_i = static_cast<size_t>(filter_type);
-            if (info.parsed_filters[filter_type_i])
+            const auto & policy = *info.policy;
+            bool match = info.roles->match(enabled.params.user_id, enabled.params.enabled_roles);
+            for (auto filter_type : collections::range(0, RowPolicyFilterType::MAX))
             {
-                MixedFiltersKey key{info.database_and_table_name->first,
-                    info.database_and_table_name->second,
-                    filter_type};
-                LOG_TRACE((&Poco::Logger::get("mixFiltersFor")), "table: {} : {}", key.database, key.table_name);
-                auto table_it = table_mixers.find(key);
-                if (table_it == table_mixers.end())
+                auto filter_type_i = static_cast<size_t>(filter_type);
+                if (info.parsed_filters[filter_type_i])
                 {
-                    LOG_TRACE((&Poco::Logger::get("mixFiltersFor")), "table: not found, looking for db");
-                    MixedFiltersKey database_key = key;
-                    database_key.table_name = "*";
+                    MixedFiltersKey key{info.database_and_table_name->first,
+                        info.database_and_table_name->second,
+                        filter_type};
+                    auto table_it = table_mixers.find(key);
+                    if (table_it == table_mixers.end())
+                    {   /// no exact match - create new mixer
+                        MixedFiltersKey database_key = key;
+                        database_key.table_name = RowPolicy::ANY_TABLE_MARK;
 
-                    auto database_it = database_mixers.find(database_key);
+                        auto database_it = database_mixers.find(database_key);
 
-                    if (database_it == database_mixers.end())
-                    {
-                        LOG_TRACE((&Poco::Logger::get("mixFiltersFor")), "table: not found, database not found");
-                        table_it = table_mixers.try_emplace(key).first;
+                        if (database_it == database_mixers.end())
+                        {
+                            table_it = table_mixers.try_emplace(key).first;
+                        }
+                        else
+                        {
+                            /// table policies are based on database ones
+                            table_it = table_mixers.insert({key, database_it->second}).first;
+                        }
                     }
-                    else
-                    {
-                        LOG_TRACE((&Poco::Logger::get("mixFiltersFor")), "table: not found, database found");
-                        table_it = table_mixers.insert({key, database_it->second}).first;
-                    }
-                }
 
-                auto & mixer = table_it->second; // table_mixers[key];    getting table level mixer
-                mixer.database_and_table_name = info.database_and_table_name;
-                if (match)
-                {
-                    mixer.mixer.add(info.parsed_filters[filter_type_i], policy.isRestrictive());
-                    mixer.policies.push_back(info.policy);
+                    auto & mixer = table_it->second; ///  getting table level mixer
+                    mixer.database_and_table_name = info.database_and_table_name;
+                    if (match)
+                    {
+                        mixer.mixer.add(info.parsed_filters[filter_type_i], policy.isRestrictive());
+                        mixer.policies.push_back(info.policy);
+                    }
                 }
             }
         }
@@ -320,7 +298,8 @@ void RowPolicyCache::mixFiltersFor(EnabledRowPolicies & enabled)
 
     auto mixed_filters = boost::make_shared<MixedFiltersMap>();
 
-    for (auto mixer_map_ptr :  { &table_mixers, &database_mixers})
+    /// retrieve aggregated policies from mixers
+    for (auto * mixer_map_ptr : {&table_mixers, &database_mixers})
     {
         for (auto & [key, mixer] : *mixer_map_ptr)
         {
@@ -332,7 +311,6 @@ void RowPolicyCache::mixFiltersFor(EnabledRowPolicies & enabled)
         }
     }
 
-
     enabled.mixed_filters.store(mixed_filters);
 }
 
diff --git a/src/Access/RowPolicyCache.h b/src/Access/RowPolicyCache.h
index 49e6a2e5df3..7260de22164 100644
--- a/src/Access/RowPolicyCache.h
+++ b/src/Access/RowPolicyCache.h
@@ -12,7 +12,6 @@ namespace DB
 class AccessControl;
 struct RolesOrUsersSet;
 struct RowPolicy;
-
 using RowPolicyPtr = std::shared_ptr<const RowPolicy>;
 
 /// Stores read and parsed row policies.
@@ -30,24 +29,21 @@ private:
         explicit PolicyInfo(const RowPolicyPtr & policy_) { setPolicy(policy_); }
         void setPolicy(const RowPolicyPtr & policy_);
 
+        bool isDatabase() const { return policy->isDatabase(); }
         RowPolicyPtr policy;
         const RolesOrUsersSet * roles = nullptr;
         std::shared_ptr<const std::pair<String, String>> database_and_table_name;
         ASTPtr parsed_filters[static_cast<size_t>(RowPolicyFilterType::MAX)];
     };
 
-    using PolicyMap = std::unordered_map<UUID, PolicyInfo>;
-
     void ensureAllRowPoliciesRead();
     void rowPolicyAddedOrChanged(const UUID & policy_id, const RowPolicyPtr & new_policy);
     void rowPolicyRemoved(const UUID & policy_id);
     void mixFilters();
     void mixFiltersFor(EnabledRowPolicies & enabled);
 
-
     const AccessControl & access_control;
-    PolicyMap database_policies;
-    PolicyMap table_policies;
+    std::unordered_map<UUID, PolicyInfo> all_policies;
     bool all_policies_read = false;
     scope_guard subscription;
     std::map<EnabledRowPolicies::Params, std::weak_ptr<EnabledRowPolicies>> enabled_row_policies;
diff --git a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp
index 03d0754ca1a..2c25fc14e7d 100644
--- a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp
+++ b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp
@@ -203,7 +203,6 @@ namespace
 
 bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
-    // poco_assert("ParserCreateRowPolicyQuery::parseImpl" == nullptr);
     bool alter = false;
     if (attach_mode)
     {
diff --git a/src/Parsers/Access/ParserRowPolicyName.cpp b/src/Parsers/Access/ParserRowPolicyName.cpp
index e087c45fad9..e5b4e01d5ac 100644
--- a/src/Parsers/Access/ParserRowPolicyName.cpp
+++ b/src/Parsers/Access/ParserRowPolicyName.cpp
@@ -7,8 +7,6 @@
 #include <Parsers/parseDatabaseAndTableName.h>
 #include <base/insertAtEnd.h>
 
-#include <Common/logger_useful.h>
-
 
 namespace DB
 {
@@ -28,19 +26,18 @@ namespace
         return IParserBase::wrapParseImpl(pos, [&]
         {
             String res_database, res_table_name;
-            // if (!parseDatabaseAndTableName(pos, expected, res_database, res_table_name))
-            bool any_database = false;
-            bool any_table = true;
+            bool is_any_database = false;
+            bool is_any_table = false;
 
-            if (!parseDatabaseAndTableNameOrAsterisks(pos, expected, res_database, any_database, res_table_name, any_table))
+            if (!parseDatabaseAndTableNameOrAsterisks(pos, expected, res_database, is_any_database, res_table_name, is_any_table)
+                || is_any_database)
             {
-                // poco_assert("parseDatabaseAndTableName failed" == nullptr);
-                LOG_TRACE((&Poco::Logger::get("ParserRowPolicyName")), "parseDatabaseAndTableName failed");
                 return false;
             }
-            if (any_table)
-                res_table_name = "*";
-
+            else if (is_any_table)
+            {
+                res_table_name = "*"; // RowPolicy::ANY_TABLE_MARK
+            }
 
             /// If table is specified without DB it cannot be followed by "ON"
             /// (but can be followed by "ON CLUSTER").
@@ -64,10 +61,8 @@ namespace
     }
 
 
-
     bool parseOnDBAndTableName(IParser::Pos & pos, Expected & expected, String & database, String & table_name)
     {
-        // poco_assert("parseOnDBAndTableNames" == nullptr);
         return IParserBase::wrapParseImpl(pos, [&]
         {
             return ParserKeyword{"ON"}.ignore(pos, expected) && parseDBAndTableName(pos, expected, database, table_name);
@@ -77,9 +72,6 @@ namespace
 
     bool parseOnDBAndTableNames(IParser::Pos & pos, Expected & expected, std::vector<std::pair<String, String>> & database_and_table_names)
     {
-        // poco_assert("parseOnDBAndTableNames" == nullptr);
-
-
         return IParserBase::wrapParseImpl(pos, [&]
         {
             if (!ParserKeyword{"ON"}.ignore(pos, expected))
@@ -164,7 +156,6 @@ namespace
 
 bool ParserRowPolicyName::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
-    // poco_assert("ParserRowPolicyName::parseImpl" == nullptr);
     std::vector<RowPolicyName> full_names;
     String cluster;
     if (!parseRowPolicyNamesAroundON(pos, expected, false, false, allow_on_cluster, full_names, cluster))
@@ -181,7 +172,6 @@ bool ParserRowPolicyName::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte
 
 bool ParserRowPolicyNames::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
-    // poco_assert("ParserRowPolicyName::parseImpl" == nullptr);
     std::vector<RowPolicyName> full_names;
     size_t num_added_names_last_time = 0;
     String cluster;
diff --git a/src/Parsers/parseQuery.cpp b/src/Parsers/parseQuery.cpp
index 1b1ff439e13..8d794409f78 100644
--- a/src/Parsers/parseQuery.cpp
+++ b/src/Parsers/parseQuery.cpp
@@ -139,8 +139,6 @@ void writeCommonErrorMessage(
     if (!query_description.empty())
         out << " (" << query_description << ")";
 
-    // poco_assert("writeCommonErrorMessage" == nullptr);
-
     out << ": failed at position " << (last_token.begin - begin + 1);
 
     if (last_token.type == TokenType::EndOfStream || last_token.type == TokenType::Semicolon)
diff --git a/tests/queries/0_stateless/02131_row_policies_combination.sql b/tests/queries/0_stateless/02131_row_policies_combination.sql
index b5be672bb1b..1cbbca754b6 100644
--- a/tests/queries/0_stateless/02131_row_policies_combination.sql
+++ b/tests/queries/0_stateless/02131_row_policies_combination.sql
@@ -8,6 +8,8 @@ DROP ROW POLICY IF EXISTS 02131_filter_3 ON 02131_rptable;
 DROP ROW POLICY IF EXISTS 02131_filter_4 ON 02131_rptable;
 DROP ROW POLICY IF EXISTS 02131_filter_5 ON 02131_rptable;
 
+-- the test assumes users_without_row_policies_can_read_rows is true
+
 SELECT 'None';
 SELECT * FROM 02131_rptable;
 
@@ -52,3 +54,9 @@ SELECT 'None';
 SELECT * FROM 02131_rptable;
 
 DROP TABLE 02131_rptable;
+
+DROP ROW POLICY IF EXISTS 02131_filter_1 ON 02131_rptable;
+DROP ROW POLICY IF EXISTS 02131_filter_2 ON 02131_rptable;
+DROP ROW POLICY IF EXISTS 02131_filter_3 ON 02131_rptable;
+DROP ROW POLICY IF EXISTS 02131_filter_4 ON 02131_rptable;
+DROP ROW POLICY IF EXISTS 02131_filter_5 ON 02131_rptable;
diff --git a/tests/queries/0_stateless/25341_row_policies_database_combination.reference b/tests/queries/0_stateless/25341_row_policies_database_combination.reference
new file mode 100644
index 00000000000..68ed02d1dc0
--- /dev/null
+++ b/tests/queries/0_stateless/25341_row_policies_database_combination.reference
@@ -0,0 +1,42 @@
+None
+1	10
+2	20
+3	30
+4	40
+R1: x == 1
+1	10
+R1, R2: (x == 1) OR (x == 2)
+1	10
+2	20
+R1, R2: (x == 2) FROM ANOTHER
+2	20
+R1, R2, R3: (x == 1) OR (x == 2) OR (x == 3)
+1	10
+2	20
+3	30
+R1, R2, R3, R4: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2)
+1	10
+2	20
+R1, R2, R3, R4, R5: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)
+2	20
+2	20
+R1, R2, R3, R4, R5: (x == 2) AND (y >= 20) FROM AFTER_RP
+2	20
+R1, R2, R3, R4, R5: (x == 2) AND (y >= 20) FROM ANOTHER
+2	20
+R2, R3, R4, R5: ((x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)
+2	20
+R3, R4, R5: (x == 3) AND (x <= 2) AND (y >= 20)
+R4, R5: (x <= 2) AND (y >= 20)
+2	20
+R5: (x >= 2)
+2	20
+3	30
+4	40
+Policy not applicable
+None
+1	10
+2	20
+3	30
+4	40
+No problematic policy, select works
diff --git a/tests/queries/0_stateless/25341_row_policies_database_combination.sql b/tests/queries/0_stateless/25341_row_policies_database_combination.sql
new file mode 100644
index 00000000000..aa9454b8c9b
--- /dev/null
+++ b/tests/queries/0_stateless/25341_row_policies_database_combination.sql
@@ -0,0 +1,88 @@
+-- Tags: no-parallel
+
+DROP DATABASE IF EXISTS 25341_db;
+CREATE DATABASE 25341_db;
+DROP TABLE IF EXISTS 25341_db.25341_rptable;
+DROP TABLE IF EXISTS 25341_db.25341_rptable_another;
+CREATE TABLE 25341_db.25341_rptable (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x;
+
+INSERT INTO 25341_db.25341_rptable VALUES (1, 10), (2, 20), (3, 30), (4, 40);
+
+CREATE TABLE 25341_db.25341_rptable_another ENGINE = MergeTree ORDER BY x AS SELECT * FROM 25341_db.25341_rptable;
+
+
+DROP ROW POLICY IF EXISTS 25341_filter_1 ON 25341_db.25341_rptable;
+DROP ROW POLICY IF EXISTS 25341_filter_2 ON 25341_db.*;
+DROP ROW POLICY IF EXISTS 25341_filter_3 ON 25341_db.25341_rptable;
+DROP ROW POLICY IF EXISTS 25341_filter_4 ON 25341_db.25341_rptable;
+DROP ROW POLICY IF EXISTS 25341_filter_5 ON 25341_db.*;
+
+-- the test assumes users_without_row_policies_can_read_rows is true
+
+SELECT 'None';
+SELECT * FROM 25341_db.25341_rptable;
+
+CREATE ROW POLICY 25341_filter_1 ON 25341_db.25341_rptable USING x=1 AS permissive TO ALL;
+SELECT 'R1: x == 1';
+SELECT * FROM 25341_db.25341_rptable;
+
+CREATE ROW POLICY 25341_filter_2 ON 25341_db.* USING x=2 AS permissive TO ALL;
+SELECT 'R1, R2: (x == 1) OR (x == 2)';
+SELECT * FROM 25341_db.25341_rptable;
+
+SELECT 'R1, R2: (x == 2) FROM ANOTHER';
+SELECT * FROM 25341_db.25341_rptable_another;
+
+CREATE ROW POLICY 25341_filter_3 ON 25341_db.25341_rptable USING x=3 AS permissive TO ALL;
+SELECT 'R1, R2, R3: (x == 1) OR (x == 2) OR (x == 3)';
+SELECT * FROM 25341_db.25341_rptable;
+
+CREATE ROW POLICY 25341_filter_4 ON 25341_db.25341_rptable USING x<=2 AS restrictive TO ALL;
+SELECT 'R1, R2, R3, R4: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2)';
+SELECT * FROM 25341_db.25341_rptable;
+
+CREATE ROW POLICY 25341_filter_5 ON 25341_db.* USING y>=20 AS restrictive TO ALL;
+SELECT 'R1, R2, R3, R4, R5: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)';
+SELECT * FROM 25341_db.25341_rptable;
+
+CREATE TABLE 25341_db.25341_after_rp ENGINE = MergeTree ORDER BY x AS SELECT * FROM 25341_db.25341_rptable;
+SELECT * FROM 25341_db.25341_after_rp;
+
+-- does not matter if policies or table are created first
+SELECT 'R1, R2, R3, R4, R5: (x == 2) AND (y >= 20) FROM AFTER_RP';
+SELECT * FROM 25341_db.25341_after_rp;
+
+SELECT 'R1, R2, R3, R4, R5: (x == 2) AND (y >= 20) FROM ANOTHER';
+SELECT * FROM 25341_db.25341_rptable_another;
+
+DROP ROW POLICY 25341_filter_1 ON 25341_db.25341_rptable;
+SELECT 'R2, R3, R4, R5: ((x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)';
+SELECT * FROM 25341_db.25341_rptable;
+
+DROP ROW POLICY 25341_filter_2 ON 25341_db.*;
+SELECT 'R3, R4, R5: (x == 3) AND (x <= 2) AND (y >= 20)';
+SELECT * FROM 25341_db.25341_rptable;
+
+DROP ROW POLICY 25341_filter_3 ON 25341_db.25341_rptable;
+SELECT 'R4, R5: (x <= 2) AND (y >= 20)';
+SELECT * FROM 25341_db.25341_rptable;
+
+DROP ROW POLICY 25341_filter_4 ON 25341_db.25341_rptable;
+SELECT 'R5: (x >= 2)';
+SELECT * FROM 25341_db.25341_rptable;
+
+CREATE TABLE 25341_db.25341_unexpected_columns (xx UInt8, yy UInt8) ENGINE = MergeTree ORDER BY xx;
+SELECT 'Policy not applicable';
+SELECT * FROM 25341_db.25341_unexpected_columns; -- { serverError 47 } -- Missing columns: 'x' while processing query
+
+DROP ROW POLICY 25341_filter_5 ON 25341_db.*;
+SELECT 'None';
+SELECT * FROM 25341_db.25341_rptable;
+
+SELECT 'No problematic policy, select works';
+SELECT 'Ok' FROM 25341_db.25341_unexpected_columns;
+
+DROP TABLE 25341_db.25341_rptable;
+DROP TABLE 25341_db.25341_rptable_another;
+DROP TABLE 25341_db.25341_unexpected_columns;
+DROP DATABASE 25341_db;
diff --git a/tests/queries/0_stateless/25341_row_policy_database.reference b/tests/queries/0_stateless/25341_row_policy_database.reference
new file mode 100644
index 00000000000..57125b64056
--- /dev/null
+++ b/tests/queries/0_stateless/25341_row_policy_database.reference
@@ -0,0 +1,22 @@
+-- database level policies
+  -- SHOW CREATE POLICY db1_25341 ON db1_25341.*
+CREATE ROW POLICY db1_25341 ON db1_25341.`*` FOR SELECT USING 1 TO ALL
+  -- SHOW CREATE POLICY ON db1_25341.*
+CREATE ROW POLICY `25341_filter_11` ON db1_25341.`25341_rqtable` FOR SELECT USING x = 2 TO ALL
+CREATE ROW POLICY db1_25341 ON db1_25341.`*` FOR SELECT USING 1 TO ALL
+CREATE ROW POLICY tbl1_25341 ON db1_25341.table FOR SELECT USING 1 TO ALL
+  -- SHOW CREATE POLICY ON db1_25341.`*`
+CREATE ROW POLICY db1_25341 ON db1_25341.`*` FOR SELECT USING 1 TO ALL
+R1, R2: (x == 1) OR (x == 2)
+1
+2
+Check system.query_log
+SELECT \'-- database level policies\';	[]
+SELECT \'  -- SHOW CREATE POLICY db1_25341 ON db1_25341.*\';	[]
+SELECT \'  -- SHOW CREATE POLICY ON db1_25341.*\';	[]
+SELECT \'  -- SHOW CREATE POLICY ON db1_25341.`*`\';	[]
+SELECT \'R1, R2: (x == 1) OR (x == 2)\';	[]
+SELECT * FROM 25341_rqtable_default;	['`25341_filter_11_db` ON default.`*`','`25341_filter_11` ON default.`25341_rqtable_default`']
+SELECT \'Check system.query_log\';	[]
+  -- CREATE DATABSE-LEVEL POLICY ON IN CURRENT DATABASE
+CREATE ROW POLICY db2_25341 ON db1_25341.`*` TO u1_25341
diff --git a/tests/queries/0_stateless/25341_row_policy_database.sql b/tests/queries/0_stateless/25341_row_policy_database.sql
new file mode 100644
index 00000000000..9d865487f0b
--- /dev/null
+++ b/tests/queries/0_stateless/25341_row_policy_database.sql
@@ -0,0 +1,53 @@
+-- Tags: no-parallel
+
+DROP DATABASE IF EXISTS db1_25341;
+DROP USER IF EXISTS u1_25341;
+CREATE USER u1_25341;
+
+CREATE DATABASE db1_25341;
+
+CREATE TABLE db1_25341.25341_rqtable (x UInt8) ENGINE = MergeTree ORDER BY x;
+INSERT INTO db1_25341.25341_rqtable VALUES (1), (2), (3), (4);
+
+
+SELECT '-- database level policies';
+CREATE ROW POLICY db1_25341 ON db1_25341.* USING 1 AS PERMISSIVE TO ALL;
+CREATE ROW POLICY tbl1_25341 ON db1_25341.table USING 1 AS PERMISSIVE TO ALL;
+SELECT '  -- SHOW CREATE POLICY db1_25341 ON db1_25341.*';
+SHOW CREATE POLICY db1_25341 ON db1_25341.*;
+SELECT '  -- SHOW CREATE POLICY ON db1_25341.*';
+SHOW CREATE POLICY ON db1_25341.*;
+SELECT '  -- SHOW CREATE POLICY ON db1_25341.`*`';
+SHOW CREATE POLICY ON db1_25341.`*`;
+DROP POLICY db1_25341 ON db1_25341.*;
+DROP POLICY tbl1_25341 ON db1_25341.table;
+
+CREATE ROW POLICY any_25341 ON *.some_table USING 1 AS PERMISSIVE TO ALL; -- { clientError 62 }
+
+CREATE TABLE 25341_rqtable_default (x UInt8) ENGINE = MergeTree ORDER BY x;
+
+CREATE ROW POLICY 25341_filter_11_db ON * USING x=1 AS permissive TO ALL;
+CREATE ROW POLICY 25341_filter_11 ON 25341_rqtable_default USING x=2 AS permissive TO ALL;
+
+INSERT INTO 25341_rqtable_default VALUES (1), (2), (3), (4);
+
+SELECT 'R1, R2: (x == 1) OR (x == 2)';
+SELECT * FROM 25341_rqtable_default;
+
+DROP TABLE 25341_rqtable_default;
+
+SELECT 'Check system.query_log';
+SYSTEM FLUSH LOGS;
+SELECT query, used_row_policies FROM system.query_log WHERE current_database == currentDatabase() AND type == 'QueryStart' AND query_kind == 'Select' ORDER BY event_time_microseconds;
+
+DROP ROW POLICY 25341_filter_11_db ON *;
+DROP ROW POLICY 25341_filter_11 ON 25341_rqtable_default;
+
+USE db1_25341;
+SELECT '  -- CREATE DATABSE-LEVEL POLICY ON IN CURRENT DATABASE';
+CREATE ROW POLICY db2_25341 ON * TO u1_25341;
+SHOW CREATE POLICY db2_25341 ON *;
+
+DROP ROW POLICY db2_25341 ON *;
+
+DROP USER u1_25341;

From 28c6d4fb630c47f821d2225677ec305a8fdfd883 Mon Sep 17 00:00:00 2001
From: Ilya Golshtein <igolshtein@altinity.com>
Date: Thu, 30 Mar 2023 01:29:02 +0200
Subject: [PATCH 511/689] row policy template - rollback old test to make
 stylecheck happy

---
 .../0_stateless/02131_row_policies_combination.sql        | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/tests/queries/0_stateless/02131_row_policies_combination.sql b/tests/queries/0_stateless/02131_row_policies_combination.sql
index 1cbbca754b6..b5be672bb1b 100644
--- a/tests/queries/0_stateless/02131_row_policies_combination.sql
+++ b/tests/queries/0_stateless/02131_row_policies_combination.sql
@@ -8,8 +8,6 @@ DROP ROW POLICY IF EXISTS 02131_filter_3 ON 02131_rptable;
 DROP ROW POLICY IF EXISTS 02131_filter_4 ON 02131_rptable;
 DROP ROW POLICY IF EXISTS 02131_filter_5 ON 02131_rptable;
 
--- the test assumes users_without_row_policies_can_read_rows is true
-
 SELECT 'None';
 SELECT * FROM 02131_rptable;
 
@@ -54,9 +52,3 @@ SELECT 'None';
 SELECT * FROM 02131_rptable;
 
 DROP TABLE 02131_rptable;
-
-DROP ROW POLICY IF EXISTS 02131_filter_1 ON 02131_rptable;
-DROP ROW POLICY IF EXISTS 02131_filter_2 ON 02131_rptable;
-DROP ROW POLICY IF EXISTS 02131_filter_3 ON 02131_rptable;
-DROP ROW POLICY IF EXISTS 02131_filter_4 ON 02131_rptable;
-DROP ROW POLICY IF EXISTS 02131_filter_5 ON 02131_rptable;

From 2ed8e318cfd9d857869de601ed08ff4d9904a7b1 Mon Sep 17 00:00:00 2001
From: Ilya Golshtein <igolshtein@altinity.com>
Date: Thu, 30 Mar 2023 10:13:50 +0200
Subject: [PATCH 512/689] row policy template - tests renumbered to keep
 monotony

---
 ...w_policies_database_combination.reference} |  0
 ...2703_row_policies_database_combination.sql | 88 +++++++++++++++++++
 .../02703_row_policy_database.reference       | 21 +++++
 .../0_stateless/02703_row_policy_database.sql | 53 +++++++++++
 ...5341_row_policies_database_combination.sql | 88 -------------------
 .../25341_row_policy_database.reference       | 22 -----
 .../0_stateless/25341_row_policy_database.sql | 53 -----------
 7 files changed, 162 insertions(+), 163 deletions(-)
 rename tests/queries/0_stateless/{25341_row_policies_database_combination.reference => 02703_row_policies_database_combination.reference} (100%)
 create mode 100644 tests/queries/0_stateless/02703_row_policies_database_combination.sql
 create mode 100644 tests/queries/0_stateless/02703_row_policy_database.reference
 create mode 100644 tests/queries/0_stateless/02703_row_policy_database.sql
 delete mode 100644 tests/queries/0_stateless/25341_row_policies_database_combination.sql
 delete mode 100644 tests/queries/0_stateless/25341_row_policy_database.reference
 delete mode 100644 tests/queries/0_stateless/25341_row_policy_database.sql

diff --git a/tests/queries/0_stateless/25341_row_policies_database_combination.reference b/tests/queries/0_stateless/02703_row_policies_database_combination.reference
similarity index 100%
rename from tests/queries/0_stateless/25341_row_policies_database_combination.reference
rename to tests/queries/0_stateless/02703_row_policies_database_combination.reference
diff --git a/tests/queries/0_stateless/02703_row_policies_database_combination.sql b/tests/queries/0_stateless/02703_row_policies_database_combination.sql
new file mode 100644
index 00000000000..f9b466f1ade
--- /dev/null
+++ b/tests/queries/0_stateless/02703_row_policies_database_combination.sql
@@ -0,0 +1,88 @@
+-- Tags: no-parallel
+
+DROP DATABASE IF EXISTS 02703_db;
+CREATE DATABASE 02703_db;
+DROP TABLE IF EXISTS 02703_db.02703_rptable;
+DROP TABLE IF EXISTS 02703_db.02703_rptable_another;
+CREATE TABLE 02703_db.02703_rptable (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x;
+
+INSERT INTO 02703_db.02703_rptable VALUES (1, 10), (2, 20), (3, 30), (4, 40);
+
+CREATE TABLE 02703_db.02703_rptable_another ENGINE = MergeTree ORDER BY x AS SELECT * FROM 02703_db.02703_rptable;
+
+
+DROP ROW POLICY IF EXISTS 02703_filter_1 ON 02703_db.02703_rptable;
+DROP ROW POLICY IF EXISTS 02703_filter_2 ON 02703_db.*;
+DROP ROW POLICY IF EXISTS 02703_filter_3 ON 02703_db.02703_rptable;
+DROP ROW POLICY IF EXISTS 02703_filter_4 ON 02703_db.02703_rptable;
+DROP ROW POLICY IF EXISTS 02703_filter_5 ON 02703_db.*;
+
+-- the test assumes users_without_row_policies_can_read_rows is true
+
+SELECT 'None';
+SELECT * FROM 02703_db.02703_rptable;
+
+CREATE ROW POLICY 02703_filter_1 ON 02703_db.02703_rptable USING x=1 AS permissive TO ALL;
+SELECT 'R1: x == 1';
+SELECT * FROM 02703_db.02703_rptable;
+
+CREATE ROW POLICY 02703_filter_2 ON 02703_db.* USING x=2 AS permissive TO ALL;
+SELECT 'R1, R2: (x == 1) OR (x == 2)';
+SELECT * FROM 02703_db.02703_rptable;
+
+SELECT 'R1, R2: (x == 2) FROM ANOTHER';
+SELECT * FROM 02703_db.02703_rptable_another;
+
+CREATE ROW POLICY 02703_filter_3 ON 02703_db.02703_rptable USING x=3 AS permissive TO ALL;
+SELECT 'R1, R2, R3: (x == 1) OR (x == 2) OR (x == 3)';
+SELECT * FROM 02703_db.02703_rptable;
+
+CREATE ROW POLICY 02703_filter_4 ON 02703_db.02703_rptable USING x<=2 AS restrictive TO ALL;
+SELECT 'R1, R2, R3, R4: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2)';
+SELECT * FROM 02703_db.02703_rptable;
+
+CREATE ROW POLICY 02703_filter_5 ON 02703_db.* USING y>=20 AS restrictive TO ALL;
+SELECT 'R1, R2, R3, R4, R5: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)';
+SELECT * FROM 02703_db.02703_rptable;
+
+CREATE TABLE 02703_db.02703_after_rp ENGINE = MergeTree ORDER BY x AS SELECT * FROM 02703_db.02703_rptable;
+SELECT * FROM 02703_db.02703_after_rp;
+
+-- does not matter if policies or table are created first
+SELECT 'R1, R2, R3, R4, R5: (x == 2) AND (y >= 20) FROM AFTER_RP';
+SELECT * FROM 02703_db.02703_after_rp;
+
+SELECT 'R1, R2, R3, R4, R5: (x == 2) AND (y >= 20) FROM ANOTHER';
+SELECT * FROM 02703_db.02703_rptable_another;
+
+DROP ROW POLICY 02703_filter_1 ON 02703_db.02703_rptable;
+SELECT 'R2, R3, R4, R5: ((x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)';
+SELECT * FROM 02703_db.02703_rptable;
+
+DROP ROW POLICY 02703_filter_2 ON 02703_db.*;
+SELECT 'R3, R4, R5: (x == 3) AND (x <= 2) AND (y >= 20)';
+SELECT * FROM 02703_db.02703_rptable;
+
+DROP ROW POLICY 02703_filter_3 ON 02703_db.02703_rptable;
+SELECT 'R4, R5: (x <= 2) AND (y >= 20)';
+SELECT * FROM 02703_db.02703_rptable;
+
+DROP ROW POLICY 02703_filter_4 ON 02703_db.02703_rptable;
+SELECT 'R5: (x >= 2)';
+SELECT * FROM 02703_db.02703_rptable;
+
+CREATE TABLE 02703_db.02703_unexpected_columns (xx UInt8, yy UInt8) ENGINE = MergeTree ORDER BY xx;
+SELECT 'Policy not applicable';
+SELECT * FROM 02703_db.02703_unexpected_columns; -- { serverError 47 } -- Missing columns: 'x' while processing query
+
+DROP ROW POLICY 02703_filter_5 ON 02703_db.*;
+SELECT 'None';
+SELECT * FROM 02703_db.02703_rptable;
+
+SELECT 'No problematic policy, select works';
+SELECT 'Ok' FROM 02703_db.02703_unexpected_columns;
+
+DROP TABLE 02703_db.02703_rptable;
+DROP TABLE 02703_db.02703_rptable_another;
+DROP TABLE 02703_db.02703_unexpected_columns;
+DROP DATABASE 02703_db;
diff --git a/tests/queries/0_stateless/02703_row_policy_database.reference b/tests/queries/0_stateless/02703_row_policy_database.reference
new file mode 100644
index 00000000000..e318dfac429
--- /dev/null
+++ b/tests/queries/0_stateless/02703_row_policy_database.reference
@@ -0,0 +1,21 @@
+-- database level policies
+  -- SHOW CREATE POLICY db1_02703 ON db1_02703.*
+CREATE ROW POLICY db1_02703 ON db1_02703.`*` FOR SELECT USING 1 TO ALL
+  -- SHOW CREATE POLICY ON db1_02703.*
+CREATE ROW POLICY db1_02703 ON db1_02703.`*` FOR SELECT USING 1 TO ALL
+CREATE ROW POLICY tbl1_02703 ON db1_02703.table FOR SELECT USING 1 TO ALL
+  -- SHOW CREATE POLICY ON db1_02703.`*`
+CREATE ROW POLICY db1_02703 ON db1_02703.`*` FOR SELECT USING 1 TO ALL
+R1, R2: (x == 1) OR (x == 2)
+1
+2
+Check system.query_log
+SELECT \'-- database level policies\';	[]
+SELECT \'  -- SHOW CREATE POLICY db1_02703 ON db1_02703.*\';	[]
+SELECT \'  -- SHOW CREATE POLICY ON db1_02703.*\';	[]
+SELECT \'  -- SHOW CREATE POLICY ON db1_02703.`*`\';	[]
+SELECT \'R1, R2: (x == 1) OR (x == 2)\';	[]
+SELECT * FROM 02703_rqtable_default;	['`02703_filter_11_db` ON default.`*`','`02703_filter_11` ON default.`02703_rqtable_default`']
+SELECT \'Check system.query_log\';	[]
+  -- CREATE DATABSE-LEVEL POLICY ON IN CURRENT DATABASE
+CREATE ROW POLICY db2_02703 ON db1_02703.`*` TO u1_02703
diff --git a/tests/queries/0_stateless/02703_row_policy_database.sql b/tests/queries/0_stateless/02703_row_policy_database.sql
new file mode 100644
index 00000000000..85f5a44dfbf
--- /dev/null
+++ b/tests/queries/0_stateless/02703_row_policy_database.sql
@@ -0,0 +1,53 @@
+-- Tags: no-parallel
+
+DROP DATABASE IF EXISTS db1_02703;
+DROP USER IF EXISTS u1_02703;
+CREATE USER u1_02703;
+
+CREATE DATABASE db1_02703;
+
+CREATE TABLE db1_02703.02703_rqtable (x UInt8) ENGINE = MergeTree ORDER BY x;
+INSERT INTO db1_02703.02703_rqtable VALUES (1), (2), (3), (4);
+
+
+SELECT '-- database level policies';
+CREATE ROW POLICY db1_02703 ON db1_02703.* USING 1 AS PERMISSIVE TO ALL;
+CREATE ROW POLICY tbl1_02703 ON db1_02703.table USING 1 AS PERMISSIVE TO ALL;
+SELECT '  -- SHOW CREATE POLICY db1_02703 ON db1_02703.*';
+SHOW CREATE POLICY db1_02703 ON db1_02703.*;
+SELECT '  -- SHOW CREATE POLICY ON db1_02703.*';
+SHOW CREATE POLICY ON db1_02703.*;
+SELECT '  -- SHOW CREATE POLICY ON db1_02703.`*`';
+SHOW CREATE POLICY ON db1_02703.`*`;
+DROP POLICY db1_02703 ON db1_02703.*;
+DROP POLICY tbl1_02703 ON db1_02703.table;
+
+CREATE ROW POLICY any_02703 ON *.some_table USING 1 AS PERMISSIVE TO ALL; -- { clientError 62 }
+
+CREATE TABLE 02703_rqtable_default (x UInt8) ENGINE = MergeTree ORDER BY x;
+
+CREATE ROW POLICY 02703_filter_11_db ON * USING x=1 AS permissive TO ALL;
+CREATE ROW POLICY 02703_filter_11 ON 02703_rqtable_default USING x=2 AS permissive TO ALL;
+
+INSERT INTO 02703_rqtable_default VALUES (1), (2), (3), (4);
+
+SELECT 'R1, R2: (x == 1) OR (x == 2)';
+SELECT * FROM 02703_rqtable_default;
+
+DROP TABLE 02703_rqtable_default;
+
+SELECT 'Check system.query_log';
+SYSTEM FLUSH LOGS;
+SELECT query, used_row_policies FROM system.query_log WHERE current_database == currentDatabase() AND type == 'QueryStart' AND query_kind == 'Select' ORDER BY event_time_microseconds;
+
+DROP ROW POLICY 02703_filter_11_db ON *;
+DROP ROW POLICY 02703_filter_11 ON 02703_rqtable_default;
+
+USE db1_02703;
+SELECT '  -- CREATE DATABSE-LEVEL POLICY ON IN CURRENT DATABASE';
+CREATE ROW POLICY db2_02703 ON * TO u1_02703;
+SHOW CREATE POLICY db2_02703 ON *;
+
+DROP ROW POLICY db2_02703 ON *;
+
+DROP USER u1_02703;
diff --git a/tests/queries/0_stateless/25341_row_policies_database_combination.sql b/tests/queries/0_stateless/25341_row_policies_database_combination.sql
deleted file mode 100644
index aa9454b8c9b..00000000000
--- a/tests/queries/0_stateless/25341_row_policies_database_combination.sql
+++ /dev/null
@@ -1,88 +0,0 @@
--- Tags: no-parallel
-
-DROP DATABASE IF EXISTS 25341_db;
-CREATE DATABASE 25341_db;
-DROP TABLE IF EXISTS 25341_db.25341_rptable;
-DROP TABLE IF EXISTS 25341_db.25341_rptable_another;
-CREATE TABLE 25341_db.25341_rptable (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x;
-
-INSERT INTO 25341_db.25341_rptable VALUES (1, 10), (2, 20), (3, 30), (4, 40);
-
-CREATE TABLE 25341_db.25341_rptable_another ENGINE = MergeTree ORDER BY x AS SELECT * FROM 25341_db.25341_rptable;
-
-
-DROP ROW POLICY IF EXISTS 25341_filter_1 ON 25341_db.25341_rptable;
-DROP ROW POLICY IF EXISTS 25341_filter_2 ON 25341_db.*;
-DROP ROW POLICY IF EXISTS 25341_filter_3 ON 25341_db.25341_rptable;
-DROP ROW POLICY IF EXISTS 25341_filter_4 ON 25341_db.25341_rptable;
-DROP ROW POLICY IF EXISTS 25341_filter_5 ON 25341_db.*;
-
--- the test assumes users_without_row_policies_can_read_rows is true
-
-SELECT 'None';
-SELECT * FROM 25341_db.25341_rptable;
-
-CREATE ROW POLICY 25341_filter_1 ON 25341_db.25341_rptable USING x=1 AS permissive TO ALL;
-SELECT 'R1: x == 1';
-SELECT * FROM 25341_db.25341_rptable;
-
-CREATE ROW POLICY 25341_filter_2 ON 25341_db.* USING x=2 AS permissive TO ALL;
-SELECT 'R1, R2: (x == 1) OR (x == 2)';
-SELECT * FROM 25341_db.25341_rptable;
-
-SELECT 'R1, R2: (x == 2) FROM ANOTHER';
-SELECT * FROM 25341_db.25341_rptable_another;
-
-CREATE ROW POLICY 25341_filter_3 ON 25341_db.25341_rptable USING x=3 AS permissive TO ALL;
-SELECT 'R1, R2, R3: (x == 1) OR (x == 2) OR (x == 3)';
-SELECT * FROM 25341_db.25341_rptable;
-
-CREATE ROW POLICY 25341_filter_4 ON 25341_db.25341_rptable USING x<=2 AS restrictive TO ALL;
-SELECT 'R1, R2, R3, R4: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2)';
-SELECT * FROM 25341_db.25341_rptable;
-
-CREATE ROW POLICY 25341_filter_5 ON 25341_db.* USING y>=20 AS restrictive TO ALL;
-SELECT 'R1, R2, R3, R4, R5: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)';
-SELECT * FROM 25341_db.25341_rptable;
-
-CREATE TABLE 25341_db.25341_after_rp ENGINE = MergeTree ORDER BY x AS SELECT * FROM 25341_db.25341_rptable;
-SELECT * FROM 25341_db.25341_after_rp;
-
--- does not matter if policies or table are created first
-SELECT 'R1, R2, R3, R4, R5: (x == 2) AND (y >= 20) FROM AFTER_RP';
-SELECT * FROM 25341_db.25341_after_rp;
-
-SELECT 'R1, R2, R3, R4, R5: (x == 2) AND (y >= 20) FROM ANOTHER';
-SELECT * FROM 25341_db.25341_rptable_another;
-
-DROP ROW POLICY 25341_filter_1 ON 25341_db.25341_rptable;
-SELECT 'R2, R3, R4, R5: ((x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)';
-SELECT * FROM 25341_db.25341_rptable;
-
-DROP ROW POLICY 25341_filter_2 ON 25341_db.*;
-SELECT 'R3, R4, R5: (x == 3) AND (x <= 2) AND (y >= 20)';
-SELECT * FROM 25341_db.25341_rptable;
-
-DROP ROW POLICY 25341_filter_3 ON 25341_db.25341_rptable;
-SELECT 'R4, R5: (x <= 2) AND (y >= 20)';
-SELECT * FROM 25341_db.25341_rptable;
-
-DROP ROW POLICY 25341_filter_4 ON 25341_db.25341_rptable;
-SELECT 'R5: (x >= 2)';
-SELECT * FROM 25341_db.25341_rptable;
-
-CREATE TABLE 25341_db.25341_unexpected_columns (xx UInt8, yy UInt8) ENGINE = MergeTree ORDER BY xx;
-SELECT 'Policy not applicable';
-SELECT * FROM 25341_db.25341_unexpected_columns; -- { serverError 47 } -- Missing columns: 'x' while processing query
-
-DROP ROW POLICY 25341_filter_5 ON 25341_db.*;
-SELECT 'None';
-SELECT * FROM 25341_db.25341_rptable;
-
-SELECT 'No problematic policy, select works';
-SELECT 'Ok' FROM 25341_db.25341_unexpected_columns;
-
-DROP TABLE 25341_db.25341_rptable;
-DROP TABLE 25341_db.25341_rptable_another;
-DROP TABLE 25341_db.25341_unexpected_columns;
-DROP DATABASE 25341_db;
diff --git a/tests/queries/0_stateless/25341_row_policy_database.reference b/tests/queries/0_stateless/25341_row_policy_database.reference
deleted file mode 100644
index 57125b64056..00000000000
--- a/tests/queries/0_stateless/25341_row_policy_database.reference
+++ /dev/null
@@ -1,22 +0,0 @@
--- database level policies
-  -- SHOW CREATE POLICY db1_25341 ON db1_25341.*
-CREATE ROW POLICY db1_25341 ON db1_25341.`*` FOR SELECT USING 1 TO ALL
-  -- SHOW CREATE POLICY ON db1_25341.*
-CREATE ROW POLICY `25341_filter_11` ON db1_25341.`25341_rqtable` FOR SELECT USING x = 2 TO ALL
-CREATE ROW POLICY db1_25341 ON db1_25341.`*` FOR SELECT USING 1 TO ALL
-CREATE ROW POLICY tbl1_25341 ON db1_25341.table FOR SELECT USING 1 TO ALL
-  -- SHOW CREATE POLICY ON db1_25341.`*`
-CREATE ROW POLICY db1_25341 ON db1_25341.`*` FOR SELECT USING 1 TO ALL
-R1, R2: (x == 1) OR (x == 2)
-1
-2
-Check system.query_log
-SELECT \'-- database level policies\';	[]
-SELECT \'  -- SHOW CREATE POLICY db1_25341 ON db1_25341.*\';	[]
-SELECT \'  -- SHOW CREATE POLICY ON db1_25341.*\';	[]
-SELECT \'  -- SHOW CREATE POLICY ON db1_25341.`*`\';	[]
-SELECT \'R1, R2: (x == 1) OR (x == 2)\';	[]
-SELECT * FROM 25341_rqtable_default;	['`25341_filter_11_db` ON default.`*`','`25341_filter_11` ON default.`25341_rqtable_default`']
-SELECT \'Check system.query_log\';	[]
-  -- CREATE DATABSE-LEVEL POLICY ON IN CURRENT DATABASE
-CREATE ROW POLICY db2_25341 ON db1_25341.`*` TO u1_25341
diff --git a/tests/queries/0_stateless/25341_row_policy_database.sql b/tests/queries/0_stateless/25341_row_policy_database.sql
deleted file mode 100644
index 9d865487f0b..00000000000
--- a/tests/queries/0_stateless/25341_row_policy_database.sql
+++ /dev/null
@@ -1,53 +0,0 @@
--- Tags: no-parallel
-
-DROP DATABASE IF EXISTS db1_25341;
-DROP USER IF EXISTS u1_25341;
-CREATE USER u1_25341;
-
-CREATE DATABASE db1_25341;
-
-CREATE TABLE db1_25341.25341_rqtable (x UInt8) ENGINE = MergeTree ORDER BY x;
-INSERT INTO db1_25341.25341_rqtable VALUES (1), (2), (3), (4);
-
-
-SELECT '-- database level policies';
-CREATE ROW POLICY db1_25341 ON db1_25341.* USING 1 AS PERMISSIVE TO ALL;
-CREATE ROW POLICY tbl1_25341 ON db1_25341.table USING 1 AS PERMISSIVE TO ALL;
-SELECT '  -- SHOW CREATE POLICY db1_25341 ON db1_25341.*';
-SHOW CREATE POLICY db1_25341 ON db1_25341.*;
-SELECT '  -- SHOW CREATE POLICY ON db1_25341.*';
-SHOW CREATE POLICY ON db1_25341.*;
-SELECT '  -- SHOW CREATE POLICY ON db1_25341.`*`';
-SHOW CREATE POLICY ON db1_25341.`*`;
-DROP POLICY db1_25341 ON db1_25341.*;
-DROP POLICY tbl1_25341 ON db1_25341.table;
-
-CREATE ROW POLICY any_25341 ON *.some_table USING 1 AS PERMISSIVE TO ALL; -- { clientError 62 }
-
-CREATE TABLE 25341_rqtable_default (x UInt8) ENGINE = MergeTree ORDER BY x;
-
-CREATE ROW POLICY 25341_filter_11_db ON * USING x=1 AS permissive TO ALL;
-CREATE ROW POLICY 25341_filter_11 ON 25341_rqtable_default USING x=2 AS permissive TO ALL;
-
-INSERT INTO 25341_rqtable_default VALUES (1), (2), (3), (4);
-
-SELECT 'R1, R2: (x == 1) OR (x == 2)';
-SELECT * FROM 25341_rqtable_default;
-
-DROP TABLE 25341_rqtable_default;
-
-SELECT 'Check system.query_log';
-SYSTEM FLUSH LOGS;
-SELECT query, used_row_policies FROM system.query_log WHERE current_database == currentDatabase() AND type == 'QueryStart' AND query_kind == 'Select' ORDER BY event_time_microseconds;
-
-DROP ROW POLICY 25341_filter_11_db ON *;
-DROP ROW POLICY 25341_filter_11 ON 25341_rqtable_default;
-
-USE db1_25341;
-SELECT '  -- CREATE DATABSE-LEVEL POLICY ON IN CURRENT DATABASE';
-CREATE ROW POLICY db2_25341 ON * TO u1_25341;
-SHOW CREATE POLICY db2_25341 ON *;
-
-DROP ROW POLICY db2_25341 ON *;
-
-DROP USER u1_25341;

From 6852ae0d938131863138ed532ed291acbf750444 Mon Sep 17 00:00:00 2001
From: Ilya Golshtein <igolshtein@altinity.com>
Date: Tue, 2 May 2023 21:11:18 +0000
Subject: [PATCH 513/689] row_policy_template - external code review changes

---
 src/Access/EnabledRowPolicies.cpp                |  2 +-
 src/Access/RowPolicy.h                           |  4 +++-
 src/Access/RowPolicyCache.cpp                    | 16 ++++++++++++----
 src/Access/RowPolicyCache.h                      |  2 +-
 ...ase.sql => 02703_row_policy_for_database.sql} |  0
 5 files changed, 17 insertions(+), 7 deletions(-)
 rename tests/queries/0_stateless/{02703_row_policy_database.sql => 02703_row_policy_for_database.sql} (100%)

diff --git a/src/Access/EnabledRowPolicies.cpp b/src/Access/EnabledRowPolicies.cpp
index 9efac6e992e..be78dd62146 100644
--- a/src/Access/EnabledRowPolicies.cpp
+++ b/src/Access/EnabledRowPolicies.cpp
@@ -35,7 +35,7 @@ RowPolicyFilterPtr EnabledRowPolicies::getFilter(const String & database, const
     auto loaded = mixed_filters.load();
     auto it = loaded->find({database, table_name, filter_type});
     if (it == loaded->end())
-    {
+    {   /// Look for a policy for database if a table policy not found
         it = loaded->find({database, RowPolicy::ANY_TABLE_MARK, filter_type});
         if (it == loaded->end())
         {
diff --git a/src/Access/RowPolicy.h b/src/Access/RowPolicy.h
index b9ba528e9bb..348ebfa1637 100644
--- a/src/Access/RowPolicy.h
+++ b/src/Access/RowPolicy.h
@@ -36,7 +36,9 @@ struct RowPolicy : public IAccessEntity
     /// in addition to all the restrictive policies.
     void setPermissive(bool permissive_ = true) { setRestrictive(!permissive_); }
     bool isPermissive() const { return !isRestrictive(); }
-    bool isDatabase() const { return full_name.table_name == ANY_TABLE_MARK; }
+
+    /// Applied for entire database
+    bool isForDatabase() const { return full_name.table_name == ANY_TABLE_MARK; }
 
     /// Sets that the policy is restrictive.
     /// A row is only accessible if at least one of the permissive policies passes,
diff --git a/src/Access/RowPolicyCache.cpp b/src/Access/RowPolicyCache.cpp
index 07bec185131..07670a8fe84 100644
--- a/src/Access/RowPolicyCache.cpp
+++ b/src/Access/RowPolicyCache.cpp
@@ -218,14 +218,13 @@ void RowPolicyCache::mixFiltersFor(EnabledRowPolicies & enabled)
         std::vector<RowPolicyPtr> policies;
     };
 
-    std::unordered_map<MixedFiltersKey, MixerWithNames, Hash> table_mixers;
     std::unordered_map<MixedFiltersKey, MixerWithNames, Hash> database_mixers;
 
     /// populate database_mixers using database-level policies
     ///  to aggregate (mix) rules per database
     for (const auto & [policy_id, info] : all_policies)
     {
-        if (info.isDatabase())
+        if (info.isForDatabase())
         {
             const auto & policy = *info.policy;
             bool match = info.roles->match(enabled.params.user_id, enabled.params.enabled_roles);
@@ -250,10 +249,12 @@ void RowPolicyCache::mixFiltersFor(EnabledRowPolicies & enabled)
         }
     }
 
+    std::unordered_map<MixedFiltersKey, MixerWithNames, Hash> table_mixers;
+
     /// populate table_mixers using database_mixers and table-level policies
     for (const auto & [policy_id, info] : all_policies)
     {
-        if (!info.isDatabase())
+        if (!info.isForDatabase())
         {
             const auto & policy = *info.policy;
             bool match = info.roles->match(enabled.params.user_id, enabled.params.enabled_roles);
@@ -298,7 +299,14 @@ void RowPolicyCache::mixFiltersFor(EnabledRowPolicies & enabled)
 
     auto mixed_filters = boost::make_shared<MixedFiltersMap>();
 
-    /// retrieve aggregated policies from mixers
+    /// Retrieve aggregated policies from mixers
+    ///  if a table has a policy for this particular table, we have all needed information in table_mixers
+    ///    (policies for the database are already applied)
+    ///  otherwise we would look for a policy for database using RowPolicy::ANY_TABLE_MARK
+    /// Consider restrictive policies a=1 for db.t, b=2 for db.* and c=3 for db.*
+    ///   We are going to have two items in mixed_filters:
+    ///     1. a=1 AND b=2 AND c=3   for db.t (comes from table_mixers, where it had been created with the help of database_mixers)
+    ///     2. b=2 AND c=3  for db.* (comes directly from database_mixers)
     for (auto * mixer_map_ptr : {&table_mixers, &database_mixers})
     {
         for (auto & [key, mixer] : *mixer_map_ptr)
diff --git a/src/Access/RowPolicyCache.h b/src/Access/RowPolicyCache.h
index 7260de22164..df263416509 100644
--- a/src/Access/RowPolicyCache.h
+++ b/src/Access/RowPolicyCache.h
@@ -29,7 +29,7 @@ private:
         explicit PolicyInfo(const RowPolicyPtr & policy_) { setPolicy(policy_); }
         void setPolicy(const RowPolicyPtr & policy_);
 
-        bool isDatabase() const { return policy->isDatabase(); }
+        bool isForDatabase() const { return policy->isForDatabase(); }
         RowPolicyPtr policy;
         const RolesOrUsersSet * roles = nullptr;
         std::shared_ptr<const std::pair<String, String>> database_and_table_name;
diff --git a/tests/queries/0_stateless/02703_row_policy_database.sql b/tests/queries/0_stateless/02703_row_policy_for_database.sql
similarity index 100%
rename from tests/queries/0_stateless/02703_row_policy_database.sql
rename to tests/queries/0_stateless/02703_row_policy_for_database.sql

From d54a62e8e9ac1db9b0e37b75902e6762e520e3aa Mon Sep 17 00:00:00 2001
From: Ilya Golshtein <igolshtein@altinity.com>
Date: Wed, 3 May 2023 08:56:45 +0000
Subject: [PATCH 514/689] row_policy_template - forgotten reference test file

---
 ...database.reference => 02703_row_policy_for_database.reference} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tests/queries/0_stateless/{02703_row_policy_database.reference => 02703_row_policy_for_database.reference} (100%)

diff --git a/tests/queries/0_stateless/02703_row_policy_database.reference b/tests/queries/0_stateless/02703_row_policy_for_database.reference
similarity index 100%
rename from tests/queries/0_stateless/02703_row_policy_database.reference
rename to tests/queries/0_stateless/02703_row_policy_for_database.reference

From 9685aa0c9174d11905871aad2bd598e03d017bcd Mon Sep 17 00:00:00 2001
From: Ilya Golshtein <igolshtein@altinity.com>
Date: Thu, 4 May 2023 14:35:31 +0000
Subject: [PATCH 515/689] row_policy_template - minor changes, test renamed

---
 ... => 02703_row_policies_for_database_combination.reference} | 0
 ...on.sql => 02703_row_policies_for_database_combination.sql} | 0
 tests/queries/0_stateless/02703_row_policy_for_database.sql   | 4 ++--
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename tests/queries/0_stateless/{02703_row_policies_database_combination.reference => 02703_row_policies_for_database_combination.reference} (100%)
 rename tests/queries/0_stateless/{02703_row_policies_database_combination.sql => 02703_row_policies_for_database_combination.sql} (100%)

diff --git a/tests/queries/0_stateless/02703_row_policies_database_combination.reference b/tests/queries/0_stateless/02703_row_policies_for_database_combination.reference
similarity index 100%
rename from tests/queries/0_stateless/02703_row_policies_database_combination.reference
rename to tests/queries/0_stateless/02703_row_policies_for_database_combination.reference
diff --git a/tests/queries/0_stateless/02703_row_policies_database_combination.sql b/tests/queries/0_stateless/02703_row_policies_for_database_combination.sql
similarity index 100%
rename from tests/queries/0_stateless/02703_row_policies_database_combination.sql
rename to tests/queries/0_stateless/02703_row_policies_for_database_combination.sql
diff --git a/tests/queries/0_stateless/02703_row_policy_for_database.sql b/tests/queries/0_stateless/02703_row_policy_for_database.sql
index 85f5a44dfbf..03183a96b98 100644
--- a/tests/queries/0_stateless/02703_row_policy_for_database.sql
+++ b/tests/queries/0_stateless/02703_row_policy_for_database.sql
@@ -10,7 +10,7 @@ CREATE TABLE db1_02703.02703_rqtable (x UInt8) ENGINE = MergeTree ORDER BY x;
 INSERT INTO db1_02703.02703_rqtable VALUES (1), (2), (3), (4);
 
 
-SELECT '-- database level policies';
+SELECT '-- row policies for database';
 CREATE ROW POLICY db1_02703 ON db1_02703.* USING 1 AS PERMISSIVE TO ALL;
 CREATE ROW POLICY tbl1_02703 ON db1_02703.table USING 1 AS PERMISSIVE TO ALL;
 SELECT '  -- SHOW CREATE POLICY db1_02703 ON db1_02703.*';
@@ -44,7 +44,7 @@ DROP ROW POLICY 02703_filter_11_db ON *;
 DROP ROW POLICY 02703_filter_11 ON 02703_rqtable_default;
 
 USE db1_02703;
-SELECT '  -- CREATE DATABSE-LEVEL POLICY ON IN CURRENT DATABASE';
+SELECT '  -- CREATE DATABASE-LEVEL POLICY IN CURRENT DATABASE';
 CREATE ROW POLICY db2_02703 ON * TO u1_02703;
 SHOW CREATE POLICY db2_02703 ON *;
 

From 1027db6acaadbb20fcfedaac3416d98bd75b8e7d Mon Sep 17 00:00:00 2001
From: Ilya Golshtein <igolshtein@altinity.com>
Date: Fri, 5 May 2023 11:06:15 +0000
Subject: [PATCH 516/689] row_policy_template - minor change, reference test
 file fix

---
 .../0_stateless/02703_row_policy_for_database.reference     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/02703_row_policy_for_database.reference b/tests/queries/0_stateless/02703_row_policy_for_database.reference
index e318dfac429..5f0b486c1bd 100644
--- a/tests/queries/0_stateless/02703_row_policy_for_database.reference
+++ b/tests/queries/0_stateless/02703_row_policy_for_database.reference
@@ -1,4 +1,4 @@
--- database level policies
+-- row policies for database
   -- SHOW CREATE POLICY db1_02703 ON db1_02703.*
 CREATE ROW POLICY db1_02703 ON db1_02703.`*` FOR SELECT USING 1 TO ALL
   -- SHOW CREATE POLICY ON db1_02703.*
@@ -10,12 +10,12 @@ R1, R2: (x == 1) OR (x == 2)
 1
 2
 Check system.query_log
-SELECT \'-- database level policies\';	[]
+SELECT \'-- row policies for database\';	[]
 SELECT \'  -- SHOW CREATE POLICY db1_02703 ON db1_02703.*\';	[]
 SELECT \'  -- SHOW CREATE POLICY ON db1_02703.*\';	[]
 SELECT \'  -- SHOW CREATE POLICY ON db1_02703.`*`\';	[]
 SELECT \'R1, R2: (x == 1) OR (x == 2)\';	[]
 SELECT * FROM 02703_rqtable_default;	['`02703_filter_11_db` ON default.`*`','`02703_filter_11` ON default.`02703_rqtable_default`']
 SELECT \'Check system.query_log\';	[]
-  -- CREATE DATABSE-LEVEL POLICY ON IN CURRENT DATABASE
+  -- CREATE DATABASE-LEVEL POLICY IN CURRENT DATABASE
 CREATE ROW POLICY db2_02703 ON db1_02703.`*` TO u1_02703

From 9ef610040f691bc39ca711566de9feb2fbde44f1 Mon Sep 17 00:00:00 2001
From: Ilya Golshtein <igolshtein@altinity.com>
Date: Fri, 12 May 2023 22:30:00 +0000
Subject: [PATCH 517/689] row_policy_template - table `*` is respected

---
 docs/en/operations/system-tables/row_policies.md      |  2 +-
 src/Access/Common/RowPolicyDefs.cpp                   |  2 +-
 src/Access/Common/RowPolicyDefs.h                     |  2 ++
 src/Access/EnabledRowPolicies.cpp                     | 10 +++++++++-
 src/Access/RowPolicy.h                                |  4 +---
 src/Access/RowPolicyCache.cpp                         |  2 +-
 src/Parsers/Access/ASTRowPolicyName.cpp               | 11 ++++++++---
 src/Parsers/Access/ASTRowPolicyName.h                 |  3 +++
 src/Parsers/Access/ParserRowPolicyName.cpp            |  2 +-
 ...03_row_policies_for_database_combination.reference |  2 ++
 .../02703_row_policies_for_database_combination.sql   | 10 ++++++++++
 .../02703_row_policy_for_database.reference           |  9 ++++-----
 12 files changed, 43 insertions(+), 16 deletions(-)

diff --git a/docs/en/operations/system-tables/row_policies.md b/docs/en/operations/system-tables/row_policies.md
index 2c4d060ce66..e92ba1ece74 100644
--- a/docs/en/operations/system-tables/row_policies.md
+++ b/docs/en/operations/system-tables/row_policies.md
@@ -12,7 +12,7 @@ Columns:
 
 - `database` ([String](../../sql-reference/data-types/string.md)) — Database name.
 
-- `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
+- `table` ([String](../../sql-reference/data-types/string.md)) — Table name. Empty if policy for database.
 
 - `id` ([UUID](../../sql-reference/data-types/uuid.md)) — Row policy ID.
 
diff --git a/src/Access/Common/RowPolicyDefs.cpp b/src/Access/Common/RowPolicyDefs.cpp
index ba7856116f6..b1f882fe971 100644
--- a/src/Access/Common/RowPolicyDefs.cpp
+++ b/src/Access/Common/RowPolicyDefs.cpp
@@ -22,7 +22,7 @@ String RowPolicyName::toString() const
         name += backQuoteIfNeed(database);
         name += '.';
     }
-    name += backQuoteIfNeed(table_name);
+    name += (table_name == RowPolicyName::ANY_TABLE_MARK ? "*" : backQuoteIfNeed(table_name));
     return name;
 }
 
diff --git a/src/Access/Common/RowPolicyDefs.h b/src/Access/Common/RowPolicyDefs.h
index 792884c56df..7ffc99e1272 100644
--- a/src/Access/Common/RowPolicyDefs.h
+++ b/src/Access/Common/RowPolicyDefs.h
@@ -9,6 +9,8 @@ namespace DB
 /// Represents the full name of a row policy, e.g. "myfilter ON mydb.mytable".
 struct RowPolicyName
 {
+    static constexpr char ANY_TABLE_MARK[] = "";
+
     String short_name;
     String database;
     String table_name;
diff --git a/src/Access/EnabledRowPolicies.cpp b/src/Access/EnabledRowPolicies.cpp
index be78dd62146..601f004e3ea 100644
--- a/src/Access/EnabledRowPolicies.cpp
+++ b/src/Access/EnabledRowPolicies.cpp
@@ -36,11 +36,19 @@ RowPolicyFilterPtr EnabledRowPolicies::getFilter(const String & database, const
     auto it = loaded->find({database, table_name, filter_type});
     if (it == loaded->end())
     {   /// Look for a policy for database if a table policy not found
-        it = loaded->find({database, RowPolicy::ANY_TABLE_MARK, filter_type});
+        it = loaded->find({database, RowPolicyName::ANY_TABLE_MARK, filter_type});
         if (it == loaded->end())
         {
             return {};
         }
+        else
+        {
+            // deep copy found policy for database and change its table name to the actual one
+            auto policy_for_database = std::make_shared<RowPolicyFilter>(*it->second);
+            auto database_and_table_name = std::make_shared<std::pair<String, String>>(database, table_name);
+            policy_for_database->database_and_table_name = database_and_table_name;
+            return policy_for_database;
+        }
     }
 
     return it->second;
diff --git a/src/Access/RowPolicy.h b/src/Access/RowPolicy.h
index 348ebfa1637..9c190458620 100644
--- a/src/Access/RowPolicy.h
+++ b/src/Access/RowPolicy.h
@@ -14,8 +14,6 @@ namespace DB
   */
 struct RowPolicy : public IAccessEntity
 {
-    static constexpr char ANY_TABLE_MARK[] = "*";
-
     void setShortName(const String & short_name);
     void setDatabase(const String & database);
     void setTableName(const String & table_name);
@@ -38,7 +36,7 @@ struct RowPolicy : public IAccessEntity
     bool isPermissive() const { return !isRestrictive(); }
 
     /// Applied for entire database
-    bool isForDatabase() const { return full_name.table_name == ANY_TABLE_MARK; }
+    bool isForDatabase() const { return full_name.table_name == RowPolicyName::ANY_TABLE_MARK; }
 
     /// Sets that the policy is restrictive.
     /// A row is only accessible if at least one of the permissive policies passes,
diff --git a/src/Access/RowPolicyCache.cpp b/src/Access/RowPolicyCache.cpp
index 07670a8fe84..bb9da674477 100644
--- a/src/Access/RowPolicyCache.cpp
+++ b/src/Access/RowPolicyCache.cpp
@@ -270,7 +270,7 @@ void RowPolicyCache::mixFiltersFor(EnabledRowPolicies & enabled)
                     if (table_it == table_mixers.end())
                     {   /// no exact match - create new mixer
                         MixedFiltersKey database_key = key;
-                        database_key.table_name = RowPolicy::ANY_TABLE_MARK;
+                        database_key.table_name = RowPolicyName::ANY_TABLE_MARK;
 
                         auto database_it = database_mixers.find(database_key);
 
diff --git a/src/Parsers/Access/ASTRowPolicyName.cpp b/src/Parsers/Access/ASTRowPolicyName.cpp
index 4edfa61f10e..81a90de9d53 100644
--- a/src/Parsers/Access/ASTRowPolicyName.cpp
+++ b/src/Parsers/Access/ASTRowPolicyName.cpp
@@ -30,6 +30,11 @@ void ASTRowPolicyName::replaceEmptyDatabase(const String & current_database)
         full_name.database = current_database;
 }
 
+String ASTRowPolicyNames::tableOrAsterisk(const String & table_name) const
+{
+    return table_name == RowPolicyName::ANY_TABLE_MARK ? "*" : backQuoteIfNeed(table_name);
+}
+
 
 void ASTRowPolicyNames::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const
 {
@@ -73,7 +78,7 @@ void ASTRowPolicyNames::formatImpl(const FormatSettings & settings, FormatState
             const String & table_name = full_name.table_name;
             if (!database.empty())
                 settings.ostr << backQuoteIfNeed(database) + ".";
-            settings.ostr << backQuoteIfNeed(table_name);
+            settings.ostr << tableOrAsterisk(table_name);
         }
     }
     else if (same_db_and_table_name)
@@ -92,7 +97,7 @@ void ASTRowPolicyNames::formatImpl(const FormatSettings & settings, FormatState
         settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : "");
         if (!database.empty())
             settings.ostr << backQuoteIfNeed(database) + ".";
-        settings.ostr << backQuoteIfNeed(table_name);
+        settings.ostr << tableOrAsterisk(table_name);
     }
     else
     {
@@ -108,7 +113,7 @@ void ASTRowPolicyNames::formatImpl(const FormatSettings & settings, FormatState
                           << (settings.hilite ? hilite_none : "");
             if (!database.empty())
                 settings.ostr << backQuoteIfNeed(database) + ".";
-            settings.ostr << backQuoteIfNeed(table_name);
+            settings.ostr << tableOrAsterisk(table_name);
         }
     }
 
diff --git a/src/Parsers/Access/ASTRowPolicyName.h b/src/Parsers/Access/ASTRowPolicyName.h
index 9f4848bd612..86171475a0a 100644
--- a/src/Parsers/Access/ASTRowPolicyName.h
+++ b/src/Parsers/Access/ASTRowPolicyName.h
@@ -45,5 +45,8 @@ public:
     ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster<ASTRowPolicyNames>(clone()); }
 
     void replaceEmptyDatabase(const String & current_database);
+
+private:
+    String tableOrAsterisk(const String & table_name) const;
 };
 }
diff --git a/src/Parsers/Access/ParserRowPolicyName.cpp b/src/Parsers/Access/ParserRowPolicyName.cpp
index e5b4e01d5ac..efdff3c24bf 100644
--- a/src/Parsers/Access/ParserRowPolicyName.cpp
+++ b/src/Parsers/Access/ParserRowPolicyName.cpp
@@ -36,7 +36,7 @@ namespace
             }
             else if (is_any_table)
             {
-                res_table_name = "*"; // RowPolicy::ANY_TABLE_MARK
+                res_table_name = RowPolicyName::ANY_TABLE_MARK;
             }
 
             /// If table is specified without DB it cannot be followed by "ON"
diff --git a/tests/queries/0_stateless/02703_row_policies_for_database_combination.reference b/tests/queries/0_stateless/02703_row_policies_for_database_combination.reference
index 68ed02d1dc0..fa01904b846 100644
--- a/tests/queries/0_stateless/02703_row_policies_for_database_combination.reference
+++ b/tests/queries/0_stateless/02703_row_policies_for_database_combination.reference
@@ -40,3 +40,5 @@ None
 3	30
 4	40
 No problematic policy, select works
+Policy for table `*` does not affect other tables in the database
+other	100	20
diff --git a/tests/queries/0_stateless/02703_row_policies_for_database_combination.sql b/tests/queries/0_stateless/02703_row_policies_for_database_combination.sql
index f9b466f1ade..9941d69979d 100644
--- a/tests/queries/0_stateless/02703_row_policies_for_database_combination.sql
+++ b/tests/queries/0_stateless/02703_row_policies_for_database_combination.sql
@@ -86,3 +86,13 @@ DROP TABLE 02703_db.02703_rptable;
 DROP TABLE 02703_db.02703_rptable_another;
 DROP TABLE 02703_db.02703_unexpected_columns;
 DROP DATABASE 02703_db;
+
+SELECT 'Policy for table `*` does not affect other tables in the database';
+CREATE DATABASE 02703_db_asterisk;
+CREATE ROW POLICY 02703_asterisk ON 02703_db_asterisk.`*` USING x=1 AS permissive TO ALL;
+CREATE TABLE 02703_db_asterisk.`*` (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x AS SELECT 100, 20;
+CREATE TABLE 02703_db_asterisk.`other` (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x AS SELECT 100, 20;
+SELECT 'star', * FROM 02703_db_asterisk.`*`;
+SELECT 'other', * FROM 02703_db_asterisk.other;
+DROP ROW POLICY 02703_asterisk ON 02703_db_asterisk.`*`;
+DROP DATABASE 02703_db_asterisk;
diff --git a/tests/queries/0_stateless/02703_row_policy_for_database.reference b/tests/queries/0_stateless/02703_row_policy_for_database.reference
index 5f0b486c1bd..ec03e538d95 100644
--- a/tests/queries/0_stateless/02703_row_policy_for_database.reference
+++ b/tests/queries/0_stateless/02703_row_policy_for_database.reference
@@ -1,11 +1,10 @@
 -- row policies for database
   -- SHOW CREATE POLICY db1_02703 ON db1_02703.*
-CREATE ROW POLICY db1_02703 ON db1_02703.`*` FOR SELECT USING 1 TO ALL
+CREATE ROW POLICY db1_02703 ON db1_02703.* FOR SELECT USING 1 TO ALL
   -- SHOW CREATE POLICY ON db1_02703.*
-CREATE ROW POLICY db1_02703 ON db1_02703.`*` FOR SELECT USING 1 TO ALL
+CREATE ROW POLICY db1_02703 ON db1_02703.* FOR SELECT USING 1 TO ALL
 CREATE ROW POLICY tbl1_02703 ON db1_02703.table FOR SELECT USING 1 TO ALL
   -- SHOW CREATE POLICY ON db1_02703.`*`
-CREATE ROW POLICY db1_02703 ON db1_02703.`*` FOR SELECT USING 1 TO ALL
 R1, R2: (x == 1) OR (x == 2)
 1
 2
@@ -15,7 +14,7 @@ SELECT \'  -- SHOW CREATE POLICY db1_02703 ON db1_02703.*\';	[]
 SELECT \'  -- SHOW CREATE POLICY ON db1_02703.*\';	[]
 SELECT \'  -- SHOW CREATE POLICY ON db1_02703.`*`\';	[]
 SELECT \'R1, R2: (x == 1) OR (x == 2)\';	[]
-SELECT * FROM 02703_rqtable_default;	['`02703_filter_11_db` ON default.`*`','`02703_filter_11` ON default.`02703_rqtable_default`']
+SELECT * FROM 02703_rqtable_default;	['`02703_filter_11_db` ON default.*','`02703_filter_11` ON default.`02703_rqtable_default`']
 SELECT \'Check system.query_log\';	[]
   -- CREATE DATABASE-LEVEL POLICY IN CURRENT DATABASE
-CREATE ROW POLICY db2_02703 ON db1_02703.`*` TO u1_02703
+CREATE ROW POLICY db2_02703 ON db1_02703.* TO u1_02703

From ccd82842a0e197f3aa1dbfda2f2977382ad16345 Mon Sep 17 00:00:00 2001
From: Ilya Golshtein <igolshtein@altinity.com>
Date: Fri, 12 May 2023 22:32:53 +0000
Subject: [PATCH 518/689] row_policy_template  - rollback tweaking found policy
 for database

---
 src/Access/EnabledRowPolicies.cpp | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/Access/EnabledRowPolicies.cpp b/src/Access/EnabledRowPolicies.cpp
index 601f004e3ea..8ab1bf5928b 100644
--- a/src/Access/EnabledRowPolicies.cpp
+++ b/src/Access/EnabledRowPolicies.cpp
@@ -41,14 +41,6 @@ RowPolicyFilterPtr EnabledRowPolicies::getFilter(const String & database, const
         {
             return {};
         }
-        else
-        {
-            // deep copy found policy for database and change its table name to the actual one
-            auto policy_for_database = std::make_shared<RowPolicyFilter>(*it->second);
-            auto database_and_table_name = std::make_shared<std::pair<String, String>>(database, table_name);
-            policy_for_database->database_and_table_name = database_and_table_name;
-            return policy_for_database;
-        }
     }
 
     return it->second;

From ad8e114bcf411104e08ffe4c71752e08fa6d308a Mon Sep 17 00:00:00 2001
From: Ilya Golshtein <igolshtein@altinity.com>
Date: Sat, 13 May 2023 07:20:38 +0000
Subject: [PATCH 519/689] row_policy_template - row policies not in query log
 if analyzer used

---
 tests/broken_tests.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/broken_tests.txt b/tests/broken_tests.txt
index 3278f720daf..0b4efacba0b 100644
--- a/tests/broken_tests.txt
+++ b/tests/broken_tests.txt
@@ -138,3 +138,4 @@
 01600_parts_states_metrics_long
 01600_parts_types_metrics_long
 01287_max_execution_speed
+02703_row_policy_for_database

From 47e7e7e60a5ac177e51d32dd9cf2080dbab85f75 Mon Sep 17 00:00:00 2001
From: Ilya Golshtein <igolshtein@altinity.com>
Date: Sat, 13 May 2023 22:26:31 +0000
Subject: [PATCH 520/689] row_policy_template - test for asterisk table in a
 dedicated file

---
 .../02703_row_policies_for_asterisk.reference         |  2 ++
 .../0_stateless/02703_row_policies_for_asterisk.sql   | 11 +++++++++++
 ...03_row_policies_for_database_combination.reference |  2 --
 .../02703_row_policies_for_database_combination.sql   | 10 ----------
 4 files changed, 13 insertions(+), 12 deletions(-)
 create mode 100644 tests/queries/0_stateless/02703_row_policies_for_asterisk.reference
 create mode 100644 tests/queries/0_stateless/02703_row_policies_for_asterisk.sql

diff --git a/tests/queries/0_stateless/02703_row_policies_for_asterisk.reference b/tests/queries/0_stateless/02703_row_policies_for_asterisk.reference
new file mode 100644
index 00000000000..528bd7ef91e
--- /dev/null
+++ b/tests/queries/0_stateless/02703_row_policies_for_asterisk.reference
@@ -0,0 +1,2 @@
+Policy for table `*` does not affect other tables in the database
+other	100	20
diff --git a/tests/queries/0_stateless/02703_row_policies_for_asterisk.sql b/tests/queries/0_stateless/02703_row_policies_for_asterisk.sql
new file mode 100644
index 00000000000..96b1c01a6d6
--- /dev/null
+++ b/tests/queries/0_stateless/02703_row_policies_for_asterisk.sql
@@ -0,0 +1,11 @@
+-- Tags: no-parallel
+
+SELECT 'Policy for table `*` does not affect other tables in the database';
+CREATE DATABASE 02703_db_asterisk;
+CREATE ROW POLICY 02703_asterisk ON 02703_db_asterisk.`*` USING x=1 AS permissive TO ALL;
+CREATE TABLE 02703_db_asterisk.`*` (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x AS SELECT 100, 20;
+CREATE TABLE 02703_db_asterisk.`other` (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x AS SELECT 100, 20;
+SELECT 'star', * FROM 02703_db_asterisk.`*`;
+SELECT 'other', * FROM 02703_db_asterisk.other;
+DROP ROW POLICY 02703_asterisk ON 02703_db_asterisk.`*`;
+DROP DATABASE 02703_db_asterisk;
diff --git a/tests/queries/0_stateless/02703_row_policies_for_database_combination.reference b/tests/queries/0_stateless/02703_row_policies_for_database_combination.reference
index fa01904b846..68ed02d1dc0 100644
--- a/tests/queries/0_stateless/02703_row_policies_for_database_combination.reference
+++ b/tests/queries/0_stateless/02703_row_policies_for_database_combination.reference
@@ -40,5 +40,3 @@ None
 3	30
 4	40
 No problematic policy, select works
-Policy for table `*` does not affect other tables in the database
-other	100	20
diff --git a/tests/queries/0_stateless/02703_row_policies_for_database_combination.sql b/tests/queries/0_stateless/02703_row_policies_for_database_combination.sql
index 9941d69979d..f9b466f1ade 100644
--- a/tests/queries/0_stateless/02703_row_policies_for_database_combination.sql
+++ b/tests/queries/0_stateless/02703_row_policies_for_database_combination.sql
@@ -86,13 +86,3 @@ DROP TABLE 02703_db.02703_rptable;
 DROP TABLE 02703_db.02703_rptable_another;
 DROP TABLE 02703_db.02703_unexpected_columns;
 DROP DATABASE 02703_db;
-
-SELECT 'Policy for table `*` does not affect other tables in the database';
-CREATE DATABASE 02703_db_asterisk;
-CREATE ROW POLICY 02703_asterisk ON 02703_db_asterisk.`*` USING x=1 AS permissive TO ALL;
-CREATE TABLE 02703_db_asterisk.`*` (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x AS SELECT 100, 20;
-CREATE TABLE 02703_db_asterisk.`other` (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x AS SELECT 100, 20;
-SELECT 'star', * FROM 02703_db_asterisk.`*`;
-SELECT 'other', * FROM 02703_db_asterisk.other;
-DROP ROW POLICY 02703_asterisk ON 02703_db_asterisk.`*`;
-DROP DATABASE 02703_db_asterisk;

From b4b5b4dcc569771b31430122a4b8671f673f70af Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 14 May 2023 02:52:23 +0200
Subject: [PATCH 521/689] Amend

---
 src/Storages/MergeTree/MergeTreeData.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 47d2bc67ddd..b9f4ba29c55 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1958,7 +1958,7 @@ try
         parts_futures.push_back(runner([&, my_part = part]()
         {
             auto res = loadDataPartWithRetries(
-            my_part->info, my_part->name, part->disk,
+            my_part->info, my_part->name, my_part->disk,
             DataPartState::Outdated, data_parts_mutex, loading_parts_initial_backoff_ms,
             loading_parts_max_backoff_ms, loading_parts_max_tries);
 

From c87b33a24dde639144648ac9c648844ef4dc1c07 Mon Sep 17 00:00:00 2001
From: Manas Alekar <manas@clickhouse.com>
Date: Sat, 13 May 2023 22:55:27 -0700
Subject: [PATCH 522/689] Fix error on OS X regarding resetting timeouts.

This happens when remote disconnects due to inactivity. It seems
to work on Linux, likely due to difference in SO_LINGER, maybe a
different default timeout on Darwin.

Verified manually using clickhouse cloud using following process:

1. Connect to instance.
2. Run `show tables`.
3. Wait 6 minutes.
4. Run `show tables`.

With this fix, the EINVAL is not reported, and client will simply
reconnect.
---
 src/IO/TimeoutSetter.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/IO/TimeoutSetter.cpp b/src/IO/TimeoutSetter.cpp
index 1beb26772e5..ed21383ccd4 100644
--- a/src/IO/TimeoutSetter.cpp
+++ b/src/IO/TimeoutSetter.cpp
@@ -31,13 +31,15 @@ TimeoutSetter::~TimeoutSetter()
 {
     try
     {
+        bool connected = socket.impl()->initialized();
+        if (!connected)
+            return;
+
         socket.setSendTimeout(old_send_timeout);
         socket.setReceiveTimeout(old_receive_timeout);
     }
     catch (...)
     {
-        /// Sometimes caught on Mac OS X. This message can be safely ignored.
-        /// If you are developer using Mac, please debug this error message by yourself.
         tryLogCurrentException("Client", "TimeoutSetter: Can't reset timeouts");
     }
 }

From 1db35384d969f9239de84563927494148100c6d5 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 15 May 2023 03:30:03 +0200
Subject: [PATCH 523/689] Support `bitCount` for big integers

---
 src/Functions/bitCount.cpp                    |  9 ++++++++-
 .../02736_bit_count_big_int.reference         | 13 +++++++++++++
 .../0_stateless/02736_bit_count_big_int.sql   | 19 +++++++++++++++++++
 3 files changed, 40 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/02736_bit_count_big_int.reference
 create mode 100644 tests/queries/0_stateless/02736_bit_count_big_int.sql

diff --git a/src/Functions/bitCount.cpp b/src/Functions/bitCount.cpp
index 984f33b7001..566a11481be 100644
--- a/src/Functions/bitCount.cpp
+++ b/src/Functions/bitCount.cpp
@@ -9,7 +9,7 @@ namespace DB
 template <typename A>
 struct BitCountImpl
 {
-    using ResultType = UInt8;
+    using ResultType = std::conditional_t<(sizeof(A) * 8 >= 256), UInt16, UInt8>;
     static constexpr bool allow_string_or_fixed_string = true;
 
     static inline ResultType apply(A a)
@@ -17,6 +17,13 @@ struct BitCountImpl
         /// We count bits in the value representation in memory. For example, we support floats.
         /// We need to avoid sign-extension when converting signed numbers to larger type. So, uint8_t(-1) has 8 bits.
 
+        if constexpr (is_big_int_v<A>)
+        {
+            ResultType res = 0;
+            for (auto item : a.items)
+                res += __builtin_popcountll(item);
+            return res;
+        }
         if constexpr (std::is_same_v<A, UInt64> || std::is_same_v<A, Int64>)
             return __builtin_popcountll(a);
         if constexpr (std::is_same_v<A, UInt32> || std::is_same_v<A, Int32> || std::is_unsigned_v<A>)
diff --git a/tests/queries/0_stateless/02736_bit_count_big_int.reference b/tests/queries/0_stateless/02736_bit_count_big_int.reference
new file mode 100644
index 00000000000..a3a725ace69
--- /dev/null
+++ b/tests/queries/0_stateless/02736_bit_count_big_int.reference
@@ -0,0 +1,13 @@
+128
+256
+128
+256
+127
+255
+126
+255
+64
+UInt8
+UInt16
+UInt8
+UInt16
diff --git a/tests/queries/0_stateless/02736_bit_count_big_int.sql b/tests/queries/0_stateless/02736_bit_count_big_int.sql
new file mode 100644
index 00000000000..35a4a641606
--- /dev/null
+++ b/tests/queries/0_stateless/02736_bit_count_big_int.sql
@@ -0,0 +1,19 @@
+SELECT bitCount(CAST(-1 AS UInt128));
+SELECT bitCount(CAST(-1 AS UInt256));
+
+SELECT bitCount(CAST(-1 AS Int128));
+SELECT bitCount(CAST(-1 AS Int256));
+
+SELECT bitCount(CAST(-1 AS UInt128) - 1);
+SELECT bitCount(CAST(-1 AS UInt256) - 2);
+
+SELECT bitCount(CAST(-1 AS Int128) - 3);
+SELECT bitCount(CAST(-1 AS Int256) - 4);
+
+SELECT bitCount(CAST(0xFFFFFFFFFFFFFFFF AS Int256));
+
+SELECT toTypeName(bitCount(1::UInt128));
+SELECT toTypeName(bitCount(1::UInt256));
+
+SELECT toTypeName(bitCount(1::Int128));
+SELECT toTypeName(bitCount(1::Int256));

From 4764259f609a024f04ec8c082b63a495efc66d70 Mon Sep 17 00:00:00 2001
From: Amos Bird <amosbird@gmail.com>
Date: Mon, 15 May 2023 14:17:16 +0800
Subject: [PATCH 524/689] Fix a bug with projections and the
 aggregate_functions_null_for_empty setting (for
 query_plan_optimize_projection)

Fix a bug with projections and the aggregate_functions_null_for_empty
setting. This was already fixed in PR #42198 but got forgotten after
using query_plan_optimize_projection.
---
 .../QueryPlan/Optimizations/projectionsCommon.cpp         | 4 ++++
 ...rojection_aggregate_functions_null_for_empty.reference | 1 +
 ...1710_projection_aggregate_functions_null_for_empty.sql | 8 ++++++++
 3 files changed, 13 insertions(+)
 create mode 100644 tests/queries/0_stateless/01710_projection_aggregate_functions_null_for_empty.reference
 create mode 100644 tests/queries/0_stateless/01710_projection_aggregate_functions_null_for_empty.sql

diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
index a334450fb41..9252137f649 100644
--- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
+++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
@@ -42,6 +42,10 @@ bool canUseProjectionForReadingStep(ReadFromMergeTree * reading)
     if (reading->getContext()->getSettingsRef().allow_experimental_query_deduplication)
         return false;
 
+    // Currently projection don't support settings which implicitly modify aggregate functions.
+    if (reading->getContext()->getSettingsRef().aggregate_functions_null_for_empty)
+        return false;
+
     return true;
 }
 
diff --git a/tests/queries/0_stateless/01710_projection_aggregate_functions_null_for_empty.reference b/tests/queries/0_stateless/01710_projection_aggregate_functions_null_for_empty.reference
new file mode 100644
index 00000000000..f2a527c4d8d
--- /dev/null
+++ b/tests/queries/0_stateless/01710_projection_aggregate_functions_null_for_empty.reference
@@ -0,0 +1 @@
+1554690688
diff --git a/tests/queries/0_stateless/01710_projection_aggregate_functions_null_for_empty.sql b/tests/queries/0_stateless/01710_projection_aggregate_functions_null_for_empty.sql
new file mode 100644
index 00000000000..a77720b6580
--- /dev/null
+++ b/tests/queries/0_stateless/01710_projection_aggregate_functions_null_for_empty.sql
@@ -0,0 +1,8 @@
+DROP TABLE IF EXISTS t1;
+
+CREATE TABLE t1 (c0 Int32, PRIMARY KEY (c0)) ENGINE=MergeTree;
+INSERT INTO t1 VALUES (1554690688);
+
+SELECT MIN(t1.c0) FROM t1 SETTINGS aggregate_functions_null_for_empty = 1;
+
+DROP TABLE IF EXISTS t1;

From 8d1bcb5c2f23ea0e29c61c8cb3f584586f5f8834 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Mon, 15 May 2023 16:28:28 +0800
Subject: [PATCH 525/689] fix typo

---
 src/Storages/StorageMongoDB.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/StorageMongoDB.h b/src/Storages/StorageMongoDB.h
index 682a027440d..2b77f076e7e 100644
--- a/src/Storages/StorageMongoDB.h
+++ b/src/Storages/StorageMongoDB.h
@@ -7,7 +7,7 @@
 namespace DB
 {
 /* Implements storage in the MongoDB database.
- * Use ENGINE = mysql(host_port, database_name, table_name, user_name, password)
+ * Use ENGINE = MongoDB(host:port, database, collection, user, password [, options]);
  * Read only.
  */
 

From 665545ec45d4fc0e15e602dcbcc990621d904623 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 09:35:04 +0200
Subject: [PATCH 526/689] Fix "reference to local binding" after fixes for
 clang-17

Follow-up for: #49851 (cc @alexey-milovidov)
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Coordination/KeeperStorage.cpp | 34 +++++++++++++++---------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp
index 58fccc83e6a..7a1a5e42632 100644
--- a/src/Coordination/KeeperStorage.cpp
+++ b/src/Coordination/KeeperStorage.cpp
@@ -339,37 +339,37 @@ void KeeperStorage::UncommittedState::applyDelta(const Delta & delta)
             nodes.emplace(delta.path, UncommittedNode{.node = nullptr});
     }
 
-    auto & [node, acls, last_applied_zxid] = nodes.at(delta.path);
-
     std::visit(
-        [&, &my_node = node, &my_acls = acls, &my_last_applied_zxid = last_applied_zxid]<typename DeltaType>(const DeltaType & operation)
+        [&]<typename DeltaType>(const DeltaType & operation)
         {
+            auto & [node, acls, last_applied_zxid] = nodes.at(delta.path);
+
             if constexpr (std::same_as<DeltaType, CreateNodeDelta>)
             {
-                assert(!my_node);
-                my_node = std::make_shared<Node>();
-                my_node->stat = operation.stat;
-                my_node->setData(operation.data);
-                my_acls = operation.acls;
-                my_last_applied_zxid = delta.zxid;
+                assert(!node);
+                node = std::make_shared<Node>();
+                node->stat = operation.stat;
+                node->setData(operation.data);
+                acls = operation.acls;
+                last_applied_zxid = delta.zxid;
             }
             else if constexpr (std::same_as<DeltaType, RemoveNodeDelta>)
             {
-                assert(my_node);
-                my_node = nullptr;
-                my_last_applied_zxid = delta.zxid;
+                assert(node);
+                node = nullptr;
+                last_applied_zxid = delta.zxid;
             }
             else if constexpr (std::same_as<DeltaType, UpdateNodeDelta>)
             {
-                assert(my_node);
-                my_node->invalidateDigestCache();
+                assert(node);
+                node->invalidateDigestCache();
                 operation.update_fn(*node);
-                my_last_applied_zxid = delta.zxid;
+                last_applied_zxid = delta.zxid;
             }
             else if constexpr (std::same_as<DeltaType, SetACLDelta>)
             {
-                my_acls = operation.acls;
-                my_last_applied_zxid = delta.zxid;
+                acls = operation.acls;
+                last_applied_zxid = delta.zxid;
             }
         },
         delta.operation);

From b06e34a77fb597b53637ec9b37e0bbebbfd98a1f Mon Sep 17 00:00:00 2001
From: Arthur Passos <arthur.ti@outlook.com>
Date: Wed, 10 May 2023 20:54:59 -0300
Subject: [PATCH 527/689] Accept key value delimiter as part of value

---
 .../keyvaluepair/impl/NeedleFactory.h         | 22 ++++++++++++++++++-
 .../keyvaluepair/impl/StateHandlerImpl.h      | 18 +++++----------
 ...t_key_value_pairs_multiple_input.reference | 16 ++++++++++++--
 ...extract_key_value_pairs_multiple_input.sql | 14 +++++++++++-
 4 files changed, 54 insertions(+), 16 deletions(-)

diff --git a/src/Functions/keyvaluepair/impl/NeedleFactory.h b/src/Functions/keyvaluepair/impl/NeedleFactory.h
index 007f970aedf..b5f0645abcc 100644
--- a/src/Functions/keyvaluepair/impl/NeedleFactory.h
+++ b/src/Functions/keyvaluepair/impl/NeedleFactory.h
@@ -38,7 +38,7 @@ public:
         return SearchSymbols {std::string{needles.data(), needles.size()}};
     }
 
-    SearchSymbols getReadNeedles(const Configuration & extractor_configuration)
+    SearchSymbols getReadKeyNeedles(const Configuration & extractor_configuration)
     {
         const auto & [key_value_delimiter, quoting_character, pair_delimiters]
             = extractor_configuration;
@@ -57,6 +57,26 @@ public:
 
         return SearchSymbols {std::string{needles.data(), needles.size()}};
     }
+
+    SearchSymbols getReadValueNeedles(const Configuration & extractor_configuration)
+    {
+        const auto & [key_value_delimiter, quoting_character, pair_delimiters]
+            = extractor_configuration;
+
+        std::vector<char> needles;
+
+        needles.push_back(quoting_character);
+
+        std::copy(pair_delimiters.begin(), pair_delimiters.end(), std::back_inserter(needles));
+
+        if constexpr (WITH_ESCAPING)
+        {
+            needles.push_back('\\');
+        }
+
+        return SearchSymbols {std::string{needles.data(), needles.size()}};
+    }
+
     SearchSymbols getReadQuotedNeedles(const Configuration & extractor_configuration)
     {
         const auto quoting_character = extractor_configuration.quoting_character;
diff --git a/src/Functions/keyvaluepair/impl/StateHandlerImpl.h b/src/Functions/keyvaluepair/impl/StateHandlerImpl.h
index 217a42b84ad..7fc3ba54833 100644
--- a/src/Functions/keyvaluepair/impl/StateHandlerImpl.h
+++ b/src/Functions/keyvaluepair/impl/StateHandlerImpl.h
@@ -41,7 +41,8 @@ public:
         NeedleFactory<WITH_ESCAPING> needle_factory;
 
         wait_needles = needle_factory.getWaitNeedles(configuration);
-        read_needles = needle_factory.getReadNeedles(configuration);
+        read_key_needles = needle_factory.getReadKeyNeedles(configuration);
+        read_value_needles = needle_factory.getReadValueNeedles(configuration);
         read_quoted_needles = needle_factory.getReadQuotedNeedles(configuration);
     }
 
@@ -77,7 +78,7 @@ public:
 
         size_t pos = 0;
 
-        while (const auto * p = find_first_symbols_or_null({file.begin() + pos, file.end()}, read_needles))
+        while (const auto * p = find_first_symbols_or_null({file.begin() + pos, file.end()}, read_key_needles))
         {
             auto character_position = p - file.begin();
             size_t next_pos = character_position + 1u;
@@ -191,10 +192,6 @@ public:
             {
                 return {pos + 1u, State::READING_QUOTED_VALUE};
             }
-            else if (isKeyValueDelimiter(current_character))
-            {
-                return {pos, State::WAITING_KEY};
-            }
 
             if constexpr (WITH_ESCAPING)
             {
@@ -218,7 +215,7 @@ public:
 
         size_t pos = 0;
 
-        while (const auto * p = find_first_symbols_or_null({file.begin() + pos, file.end()}, read_needles))
+        while (const auto * p = find_first_symbols_or_null({file.begin() + pos, file.end()}, read_value_needles))
         {
             const size_t character_position = p - file.begin();
             size_t next_pos = character_position + 1u;
@@ -237,10 +234,6 @@ public:
                     }
                 }
             }
-            else if (isKeyValueDelimiter(*p))
-            {
-                return {next_pos, State::WAITING_KEY};
-            }
             else if (isPairDelimiter(*p))
             {
                 value.append(file.begin() + pos, file.begin() + character_position);
@@ -300,7 +293,8 @@ public:
 
 private:
     SearchSymbols wait_needles;
-    SearchSymbols read_needles;
+    SearchSymbols read_key_needles;
+    SearchSymbols read_value_needles;
     SearchSymbols read_quoted_needles;
 
     /*
diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
index 125afa19427..e3bd1d7fc45 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
@@ -123,7 +123,7 @@ SELECT
 -- semi-colon as pair delimiter
 -- expected output: {'age':'31','name':'neymar','team':'psg'}
 WITH
-    extractKeyValuePairs('name:neymar;age:31;team:psg;invalid1:invalid1,invalid2:invalid2', ':', ';') AS s_map,
+    extractKeyValuePairs('name:neymar;age:31;team:psg;random_key:value_with_comma,still_part_of_value:still_part_of_value', ':', ';') AS s_map,
     CAST(
         arrayMap(
             (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
@@ -132,7 +132,7 @@ WITH
     ) AS x
 SELECT
     x;
-{'age':'31','name':'neymar','team':'psg'}
+{'age':'31','name':'neymar','random_key':'value_with_comma,still_part_of_value:still_part_of_value','team':'psg'}
 -- both comma and semi-colon as pair delimiters
 -- expected output: {'age':'31','last_key':'last_value','name':'neymar','nationality':'brazil','team':'psg'}
 WITH
@@ -304,6 +304,18 @@ WITH
 SELECT
     x;
 {}
+-- key value delimiter should be considered valid part of value, char2 has a problem with ==
+WITH
+    extractKeyValuePairs('formula=1+2=3 argument1=1 argument2=2 result=3, char="=" char2== string="foo=bar"', '=') AS s_map,
+    CAST(
+            arrayMap(
+                    (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
+                ),
+            'Map(String,String)'
+        ) AS x
+SELECT
+    x;
+{'argument1':'1','argument2':'2','char':'=','char2':'=','formula':'1+2=3','result':'3','string':'foo=bar'}
 -- check str_to_map alias (it is case-insensitive)
 WITH
     sTr_tO_mAp('name:neymar, age:31 team:psg,nationality:brazil') AS s_map,
diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
index 0a1500b1796..578a30560c8 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
@@ -123,7 +123,7 @@ SELECT
 -- semi-colon as pair delimiter
 -- expected output: {'age':'31','name':'neymar','team':'psg'}
 WITH
-    extractKeyValuePairs('name:neymar;age:31;team:psg;invalid1:invalid1,invalid2:invalid2', ':', ';') AS s_map,
+    extractKeyValuePairs('name:neymar;age:31;team:psg;random_key:value_with_comma,still_part_of_value:still_part_of_value', ':', ';') AS s_map,
     CAST(
         arrayMap(
             (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
@@ -427,6 +427,18 @@ WITH
 SELECT
     x;
 
+-- key value delimiter should be considered valid part of value, char2 has a problem with ==
+WITH
+    extractKeyValuePairs('formula=1+2=3 argument1=1 argument2=2 result=3, char="=" char2== string="foo=bar"', '=') AS s_map,
+    CAST(
+            arrayMap(
+                    (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
+                ),
+            'Map(String,String)'
+        ) AS x
+SELECT
+    x;
+
 -- check str_to_map alias (it is case-insensitive)
 WITH
     sTr_tO_mAp('name:neymar, age:31 team:psg,nationality:brazil') AS s_map,

From fff0cf28b70054ca2ceb1e1f0bedd57fdf90a4f2 Mon Sep 17 00:00:00 2001
From: Arthur Passos <arthur.ti@outlook.com>
Date: Thu, 11 May 2023 09:30:09 -0300
Subject: [PATCH 528/689] address pr comments

---
 .../02499_extract_key_value_pairs_multiple_input.reference  | 6 +++---
 .../02499_extract_key_value_pairs_multiple_input.sql        | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
index e3bd1d7fc45..ccccec10667 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
@@ -121,9 +121,9 @@ SELECT
     x;
 {}
 -- semi-colon as pair delimiter
--- expected output: {'age':'31','name':'neymar','team':'psg'}
+-- expected output: {'age':'31','anotherkey':'anothervalue','name':'neymar','random_key':'value_with_comma,still_part_of_value:still_part_of_value','team':'psg'}
 WITH
-    extractKeyValuePairs('name:neymar;age:31;team:psg;random_key:value_with_comma,still_part_of_value:still_part_of_value', ':', ';') AS s_map,
+    extractKeyValuePairs('name:neymar;age:31;team:psg;random_key:value_with_comma,still_part_of_value:still_part_of_value;anotherkey:anothervalue', ':', ';') AS s_map,
     CAST(
         arrayMap(
             (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))
@@ -132,7 +132,7 @@ WITH
     ) AS x
 SELECT
     x;
-{'age':'31','name':'neymar','random_key':'value_with_comma,still_part_of_value:still_part_of_value','team':'psg'}
+{'age':'31','anotherkey':'anothervalue','name':'neymar','random_key':'value_with_comma,still_part_of_value:still_part_of_value','team':'psg'}
 -- both comma and semi-colon as pair delimiters
 -- expected output: {'age':'31','last_key':'last_value','name':'neymar','nationality':'brazil','team':'psg'}
 WITH
diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
index 578a30560c8..58975bdfeec 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
@@ -121,9 +121,9 @@ SELECT
     x;
 
 -- semi-colon as pair delimiter
--- expected output: {'age':'31','name':'neymar','team':'psg'}
+-- expected output: {'age':'31','anotherkey':'anothervalue','name':'neymar','random_key':'value_with_comma,still_part_of_value:still_part_of_value','team':'psg'}
 WITH
-    extractKeyValuePairs('name:neymar;age:31;team:psg;random_key:value_with_comma,still_part_of_value:still_part_of_value', ':', ';') AS s_map,
+    extractKeyValuePairs('name:neymar;age:31;team:psg;random_key:value_with_comma,still_part_of_value:still_part_of_value;anotherkey:anothervalue', ':', ';') AS s_map,
     CAST(
         arrayMap(
             (x) -> (x, s_map[x]), arraySort(mapKeys(s_map))

From c16cba48b1d3895aa29114309679c15aed9ca60e Mon Sep 17 00:00:00 2001
From: Arthur Passos <arthur.ti@outlook.com>
Date: Thu, 11 May 2023 09:32:26 -0300
Subject: [PATCH 529/689] remove old / debug comment in tests

---
 .../02499_extract_key_value_pairs_multiple_input.reference      | 2 +-
 .../02499_extract_key_value_pairs_multiple_input.sql            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
index ccccec10667..a56e0881471 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference
@@ -304,7 +304,7 @@ WITH
 SELECT
     x;
 {}
--- key value delimiter should be considered valid part of value, char2 has a problem with ==
+-- key value delimiter should be considered valid part of value
 WITH
     extractKeyValuePairs('formula=1+2=3 argument1=1 argument2=2 result=3, char="=" char2== string="foo=bar"', '=') AS s_map,
     CAST(
diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
index 58975bdfeec..5a1727f91e2 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
@@ -427,7 +427,7 @@ WITH
 SELECT
     x;
 
--- key value delimiter should be considered valid part of value, char2 has a problem with ==
+-- key value delimiter should be considered valid part of value
 WITH
     extractKeyValuePairs('formula=1+2=3 argument1=1 argument2=2 result=3, char="=" char2== string="foo=bar"', '=') AS s_map,
     CAST(

From 424a20ee1cb53a0c64d13c6ed08d1ce54872ddb5 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 14:17:32 +0200
Subject: [PATCH 530/689] Log with warning if the server was terminated
 forcefully

In case of it is terminated forcefully it will not be terminated
gracefully (i.e. run dtors and stuff), and by using warning log level
those messages will go to clickhouse-server.err.log, in which messages
are kept for a longer period then in clickhouse-server.log (at least
because it contains only warnings, errors and fatals only).

This will help with investigating some obscure issues.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 programs/server/Server.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index b5e44d90129..632f3f3a02d 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -1872,7 +1872,7 @@ try
             }
 
             if (current_connections)
-                LOG_INFO(log, "Closed all listening sockets. Waiting for {} outstanding connections.", current_connections);
+                LOG_WARNING(log, "Closed all listening sockets. Waiting for {} outstanding connections.", current_connections);
             else
                 LOG_INFO(log, "Closed all listening sockets.");
 
@@ -1884,7 +1884,7 @@ try
                 current_connections = waitServersToFinish(servers, config().getInt("shutdown_wait_unfinished", 5));
 
             if (current_connections)
-                LOG_INFO(log, "Closed connections. But {} remain."
+                LOG_WARNING(log, "Closed connections. But {} remain."
                     " Tip: To increase wait time add to config: <shutdown_wait_unfinished>60</shutdown_wait_unfinished>", current_connections);
             else
                 LOG_INFO(log, "Closed connections.");
@@ -1900,7 +1900,7 @@ try
 
                 /// Dump coverage here, because std::atexit callback would not be called.
                 dumpCoverageReportIfPossible();
-                LOG_INFO(log, "Will shutdown forcefully.");
+                LOG_WARNING(log, "Will shutdown forcefully.");
                 safeExit(0);
             }
         });

From e8f971aa2bdfcdfb12e0091a8aa6a7eb4e96ff8c Mon Sep 17 00:00:00 2001
From: Arthur Passos <arthur.ti@outlook.com>
Date: Mon, 15 May 2023 09:25:10 -0300
Subject: [PATCH 531/689] use LIMIT_EXCEEDED instead of TOO_LARGE_MAP_SIZE

---
 src/Common/ErrorCodes.cpp                                     | 1 -
 src/Functions/keyvaluepair/impl/CHKeyValuePairExtractor.h     | 4 ++--
 .../02499_extract_key_value_pairs_multiple_input.sql          | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp
index 382b8ed8019..e9dc5649245 100644
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@@ -576,7 +576,6 @@
     M(691, UNKNOWN_ELEMENT_OF_ENUM) \
     M(692, TOO_MANY_MUTATIONS) \
     M(693, AWS_ERROR) \
-    M(694, TOO_LARGE_MAP_SIZE) \
     \
     M(999, KEEPER_EXCEPTION) \
     M(1000, POCO_EXCEPTION) \
diff --git a/src/Functions/keyvaluepair/impl/CHKeyValuePairExtractor.h b/src/Functions/keyvaluepair/impl/CHKeyValuePairExtractor.h
index 24c8d3cd89a..3895cf3e77d 100644
--- a/src/Functions/keyvaluepair/impl/CHKeyValuePairExtractor.h
+++ b/src/Functions/keyvaluepair/impl/CHKeyValuePairExtractor.h
@@ -13,7 +13,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
-    extern const int TOO_LARGE_MAP_SIZE;
+    extern const int LIMIT_EXCEEDED;
 }
 
 /*
@@ -118,7 +118,7 @@ private:
 
         if (row_offset > max_number_of_pairs)
         {
-            throw Exception(ErrorCodes::TOO_LARGE_MAP_SIZE, "Number of pairs produced exceeded the limit of {}", max_number_of_pairs);
+            throw Exception(ErrorCodes::LIMIT_EXCEEDED, "Number of pairs produced exceeded the limit of {}", max_number_of_pairs);
         }
 
         key.commit();
diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
index 8c5f0365cc7..38d09338774 100644
--- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
+++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql
@@ -425,7 +425,7 @@ WITH
             'Map(String,String)'
         ) AS x
 SELECT
-    x; -- {serverError TOO_LARGE_MAP_SIZE}
+    x; -- {serverError LIMIT_EXCEEDED}
 
 -- { echoOn }
 

From dccdb3e6786d10803968504d1179804789edaf0a Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Mon, 15 May 2023 14:41:51 +0200
Subject: [PATCH 532/689] work with comments on PR

---
 ...oryStream.cpp => StdIStreamFromMemory.cpp} | 20 ++++-------
 ...3MemoryStream.h => StdIStreamFromMemory.h} | 17 +++++-----
 src/IO/WriteBufferFromS3.cpp                  | 24 ++++++-------
 src/IO/WriteBufferFromS3.h                    | 14 +++++++-
 ...riteBufferFromS3BufferAllocationPolicy.cpp | 34 +++++++++++--------
 .../WriteBufferFromS3BufferAllocationPolicy.h | 26 --------------
 src/IO/WriteBufferFromS3TaskTracker.cpp       | 10 +++---
 src/IO/WriteBufferFromS3TaskTracker.h         | 10 ++++--
 8 files changed, 72 insertions(+), 83 deletions(-)
 rename src/IO/{WriteBufferFromS3MemoryStream.cpp => StdIStreamFromMemory.cpp} (67%)
 rename src/IO/{WriteBufferFromS3MemoryStream.h => StdIStreamFromMemory.h} (60%)
 delete mode 100644 src/IO/WriteBufferFromS3BufferAllocationPolicy.h

diff --git a/src/IO/WriteBufferFromS3MemoryStream.cpp b/src/IO/StdIStreamFromMemory.cpp
similarity index 67%
rename from src/IO/WriteBufferFromS3MemoryStream.cpp
rename to src/IO/StdIStreamFromMemory.cpp
index 6271f15f055..3242a7e6383 100644
--- a/src/IO/WriteBufferFromS3MemoryStream.cpp
+++ b/src/IO/StdIStreamFromMemory.cpp
@@ -1,27 +1,24 @@
-#include "config.h"
-
-#if USE_AWS_S3
-
-#include <IO/WriteBufferFromS3MemoryStream.h>
+#include <IO/StdIStreamFromMemory.h>
 
 namespace DB
 {
 
-MemoryStream::MemoryBuf::MemoryBuf(char * begin_, size_t size_)
+StdIStreamFromMemory::MemoryBuf::MemoryBuf(char * begin_, size_t size_)
     : begin(begin_)
     , size(size_)
 {
     this->setg(begin, begin, begin + size);
 }
 
-MemoryStream::MemoryBuf::int_type MemoryStream::MemoryBuf::underflow()
+StdIStreamFromMemory::MemoryBuf::int_type StdIStreamFromMemory::MemoryBuf::underflow()
 {
     if (gptr() < egptr())
             return traits_type::to_int_type(*gptr());
     return traits_type::eof();
 }
 
-MemoryStream::MemoryBuf::pos_type MemoryStream::MemoryBuf::seekoff(off_type off, std::ios_base::seekdir way,
+StdIStreamFromMemory::MemoryBuf::pos_type
+StdIStreamFromMemory::MemoryBuf::seekoff(off_type off, std::ios_base::seekdir way,
                  std::ios_base::openmode mode)
 {
     bool out_mode = (std::ios_base::out & mode) != 0;
@@ -49,13 +46,13 @@ MemoryStream::MemoryBuf::pos_type MemoryStream::MemoryBuf::seekoff(off_type off,
     return pos_type(ret);
 }
 
-MemoryStream::MemoryBuf::pos_type MemoryStream::MemoryBuf::seekpos(pos_type sp,
+StdIStreamFromMemory::MemoryBuf::pos_type StdIStreamFromMemory::MemoryBuf::seekpos(pos_type sp,
                  std::ios_base::openmode mode)
 {
     return seekoff(off_type(sp), std::ios_base::beg, mode);
 }
 
-MemoryStream::MemoryStream(char * begin_, size_t size_)
+StdIStreamFromMemory::StdIStreamFromMemory(char * begin_, size_t size_)
     : std::iostream(nullptr)
     , mem_buf(begin_, size_)
 {
@@ -63,6 +60,3 @@ MemoryStream::MemoryStream(char * begin_, size_t size_)
 }
 
 }
-
-#endif
-
diff --git a/src/IO/WriteBufferFromS3MemoryStream.h b/src/IO/StdIStreamFromMemory.h
similarity index 60%
rename from src/IO/WriteBufferFromS3MemoryStream.h
rename to src/IO/StdIStreamFromMemory.h
index e9606798910..64b147fd296 100644
--- a/src/IO/WriteBufferFromS3MemoryStream.h
+++ b/src/IO/StdIStreamFromMemory.h
@@ -1,15 +1,15 @@
 #pragma once
 
-#include "config.h"
-
-#if USE_AWS_S3
-
 #include <iostream>
 
 namespace DB
 {
 
-struct MemoryStream: std::iostream
+/// StdIStreamFromMemory is used in WriteBufferFromS3 as a stream which is passed to the S3::Client
+/// It provides istream interface (only reading) over the memory.
+/// However S3::Client requires iostream interface it only reads from the stream
+
+class StdIStreamFromMemory : public std::iostream
 {
     struct MemoryBuf: std::streambuf
     {
@@ -27,11 +27,10 @@ struct MemoryStream: std::iostream
             size_t size = 0;
     };
 
-    MemoryStream(char * begin_, size_t size_);
-
     MemoryBuf mem_buf;
+
+public:
+    StdIStreamFromMemory(char * begin_, size_t size_);
 };
 
 }
-
-#endif
diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp
index 73d78cb13be..01ab8ff7cbb 100644
--- a/src/IO/WriteBufferFromS3.cpp
+++ b/src/IO/WriteBufferFromS3.cpp
@@ -2,8 +2,8 @@
 
 #if USE_AWS_S3
 
+#include "StdIStreamFromMemory.h"
 #include "WriteBufferFromS3.h"
-#include "WriteBufferFromS3MemoryStream.h"
 #include "WriteBufferFromS3TaskTracker.h"
 
 #include <Common/logger_useful.h>
@@ -63,7 +63,7 @@ struct WriteBufferFromS3::PartData
 
     std::shared_ptr<std::iostream> createAwsBuffer()
     {
-        auto buffer = std::make_shared<MemoryStream>(memory.data(), data_size);
+        auto buffer = std::make_shared<StdIStreamFromMemory>(memory.data(), data_size);
         buffer->exceptions(std::ios::badbit);
         return buffer;
     }
@@ -108,7 +108,7 @@ void WriteBufferFromS3::nextImpl()
                 "Cannot write to prefinalized buffer for S3, the file could have been created with PutObjectRequest");
 
     /// Make sense to call to before adding new async task to check if there is an exception
-    task_tracker->getReady();
+    task_tracker->waitReady();
 
     hidePartialData();
 
@@ -132,7 +132,7 @@ void WriteBufferFromS3::preFinalize()
 
     LOG_TRACE(log, "preFinalize WriteBufferFromS3. {}", getLogDetails());
 
-    task_tracker->getReady();
+    task_tracker->waitReady();
 
     hidePartialData();
 
@@ -178,7 +178,7 @@ void WriteBufferFromS3::finalizeImpl()
     chassert(offset() == 0);
     chassert(hidden_size == 0);
 
-    task_tracker->getAll();
+    task_tracker->waitAll();
 
     if (!multipart_upload_id.empty())
     {
@@ -266,10 +266,10 @@ void WriteBufferFromS3::reallocateFirstBuffer()
 {
     chassert(offset() == 0);
 
-    if (buffer_allocation_policy->getNumber() > 1 || available() > 0)
+    if (buffer_allocation_policy->getBufferNumber() > 1 || available() > 0)
         return;
 
-    const size_t max_first_buffer = buffer_allocation_policy->getSize();
+    const size_t max_first_buffer = buffer_allocation_policy->getBufferSize();
     if (memory.size() == max_first_buffer)
         return;
 
@@ -299,7 +299,7 @@ void WriteBufferFromS3::detachBuffer()
 
 void WriteBufferFromS3::allocateFirstBuffer()
 {
-    const auto max_first_buffer = buffer_allocation_policy->getSize();
+    const auto max_first_buffer = buffer_allocation_policy->getBufferSize();
     const auto size = std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), max_first_buffer);
     memory = Memory(size);
     WriteBuffer::set(memory.data(), memory.size());
@@ -309,16 +309,16 @@ void WriteBufferFromS3::allocateFirstBuffer()
 
 void WriteBufferFromS3::allocateBuffer()
 {
-    buffer_allocation_policy->next();
+    buffer_allocation_policy->nextBuffer();
     chassert(0 == hidden_size);
 
-    if (buffer_allocation_policy->getNumber() == 1)
+    if (buffer_allocation_policy->getBufferNumber() == 1)
         return allocateFirstBuffer();
 
-    memory = Memory(buffer_allocation_policy->getSize());
+    memory = Memory(buffer_allocation_policy->getBufferSize());
     WriteBuffer::set(memory.data(), memory.size());
 
-    LOG_TRACE(log, "Allocated buffer with size {}. {}", buffer_allocation_policy->getSize(), getLogDetails());
+    LOG_TRACE(log, "Allocated buffer with size {}. {}", buffer_allocation_policy->getBufferSize(), getLogDetails());
 }
 
 void WriteBufferFromS3::setFakeBufferWhenPreFinalized()
diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h
index e65127872fa..ac6c430606f 100644
--- a/src/IO/WriteBufferFromS3.h
+++ b/src/IO/WriteBufferFromS3.h
@@ -8,7 +8,6 @@
 #include <IO/BufferWithOwnMemory.h>
 #include <IO/WriteBuffer.h>
 #include <IO/WriteSettings.h>
-#include <IO/WriteBufferFromS3BufferAllocationPolicy.h>
 #include <Storages/StorageS3Settings.h>
 #include <Interpreters/threadPoolCallbackRunner.h>
 
@@ -41,6 +40,19 @@ public:
     void nextImpl() override;
     void preFinalize() override;
 
+public:
+    class IBufferAllocationPolicy
+    {
+    public:
+        virtual size_t getBufferNumber() const = 0;
+        virtual size_t getBufferSize() const = 0;
+        virtual void nextBuffer() = 0;
+        virtual ~IBufferAllocationPolicy() = 0;
+    };
+    using IBufferAllocationPolicyPtr = std::unique_ptr<IBufferAllocationPolicy>;
+
+    static IBufferAllocationPolicyPtr ChooseBufferPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings_);
+
 private:
     /// Receives response from the server after sending all data.
     void finalizeImpl() override;
diff --git a/src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp b/src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp
index 0eec6b0d034..6347c1acfd7 100644
--- a/src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp
+++ b/src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp
@@ -2,38 +2,41 @@
 
 #if USE_AWS_S3
 
-#include <IO/WriteBufferFromS3BufferAllocationPolicy.h>
+#include <IO/WriteBufferFromS3.h>
+
+#include <memory>
 
 namespace
 {
 
-struct FixedSizeBufferAllocationPolicy : DB::IBufferAllocationPolicy
+class FixedSizeBufferAllocationPolicy : public DB::WriteBufferFromS3::IBufferAllocationPolicy
 {
-    const size_t size = 0;
+    const size_t buffer_size = 0;
     size_t buffer_number = 0;
 
+public:
     explicit FixedSizeBufferAllocationPolicy(const DB::S3Settings::RequestSettings::PartUploadSettings & settings_)
-        : size(settings_.strict_upload_part_size)
+        : buffer_size(settings_.strict_upload_part_size)
     {
-        chassert(size > 0);
+        chassert(buffer_size > 0);
     }
 
-    size_t getNumber() const override { return buffer_number; }
+    size_t getBufferNumber() const override { return buffer_number; }
 
-    size_t getSize() const override
+    size_t getBufferSize() const override
     {
         chassert(buffer_number > 0);
-        return size;
+        return buffer_size;
     }
 
-    void next() override
+    void nextBuffer() override
     {
         ++buffer_number;
     }
 };
 
 
-struct ExpBufferAllocationPolicy : DB::IBufferAllocationPolicy
+class ExpBufferAllocationPolicy : public DB::WriteBufferFromS3::IBufferAllocationPolicy
 {
     const size_t first_size = 0;
     const size_t second_size = 0;
@@ -45,6 +48,7 @@ struct ExpBufferAllocationPolicy : DB::IBufferAllocationPolicy
     size_t current_size = 0;
     size_t buffer_number = 0;
 
+public:
     explicit ExpBufferAllocationPolicy(const DB::S3Settings::RequestSettings::PartUploadSettings & settings_)
         : first_size(std::max(settings_.max_single_part_upload_size, settings_.min_upload_part_size))
         , second_size(settings_.min_upload_part_size)
@@ -59,15 +63,15 @@ struct ExpBufferAllocationPolicy : DB::IBufferAllocationPolicy
         chassert(max_size > 0);
     }
 
-    size_t getNumber() const override { return buffer_number; }
+    size_t getBufferNumber() const override { return buffer_number; }
 
-    size_t getSize() const override
+    size_t getBufferSize() const override
     {
         chassert(buffer_number > 0);
         return current_size;
     }
 
-    void next() override
+    void nextBuffer() override
     {
         ++buffer_number;
 
@@ -93,9 +97,9 @@ struct ExpBufferAllocationPolicy : DB::IBufferAllocationPolicy
 namespace DB
 {
 
-IBufferAllocationPolicy::~IBufferAllocationPolicy() = default;
+WriteBufferFromS3::IBufferAllocationPolicy::~IBufferAllocationPolicy() = default;
 
-IBufferAllocationPolicyPtr ChooseBufferPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings_)
+WriteBufferFromS3::IBufferAllocationPolicyPtr WriteBufferFromS3::ChooseBufferPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings_)
 {
     if (settings_.strict_upload_part_size > 0)
         return std::make_unique<FixedSizeBufferAllocationPolicy>(settings_);
diff --git a/src/IO/WriteBufferFromS3BufferAllocationPolicy.h b/src/IO/WriteBufferFromS3BufferAllocationPolicy.h
deleted file mode 100644
index 1ee7c982ed2..00000000000
--- a/src/IO/WriteBufferFromS3BufferAllocationPolicy.h
+++ /dev/null
@@ -1,26 +0,0 @@
-#pragma once
-
-#include "config.h"
-
-#if USE_AWS_S3
-
-#include <Storages/StorageS3Settings.h>
-
-namespace DB
-{
-
-struct IBufferAllocationPolicy
-{
-    virtual size_t getNumber() const = 0;
-    virtual size_t getSize() const = 0;
-    virtual void next() = 0;
-    virtual ~IBufferAllocationPolicy() = 0;
-};
-
-using IBufferAllocationPolicyPtr = std::unique_ptr<IBufferAllocationPolicy>;
-
-IBufferAllocationPolicyPtr ChooseBufferPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings_);
-
-}
-
-#endif
diff --git a/src/IO/WriteBufferFromS3TaskTracker.cpp b/src/IO/WriteBufferFromS3TaskTracker.cpp
index 0769f7731c2..7826747c0a4 100644
--- a/src/IO/WriteBufferFromS3TaskTracker.cpp
+++ b/src/IO/WriteBufferFromS3TaskTracker.cpp
@@ -28,9 +28,9 @@ ThreadPoolCallbackRunner<void> WriteBufferFromS3::TaskTracker::syncRunner()
     };
 }
 
-void WriteBufferFromS3::TaskTracker::getReady()
+void WriteBufferFromS3::TaskTracker::waitReady()
 {
-    LOG_TEST(log, "getReady, in queue {}", futures.size());
+    LOG_TEST(log, "waitReady, in queue {}", futures.size());
 
     /// Exceptions are propagated
     auto it = futures.begin();
@@ -55,12 +55,12 @@ void WriteBufferFromS3::TaskTracker::getReady()
         it = futures.erase(it);
     }
 
-    LOG_TEST(log, "getReady ended, in queue {}", futures.size());
+    LOG_TEST(log, "waitReady ended, in queue {}", futures.size());
 }
 
-void WriteBufferFromS3::TaskTracker::getAll()
+void WriteBufferFromS3::TaskTracker::waitAll()
 {
-    LOG_TEST(log, "getAll, in queue {}", futures.size());
+    LOG_TEST(log, "waitAll, in queue {}", futures.size());
 
     /// Exceptions are propagated
     for (auto & future : futures)
diff --git a/src/IO/WriteBufferFromS3TaskTracker.h b/src/IO/WriteBufferFromS3TaskTracker.h
index fa214a4f8c5..c978b9a78f0 100644
--- a/src/IO/WriteBufferFromS3TaskTracker.h
+++ b/src/IO/WriteBufferFromS3TaskTracker.h
@@ -9,6 +9,12 @@
 namespace DB
 {
 
+/// That class is used only in WriteBufferFromS3 for now.
+/// Therefore it declared as a part of  WriteBufferFromS3.
+/// TaskTracker takes a Callback which is run by scheduler in some external shared ThreadPool.
+/// TaskTracker brings the methods waitReady, waitAll/safeWaitAll
+/// to help with coordination of the running tasks.
+
 class WriteBufferFromS3::TaskTracker
 {
 public:
@@ -20,8 +26,8 @@ public:
     static ThreadPoolCallbackRunner<void> syncRunner();
 
     bool isAsync() const;
-    void getReady();
-    void getAll();
+    void waitReady();
+    void waitAll();
     void safeWaitAll();
     void add(Callback && func);
 

From 418a61a68c2e8089b5a7372b08e398757eb9522e Mon Sep 17 00:00:00 2001
From: AVMusorin <aleksandr.musorin@semrush.com>
Date: Fri, 12 May 2023 12:47:14 +0200
Subject: [PATCH 533/689] Allow using Alias column type for KafkaEngine

```
create table kafka
(
 a UInt32,
 a_str String Alias toString(a)
) engine = Kafka;

create table data
(
  a UInt32;
  a_str String
) engine = MergeTree
order by tuple();

create materialized view data_mv to data
(
  a UInt32,
  a_str String
) as
select a, a_str from kafka;
```
Alias type works as expected in comparison with MATERIALIZED/EPHEMERAL
or column with default expression.

Ref: https://github.com/ClickHouse/ClickHouse/pull/47138

Co-authored-by: Azat Khuzhin <a3at.mail@gmail.com>
---
 .../table-engines/integrations/kafka.md       |  4 +-
 src/Storages/ColumnsDescription.cpp           |  9 -----
 src/Storages/ColumnsDescription.h             |  1 -
 src/Storages/Kafka/StorageKafka.cpp           | 14 ++++++-
 tests/integration/test_storage_kafka/test.py  | 38 +++++++++++++++----
 5 files changed, 44 insertions(+), 22 deletions(-)

diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md
index ab69e4e90ce..ccfca4c1f1f 100644
--- a/docs/en/engines/table-engines/integrations/kafka.md
+++ b/docs/en/engines/table-engines/integrations/kafka.md
@@ -19,8 +19,8 @@ Kafka lets you:
 ``` sql
 CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
 (
-    name1 [type1],
-    name2 [type2],
+    name1 [type1] [ALIAS expr1],
+    name2 [type2] [ALIAS expr2],
     ...
 ) ENGINE = Kafka()
 SETTINGS
diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp
index 21b140bd73a..8eabae7929c 100644
--- a/src/Storages/ColumnsDescription.cpp
+++ b/src/Storages/ColumnsDescription.cpp
@@ -383,15 +383,6 @@ NamesAndTypesList ColumnsDescription::getEphemeral() const
     return ret;
 }
 
-NamesAndTypesList ColumnsDescription::getWithDefaultExpression() const
-{
-    NamesAndTypesList ret;
-    for (const auto & col : columns)
-        if (col.default_desc.expression)
-            ret.emplace_back(col.name, col.type);
-    return ret;
-}
-
 NamesAndTypesList ColumnsDescription::getAll() const
 {
     NamesAndTypesList ret;
diff --git a/src/Storages/ColumnsDescription.h b/src/Storages/ColumnsDescription.h
index e5ec867cd64..365a999673e 100644
--- a/src/Storages/ColumnsDescription.h
+++ b/src/Storages/ColumnsDescription.h
@@ -132,7 +132,6 @@ public:
     NamesAndTypesList getInsertable() const; /// ordinary + ephemeral
     NamesAndTypesList getAliases() const;
     NamesAndTypesList getEphemeral() const;
-    NamesAndTypesList getWithDefaultExpression() const; // columns with default expression, for example set by `CREATE TABLE` statement
     NamesAndTypesList getAllPhysical() const; /// ordinary + materialized.
     NamesAndTypesList getAll() const; /// ordinary + materialized + aliases + ephemeral
     /// Returns .size0/.null/...
diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp
index 3381561eb1b..7d504833a0a 100644
--- a/src/Storages/Kafka/StorageKafka.cpp
+++ b/src/Storages/Kafka/StorageKafka.cpp
@@ -41,6 +41,7 @@
 #include <Common/setThreadName.h>
 #include <Formats/FormatFactory.h>
 
+#include "Storages/ColumnDefault.h"
 #include "config_version.h"
 
 #include <Common/CurrentMetrics.h>
@@ -966,9 +967,18 @@ void registerStorageKafka(StorageFactory & factory)
         {
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "kafka_poll_max_batch_size can not be lower than 1");
         }
-        if (args.columns.getOrdinary() != args.columns.getAll() || !args.columns.getWithDefaultExpression().empty())
+        NamesAndTypesList supported_columns;
+        for (const auto & column : args.columns)
         {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "KafkaEngine doesn't support DEFAULT/MATERIALIZED/EPHEMERAL/ALIAS expressions for columns. "
+            if (column.default_desc.kind == ColumnDefaultKind::Alias)
+                supported_columns.emplace_back(column.name, column.type);
+            if (column.default_desc.kind == ColumnDefaultKind::Default && !column.default_desc.expression)
+                supported_columns.emplace_back(column.name, column.type);
+        }
+        // Kafka engine allows only ordinary columns without default expression or alias columns.
+        if (args.columns.getAll() != supported_columns)
+        {
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "KafkaEngine doesn't support DEFAULT/MATERIALIZED/EPHEMERAL expressions for columns. "
                                                        "See https://clickhouse.com/docs/en/engines/table-engines/integrations/kafka/#configuration");
         }
 
diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py
index 3a4fa6c6bfe..9a6d3e0513c 100644
--- a/tests/integration/test_storage_kafka/test.py
+++ b/tests/integration/test_storage_kafka/test.py
@@ -285,11 +285,11 @@ def avro_confluent_message(schema_registry_client, value):
 # Tests
 
 
-def test_kafka_prohibited_column_types(kafka_cluster):
+def test_kafka_column_types(kafka_cluster):
     def assert_returned_exception(e):
         assert e.value.returncode == 36
         assert (
-            "KafkaEngine doesn't support DEFAULT/MATERIALIZED/EPHEMERAL/ALIAS expressions for columns."
+            "KafkaEngine doesn't support DEFAULT/MATERIALIZED/EPHEMERAL expressions for columns."
             in str(e.value)
         )
 
@@ -314,17 +314,39 @@ def test_kafka_prohibited_column_types(kafka_cluster):
     assert_returned_exception(exception)
 
     # check ALIAS
-    with pytest.raises(QueryRuntimeException) as exception:
-        instance.query(
-            """
+    instance.query(
+        """
                 CREATE TABLE test.kafka (a Int, b String Alias toString(a))
                 ENGINE = Kafka('{kafka_broker}:19092', '{kafka_topic_new}', '{kafka_group_name_new}', '{kafka_format_json_each_row}', '\\n')
+                SETTINGS kafka_commit_on_select = 1;
                 """
-        )
-    assert_returned_exception(exception)
+    )
+    messages = []
+    for i in range(5):
+        messages.append(json.dumps({"a": i}))
+    kafka_produce(kafka_cluster, "new", messages)
+    result = ""
+    expected = TSV(
+        """
+0\t0
+1\t1
+2\t2
+3\t3
+4\t4
+                              """
+    )
+    retries = 50
+    while retries > 0:
+        result += instance.query("SELECT a, b FROM test.kafka", ignore_error=True)
+        if TSV(result) == expected:
+            break
+        retries -= 1
+
+    assert TSV(result) == expected
+
+    instance.query("DROP TABLE test.kafka SYNC")
 
     # check MATERIALIZED
-    # check ALIAS
     with pytest.raises(QueryRuntimeException) as exception:
         instance.query(
             """

From bf201a09b7e68fde4d84f87388adeb7047416fb0 Mon Sep 17 00:00:00 2001
From: Diego Nieto <lesandie@gmail.com>
Date: Mon, 15 May 2023 15:43:39 +0200
Subject: [PATCH 534/689] Update docs/en/operations/storing-data.md

@kssenii comments

Co-authored-by: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
---
 docs/en/operations/storing-data.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md
index e8b043e7a27..5804ad8545b 100644
--- a/docs/en/operations/storing-data.md
+++ b/docs/en/operations/storing-data.md
@@ -194,7 +194,7 @@ These settings should be defined in the disk configuration section.
 
 File Cache **query/profile settings**:
 
-Some of these settings will disable cache features per query/profile that are enabled by default. For example, you can enable cache in disk configuration and disable it per query/profile setting `enable_filesystem_cache` to `false`. Also setting `cache_on_write_operations` to `true` in disk configuration means that general file and per query cache are enabled. But if you need to disable this general setting per specific queries then setting `enable_filesystem_cache_on_write_operations` to `false` means that write operations cache will be disabled for a specific query/profile.
+Some of these settings will disable cache features per query/profile that are enabled by default or in disk configuration settings. For example, you can enable cache in disk configuration and disable it per query/profile setting `enable_filesystem_cache` to `false`. Also setting `cache_on_write_operations` to `true` in disk configuration means that "write-though" cache is enabled. But if you need to disable this general setting per specific queries then setting `enable_filesystem_cache_on_write_operations` to `false` means that write operations cache will be disabled for a specific query/profile.
 
 - `enable_filesystem_cache` - allows to disable cache per query even if storage policy was configured with `cache` disk type. Default: `true`.
 

From f2a023140e6134aca52fa1bd45e3a616f790be70 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 15:38:50 +0200
Subject: [PATCH 535/689] Fix processing pending batch for Distributed async
 INSERT after restart

After abnormal server restart current_batch.txt (that contains list of
files to send to the remote shard), may not have all files, if it was
terminated between unlink .bin files and truncation of current_batch.txt

But it should be fixed in a more reliable way, though to backport the
patch I kept it simple.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Storages/Distributed/DistributedAsyncInsertBatch.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp
index ba5938e057d..b82cf1d7ffc 100644
--- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp
+++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp
@@ -201,6 +201,14 @@ void DistributedAsyncInsertBatch::sendBatch()
     {
         for (const auto & file : files)
         {
+            /// In case of recovery it is possible that some of files will be
+            /// missing, if server had been restarted abnormally
+            if (recovered && !fs::exists(file))
+            {
+                LOG_WARNING(parent.log, "File {} does not exists, likely due abnormal shutdown", file);
+                continue;
+            }
+
             ReadBufferFromFile in(file);
             const auto & distributed_header = DistributedAsyncInsertHeader::read(in, parent.log);
 

From b23afdc53390149236d02140dfc96775317a2e6c Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 15 May 2023 15:48:00 +0000
Subject: [PATCH 536/689] Fix build for aarch64-darwin

---
 contrib/boost-cmake/CMakeLists.txt            |  18 +-
 src/Common/AsyncTaskExecutor.h                |  44 +-
 src/Common/OpenTelemetryTraceContext.cpp      |  45 +-
 .../FunctionGenerateRandomStructure.cpp       | 457 ++++++++++++++++++
 .../FunctionGenerateRandomStructure.h         |  45 ++
 5 files changed, 561 insertions(+), 48 deletions(-)
 create mode 100644 src/Functions/FunctionGenerateRandomStructure.cpp
 create mode 100644 src/Functions/FunctionGenerateRandomStructure.h

diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt
index c9a759eab9c..6f9dce0b042 100644
--- a/contrib/boost-cmake/CMakeLists.txt
+++ b/contrib/boost-cmake/CMakeLists.txt
@@ -103,11 +103,19 @@ set (SRCS_CONTEXT
 )
 
 if (ARCH_AARCH64)
-    set (SRCS_CONTEXT ${SRCS_CONTEXT}
-        "${LIBRARY_DIR}/libs/context/src/asm/jump_arm64_aapcs_elf_gas.S"
-        "${LIBRARY_DIR}/libs/context/src/asm/make_arm64_aapcs_elf_gas.S"
-        "${LIBRARY_DIR}/libs/context/src/asm/ontop_arm64_aapcs_elf_gas.S"
-    )
+    if (OS_DARWIN)
+        set (SRCS_CONTEXT ${SRCS_CONTEXT}
+            "${LIBRARY_DIR}/libs/context/src/asm/jump_arm64_aapcs_macho_gas.S"
+            "${LIBRARY_DIR}/libs/context/src/asm/make_arm64_aapcs_macho_gas.S"
+            "${LIBRARY_DIR}/libs/context/src/asm/ontop_arm64_aapcs_macho_gas.S"
+        )
+    else()
+        set (SRCS_CONTEXT ${SRCS_CONTEXT}
+            "${LIBRARY_DIR}/libs/context/src/asm/jump_arm64_aapcs_elf_gas.S"
+            "${LIBRARY_DIR}/libs/context/src/asm/make_arm64_aapcs_elf_gas.S"
+            "${LIBRARY_DIR}/libs/context/src/asm/ontop_arm64_aapcs_elf_gas.S"
+        )
+    endif()
 elseif (ARCH_PPC64LE)
     set (SRCS_CONTEXT ${SRCS_CONTEXT}
         "${LIBRARY_DIR}/libs/context/src/asm/jump_ppc64_sysv_elf_gas.S"
diff --git a/src/Common/AsyncTaskExecutor.h b/src/Common/AsyncTaskExecutor.h
index 1c2f758504a..f87abd7eb9b 100644
--- a/src/Common/AsyncTaskExecutor.h
+++ b/src/Common/AsyncTaskExecutor.h
@@ -79,8 +79,8 @@ public:
         ERROR = 4,
     };
 #endif
-    static FiberInfo getCurrentFiberInfo();
 
+    static FiberInfo getCurrentFiberInfo();
 protected:
     /// Method that is called in resume() before actual fiber resuming.
     /// If it returns false, resume() will return immediately without actual fiber resuming.
@@ -124,6 +124,48 @@ private:
     std::unique_ptr<AsyncTask> task;
 };
 
+/// Simple implementation for fiber local variable.
+template <typename T>
+struct FiberLocal
+{
+public:
+    FiberLocal()
+    {
+        /// Initialize main instance for this thread.
+        /// Contexts for fibers will inherit this instance
+        /// (it could be changed before creating fibers).
+        data[nullptr] = T();
+    }
+
+    T & operator*()
+    {
+        return get();
+    }
+
+    T * operator->()
+    {
+        return &get();
+    }
+
+private:
+    T & get()
+    {
+        /// Get instance for current fiber.
+        return getInstanceForFiber(AsyncTaskExecutor::getCurrentFiberInfo());
+    }
+
+    T & getInstanceForFiber(FiberInfo info)
+    {
+        auto it = data.find(info.fiber);
+        /// If it's the first request, we need to initialize instance for the fiber using instance from parent fiber.
+        if (it == data.end())
+            it = data.insert({info.fiber, getInstanceForFiber(*info.parent_fiber_info)}).first;
+        return it->second;
+    }
+
+    std::unordered_map<const Fiber *, T> data;
+};
+
 String getSocketTimeoutExceededMessageByTimeoutType(AsyncEventTimeoutType type, Poco::Timespan timeout, const String & socket_description);
 
 }
diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp
index 8cf4879c1e2..40d06e71456 100644
--- a/src/Common/OpenTelemetryTraceContext.cpp
+++ b/src/Common/OpenTelemetryTraceContext.cpp
@@ -14,48 +14,9 @@ namespace DB
 namespace OpenTelemetry
 {
 
-/// This code can be executed inside several fibers in one thread,
-/// we should use fiber local tracing context.
-struct FiberLocalTracingContextOnThread
-{
-public:
-    FiberLocalTracingContextOnThread()
-    {
-        /// Initialize main context for this thread.
-        /// Contexts for fibers will inherit this main context.
-        data[nullptr] = TracingContextOnThread();
-    }
-
-    TracingContextOnThread & operator*()
-    {
-        return get();
-    }
-
-    TracingContextOnThread * operator->()
-    {
-        return &get();
-    }
-
-private:
-    TracingContextOnThread & get()
-    {
-        /// Get context for current fiber.
-        return getContextForFiber(AsyncTaskExecutor::getCurrentFiberInfo());
-    }
-
-    TracingContextOnThread & getContextForFiber(FiberInfo info)
-    {
-        auto it = data.find(info.fiber);
-        /// If it's the first request, we need to initialize context for the fiber using context from parent fiber.
-        if (it == data.end())
-            it = data.insert({info.fiber, getContextForFiber(*info.parent_fiber_info)}).first;
-        return it->second;
-    }
-
-    std::unordered_map<const Fiber *, TracingContextOnThread> data;
-};
-
-thread_local FiberLocalTracingContextOnThread current_fiber_trace_context;
+///// This code can be executed inside several fibers in one thread,
+///// we should use fiber local tracing context.
+thread_local FiberLocal<TracingContextOnThread> current_fiber_trace_context;
 
 bool Span::addAttribute(std::string_view name, UInt64 value) noexcept
 {
diff --git a/src/Functions/FunctionGenerateRandomStructure.cpp b/src/Functions/FunctionGenerateRandomStructure.cpp
new file mode 100644
index 00000000000..023a73fe147
--- /dev/null
+++ b/src/Functions/FunctionGenerateRandomStructure.cpp
@@ -0,0 +1,457 @@
+#include <Columns/ColumnString.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeFixedString.h>
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionHelpers.h>
+#include <Functions/IFunction.h>
+#include <Interpreters/Context.h>
+#include <Common/randomSeed.h>
+
+#include <pcg_random.hpp>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int BAD_ARGUMENTS;
+}
+
+class FunctionGenerateRandomStructure : public IFunction
+{
+private:
+    static constexpr std::array<TypeIndex, 29> simple_types
+    {
+        TypeIndex::Int8,
+        TypeIndex::UInt8,
+        TypeIndex::Int16,
+        TypeIndex::UInt16,
+        TypeIndex::Int32,
+        TypeIndex::UInt32,
+        TypeIndex::Int64,
+        TypeIndex::UInt64,
+        TypeIndex::Int128,
+        TypeIndex::UInt128,
+        TypeIndex::Int256,
+        TypeIndex::UInt256,
+        TypeIndex::Float32,
+        TypeIndex::Float64,
+        TypeIndex::Decimal32,
+        TypeIndex::Decimal64,
+        TypeIndex::Decimal128,
+        TypeIndex::Decimal256,
+        TypeIndex::Date,
+        TypeIndex::Date32,
+        TypeIndex::DateTime,
+        TypeIndex::DateTime64,
+        TypeIndex::String,
+        TypeIndex::FixedString,
+        TypeIndex::Enum8,
+        TypeIndex::Enum16,
+        TypeIndex::IPv4,
+        TypeIndex::IPv6,
+        TypeIndex::UUID,
+    };
+
+    static constexpr std::array<TypeIndex, 5> complex_types
+    {
+        TypeIndex::Nullable,
+        TypeIndex::LowCardinality,
+        TypeIndex::Array,
+        TypeIndex::Tuple,
+        TypeIndex::Map,
+    };
+
+    static constexpr std::array<TypeIndex, 22> map_key_types
+    {
+        TypeIndex::Int8,
+        TypeIndex::UInt8,
+        TypeIndex::Int16,
+        TypeIndex::UInt16,
+        TypeIndex::Int32,
+        TypeIndex::UInt32,
+        TypeIndex::Int64,
+        TypeIndex::UInt64,
+        TypeIndex::Int128,
+        TypeIndex::UInt128,
+        TypeIndex::Int256,
+        TypeIndex::UInt256,
+        TypeIndex::Date,
+        TypeIndex::Date32,
+        TypeIndex::DateTime,
+        TypeIndex::String,
+        TypeIndex::FixedString,
+        TypeIndex::IPv4,
+        TypeIndex::Enum8,
+        TypeIndex::Enum16,
+        TypeIndex::UUID,
+        TypeIndex::LowCardinality,
+    };
+
+    static constexpr std::array<TypeIndex, 22> suspicious_lc_types
+    {
+        TypeIndex::Int8,
+        TypeIndex::UInt8,
+        TypeIndex::Int16,
+        TypeIndex::UInt16,
+        TypeIndex::Int32,
+        TypeIndex::UInt32,
+        TypeIndex::Int64,
+        TypeIndex::UInt64,
+        TypeIndex::Int128,
+        TypeIndex::UInt128,
+        TypeIndex::Int256,
+        TypeIndex::UInt256,
+        TypeIndex::Float32,
+        TypeIndex::Float64,
+        TypeIndex::Date,
+        TypeIndex::Date32,
+        TypeIndex::DateTime,
+        TypeIndex::String,
+        TypeIndex::FixedString,
+        TypeIndex::IPv4,
+        TypeIndex::IPv6,
+        TypeIndex::UUID,
+    };
+
+    static constexpr size_t MAX_NUMBER_OF_COLUMNS = 128;
+    static constexpr size_t MAX_TUPLE_ELEMENTS = 16;
+    static constexpr size_t MAX_DATETIME64_PRECISION = 9;
+    static constexpr size_t MAX_DECIMAL32_PRECISION = 9;
+    static constexpr size_t MAX_DECIMAL64_PRECISION = 18;
+    static constexpr size_t MAX_DECIMAL128_PRECISION = 38;
+    static constexpr size_t MAX_DECIMAL256_PRECISION = 76;
+    static constexpr size_t MAX_DEPTH = 32;
+
+public:
+    static constexpr auto name = "generateRandomStructure";
+
+    explicit FunctionGenerateRandomStructure(bool allow_suspicious_lc_types_) : allow_suspicious_lc_types(allow_suspicious_lc_types_)
+    {
+    }
+
+    static FunctionPtr create(ContextPtr context)
+    {
+        return std::make_shared<FunctionGenerateRandomStructure>(context->getSettingsRef().allow_suspicious_low_cardinality_types.value);
+    }
+
+    String getName() const override { return name; }
+
+    size_t getNumberOfArguments() const override { return 0; }
+
+    bool isVariadic() const override { return true; }
+    bool isDeterministic() const override { return false; }
+    bool isDeterministicInScopeOfQuery() const override { return false; }
+    bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const  override { return {0, 1}; }
+    bool useDefaultImplementationForConstants() const override { return false; }
+    bool useDefaultImplementationForNulls() const override { return false; }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    {
+        if (arguments.size() > 2)
+            throw Exception(
+                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+                "Number of arguments for function {} doesn't match: passed {}, expected from 0 to 2",
+                getName(), arguments.size());
+
+
+        for (size_t i = 0; i != arguments.size(); ++i)
+        {
+            if (!isUnsignedInteger(arguments[i]) && !arguments[i]->onlyNull())
+            {
+                throw Exception(
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Illegal type {} of the {} argument of function {}, expected unsigned integer or Null",
+                    arguments[i]->getName(),
+                    i + 1,
+                    getName());
+            }
+        }
+
+        return std::make_shared<DataTypeString>();
+    }
+
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
+    {
+        size_t seed = randomSeed();
+        size_t number_of_columns = 0;
+
+        if (!arguments.empty() && !arguments[0].column->onlyNull())
+        {
+            number_of_columns = arguments[0].column->getUInt(0);
+            if (number_of_columns > MAX_NUMBER_OF_COLUMNS)
+                throw Exception(
+                    ErrorCodes::BAD_ARGUMENTS,
+                    "Maximum allowed number of columns is {}, got {}",
+                    MAX_NUMBER_OF_COLUMNS,
+                    number_of_columns);
+        }
+
+        if (arguments.size() > 1 && !arguments[1].column->onlyNull())
+            seed = arguments[1].column->getUInt(0);
+
+        pcg64 rng(seed);
+        if (number_of_columns == 0)
+            number_of_columns = generateNumberOfColumns(rng);
+
+        auto col_res = ColumnString::create();
+        auto & string_column = assert_cast<ColumnString &>(*col_res);
+        auto & chars = string_column.getChars();
+        WriteBufferFromVector buf(chars);
+        writeRandomStructure(rng, number_of_columns, buf, allow_suspicious_lc_types);
+        buf.finalize();
+        chars.push_back(0);
+        string_column.getOffsets().push_back(chars.size());
+        return ColumnConst::create(std::move(col_res), input_rows_count);
+    }
+
+    static String getRandomStructure(size_t seed, const ContextPtr & context)
+    {
+        pcg64 rng(seed);
+        size_t number_of_columns = generateNumberOfColumns(rng);
+        WriteBufferFromOwnString buf;
+        writeRandomStructure(rng, number_of_columns, buf, context->getSettingsRef().allow_suspicious_low_cardinality_types);
+        return buf.str();
+    }
+
+private:
+
+    static size_t generateNumberOfColumns(pcg64 & rng)
+    {
+        return rng() % MAX_NUMBER_OF_COLUMNS + 1;
+    }
+
+    static void writeRandomStructure(pcg64 & rng, size_t number_of_columns, WriteBuffer & buf, bool allow_suspicious_lc_types)
+    {
+        for (size_t i = 0; i != number_of_columns; ++i)
+        {
+            if (i != 0)
+                writeCString(", ", buf);
+            String column_name = "c" + std::to_string(i + 1);
+            writeString(column_name, buf);
+            writeChar(' ', buf);
+            writeRandomType(column_name, rng, buf, allow_suspicious_lc_types);
+        }
+    }
+
+    template <bool allow_complex_types = true>
+    static void writeRandomType(const String & column_name, pcg64 & rng, WriteBuffer & buf, bool allow_suspicious_lc_types, size_t depth = 0)
+    {
+        if (allow_complex_types && depth > MAX_DEPTH)
+            writeRandomType<false>(column_name, rng, buf, depth);
+
+        constexpr auto all_types = getAllTypes<allow_complex_types>();
+        auto type = all_types[rng() % all_types.size()];
+
+        switch (type)
+        {
+            case TypeIndex::UInt8:
+                if (rng() % 2)
+                    writeCString("UInt8", buf);
+                else
+                    writeCString("Bool", buf);
+                return;
+            case TypeIndex::FixedString:
+                writeString("FixedString(" + std::to_string(rng() % MAX_FIXEDSTRING_SIZE_WITHOUT_SUSPICIOUS + 1) + ")", buf);
+                return;
+            case TypeIndex::DateTime64:
+                writeString("DateTime64(" + std::to_string(rng() % MAX_DATETIME64_PRECISION + 1) + ")", buf);
+                return;
+            case TypeIndex::Decimal32:
+                writeString("Decimal32(" + std::to_string(rng() % MAX_DECIMAL32_PRECISION + 1) + ")", buf);
+                return;
+            case TypeIndex::Decimal64:
+                writeString("Decimal64(" + std::to_string(rng() % MAX_DECIMAL64_PRECISION + 1) + ")", buf);
+                return;
+            case TypeIndex::Decimal128:
+                writeString("Decimal128(" + std::to_string(rng() % MAX_DECIMAL128_PRECISION + 1) + ")", buf);
+                return;
+            case TypeIndex::Decimal256:
+                writeString("Decimal256(" + std::to_string(rng() % MAX_DECIMAL256_PRECISION + 1) + ")", buf);
+                return;
+            case TypeIndex::Enum8:
+                writeCString("Enum8(", buf);
+                writeEnumValues(column_name, rng, buf, INT8_MAX);
+                writeChar(')', buf);
+                return;
+            case TypeIndex::Enum16:
+                writeCString("Enum16(", buf);
+                writeEnumValues(column_name, rng, buf, INT16_MAX);
+                writeChar(')', buf);
+                return;
+            case TypeIndex::LowCardinality:
+                writeCString("LowCardinality(", buf);
+                writeLowCardinalityNestedType(rng, buf, allow_suspicious_lc_types);
+                writeChar(')', buf);
+                return;
+            case TypeIndex::Nullable:
+            {
+                writeCString("Nullable(", buf);
+                writeRandomType<false>(column_name, rng, buf, allow_suspicious_lc_types, depth + 1);
+                writeChar(')', buf);
+                return;
+            }
+            case TypeIndex::Array:
+            {
+                writeCString("Array(", buf);
+                writeRandomType(column_name, rng, buf, allow_suspicious_lc_types, depth + 1);
+                writeChar(')', buf);
+                return;
+            }
+            case TypeIndex::Map:
+            {
+                writeCString("Map(", buf);
+                writeMapKeyType(rng, buf, allow_suspicious_lc_types);
+                writeCString(", ", buf);
+                writeRandomType(column_name, rng, buf, allow_suspicious_lc_types, depth + 1);
+                writeChar(')', buf);
+                return;
+            }
+            case TypeIndex::Tuple:
+            {
+                size_t elements = rng() % MAX_TUPLE_ELEMENTS + 1;
+                bool generate_nested = rng() % 2;
+                bool generate_named_tuple = rng() % 2;
+                if (generate_nested)
+                    writeCString("Nested(", buf);
+                else
+                    writeCString("Tuple(", buf);
+
+                for (size_t i = 0; i != elements; ++i)
+                {
+                    if (i != 0)
+                        writeCString(", ", buf);
+
+                    String element_name = "e" + std::to_string(i + 1);
+                    if (generate_named_tuple || generate_nested)
+                    {
+                        writeString(element_name, buf);
+                        writeChar(' ', buf);
+                    }
+                    writeRandomType(element_name, rng, buf, allow_suspicious_lc_types, depth + 1);
+                }
+                writeChar(')', buf);
+                return;
+            }
+            default:
+                writeString(magic_enum::enum_name<TypeIndex>(type), buf);
+                return;
+        }
+    }
+
+    static void writeMapKeyType(pcg64 & rng, WriteBuffer & buf, bool allow_suspicious_lc_types)
+    {
+        TypeIndex type = map_key_types[rng() % map_key_types.size()];
+        if (type == TypeIndex::FixedString)
+        {
+            writeString("FixedString(" + std::to_string(rng() % MAX_FIXEDSTRING_SIZE_WITHOUT_SUSPICIOUS + 1) + ")", buf);
+        }
+        else if (type == TypeIndex::LowCardinality)
+        {
+            writeCString("LowCardinality(", buf);
+            writeLowCardinalityNestedType(rng, buf, allow_suspicious_lc_types);
+            writeChar(')', buf);
+        }
+        else
+        {
+            writeString(magic_enum::enum_name<TypeIndex>(type), buf);
+        }
+    }
+
+    static void writeLowCardinalityNestedType(pcg64 & rng, WriteBuffer & buf, bool allow_suspicious_lc_types)
+    {
+        bool make_nullable = rng() % 2;
+        if (make_nullable)
+            writeCString("Nullable(", buf);
+
+        if (allow_suspicious_lc_types)
+        {
+            TypeIndex type = suspicious_lc_types[rng() % map_key_types.size()];
+            if (type == TypeIndex::FixedString)
+                writeString("FixedString(" + std::to_string(rng() % MAX_FIXEDSTRING_SIZE_WITHOUT_SUSPICIOUS + 1) + ")", buf);
+            else
+                writeString(magic_enum::enum_name<TypeIndex>(type), buf);
+        }
+        else
+        {
+            /// Support only String and FixedString.
+            if (rng() % 2)
+                writeCString("String", buf);
+            else
+                writeString("FixedString(" + std::to_string(rng() % MAX_FIXEDSTRING_SIZE_WITHOUT_SUSPICIOUS + 1) + ")", buf);
+        }
+
+        if (make_nullable)
+            writeChar(')', buf);
+    }
+
+    static void writeEnumValues(const String & column_name, pcg64 & rng, WriteBuffer & buf, ssize_t max_value)
+    {
+        /// Don't generate big enums, because it will lead to really big result
+        /// and slowness of this function, and it can lead to `Max query size exceeded`
+        /// while using this function with generateRandom.
+        size_t num_values = rng() % 16 + 1;
+        std::vector<Int16> values(num_values);
+
+        /// Generate random numbers from range [-(max_value + 1), max_value - num_values + 1]
+        for (Int16 & x : values)
+            x = rng() % (2 * (max_value + 1) - num_values) - max_value - 1;
+        /// Make all numbers unique.
+        std::sort(values.begin(), values.end());
+        for (size_t i = 0; i < num_values; ++i)
+            values[i] += i;
+        std::shuffle(values.begin(), values.end(), rng);
+        for (size_t i = 0; i != num_values; ++i)
+        {
+            if (i != 0)
+                writeCString(", ", buf);
+            writeString("'" + column_name + "V" + std::to_string(values[i]) + "' = " + std::to_string(i), buf);
+        }
+    }
+
+    template <bool allow_complex_types>
+    static constexpr auto getAllTypes()
+    {
+        constexpr size_t complex_types_size = complex_types.size() * allow_complex_types;
+        constexpr size_t result_size = simple_types.size() + complex_types_size;
+        std::array<TypeIndex, result_size> result;
+        size_t index = 0;
+
+        for (size_t i = 0; i != simple_types.size(); ++i, ++index)
+            result[index] = simple_types[i];
+
+        for (size_t i = 0; i != complex_types_size; ++i, ++index)
+            result[index] = complex_types[i];
+
+        return result;
+    }
+
+    bool allow_suspicious_lc_types;
+};
+
+
+REGISTER_FUNCTION(GenerateRandomStructure)
+{
+    factory.registerFunction<FunctionGenerateRandomStructure>(
+        {
+            R"(
+Generates a random table structure.
+This function takes 2 optional constant arguments:
+the number of columns in the result structure (random by default) and random seed (random by default)
+The maximum number of columns is 128.
+The function returns a value of type String.
+)",
+            Documentation::Examples{
+                {"random", "SELECT generateRandomStructure()"},
+                {"with specified number of columns", "SELECT generateRandomStructure(10)"},
+                {"with specified seed", "SELECT generateRandomStructure(10, 42)"},
+            },
+            Documentation::Categories{"Random"}
+        },
+        FunctionFactory::CaseSensitive);
+}
+
+}
diff --git a/src/Functions/FunctionGenerateRandomStructure.h b/src/Functions/FunctionGenerateRandomStructure.h
new file mode 100644
index 00000000000..1d1bcb1a0a8
--- /dev/null
+++ b/src/Functions/FunctionGenerateRandomStructure.h
@@ -0,0 +1,45 @@
+#include <Functions/IFunction.h>
+#include <Interpreters/Context.h>
+
+#include <pcg_random.hpp>
+
+namespace DB
+{
+
+class FunctionGenerateRandomStructure : public IFunction
+{
+public:
+    static constexpr auto name = "generateRandomStructure";
+
+    explicit FunctionGenerateRandomStructure(bool allow_suspicious_lc_types_) : allow_suspicious_lc_types(allow_suspicious_lc_types_)
+    {
+    }
+
+    static FunctionPtr create(ContextPtr context)
+    {
+        return std::make_shared<FunctionGenerateRandomStructure>(context->getSettingsRef().allow_suspicious_low_cardinality_types.value);
+    }
+
+    String getName() const override { return name; }
+
+    size_t getNumberOfArguments() const override { return 0; }
+
+    bool isVariadic() const override { return true; }
+    bool isDeterministic() const override { return false; }
+    bool isDeterministicInScopeOfQuery() const override { return false; }
+    bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const  override { return {0, 1}; }
+    bool useDefaultImplementationForConstants() const override { return false; }
+    bool useDefaultImplementationForNulls() const override { return false; }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override;
+
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override;
+
+    static String generateRandomStructure(size_t seed, const ContextPtr & context);
+
+private:
+    bool allow_suspicious_lc_types;
+};
+
+}

From 78064d062266559b9032d328b84a22e5f60b75d7 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 15 May 2023 15:52:14 +0000
Subject: [PATCH 537/689] Better comments

---
 src/Common/AsyncTaskExecutor.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/Common/AsyncTaskExecutor.h b/src/Common/AsyncTaskExecutor.h
index f87abd7eb9b..55dc2913c13 100644
--- a/src/Common/AsyncTaskExecutor.h
+++ b/src/Common/AsyncTaskExecutor.h
@@ -131,9 +131,9 @@ struct FiberLocal
 public:
     FiberLocal()
     {
-        /// Initialize main instance for this thread.
-        /// Contexts for fibers will inherit this instance
-        /// (it could be changed before creating fibers).
+        /// Initialize main instance for this thread. Instances for fibers will inherit it,
+        /// (it's needed because main instance could be changed before creating fibers
+        /// and changes should be visible in fibers).
         data[nullptr] = T();
     }
 
@@ -150,14 +150,14 @@ public:
 private:
     T & get()
     {
-        /// Get instance for current fiber.
         return getInstanceForFiber(AsyncTaskExecutor::getCurrentFiberInfo());
     }
 
     T & getInstanceForFiber(FiberInfo info)
     {
         auto it = data.find(info.fiber);
-        /// If it's the first request, we need to initialize instance for the fiber using instance from parent fiber.
+        /// If it's the first request, we need to initialize instance for the fiber
+        /// using instance from parent fiber or main thread that created fiber.
         if (it == data.end())
             it = data.insert({info.fiber, getInstanceForFiber(*info.parent_fiber_info)}).first;
         return it->second;

From 05ae7b2c2d7337c2ade92634f859718061a31064 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Mon, 15 May 2023 18:28:12 +0200
Subject: [PATCH 538/689] fix some tests

---
 src/Interpreters/MutationsInterpreter.cpp     |  1 +
 src/Interpreters/MutationsInterpreter.h       |  4 +--
 .../MergeTree/MutateFromLogEntryTask.cpp      |  5 ++-
 src/Storages/MergeTree/MutateTask.cpp         | 30 +++++++++++++++--
 .../ReplicatedMergeTreeMutationEntry.cpp      |  9 +++++
 .../ReplicatedMergeTreeMutationEntry.h        |  2 ++
 .../MergeTree/ReplicatedMergeTreeQueue.cpp    | 11 ++++---
 .../MergeTree/ReplicatedMergeTreeQueue.h      |  3 +-
 src/Storages/StorageReplicatedMergeTree.cpp   |  9 +++--
 tests/clickhouse-test                         |  8 ++---
 .../test_default_compression_codec/test.py    |  4 +++
 .../02440_mutations_finalization.reference    |  5 +++
 .../02440_mutations_finalization.sql          | 33 +++++++++++++++++++
 ...441_alter_delete_and_drop_column.reference |  2 ++
 .../02441_alter_delete_and_drop_column.sql    | 20 +++++++++++
 15 files changed, 130 insertions(+), 16 deletions(-)
 create mode 100644 tests/queries/0_stateless/02440_mutations_finalization.reference
 create mode 100644 tests/queries/0_stateless/02440_mutations_finalization.sql
 create mode 100644 tests/queries/0_stateless/02441_alter_delete_and_drop_column.reference
 create mode 100644 tests/queries/0_stateless/02441_alter_delete_and_drop_column.sql

diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp
index 2f05ba5d7f8..2f9e647b99e 100644
--- a/src/Interpreters/MutationsInterpreter.cpp
+++ b/src/Interpreters/MutationsInterpreter.cpp
@@ -542,6 +542,7 @@ void MutationsInterpreter::prepare(bool dry_run)
     if (commands.empty())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty mutation commands list");
 
+    /// TODO Should we get columns, indices and projections from the part itself? Table metadata may be different
     const ColumnsDescription & columns_desc = metadata_snapshot->getColumns();
     const IndicesDescription & indices_desc = metadata_snapshot->getSecondaryIndices();
     const ProjectionsDescription & projections_desc = metadata_snapshot->getProjections();
diff --git a/src/Interpreters/MutationsInterpreter.h b/src/Interpreters/MutationsInterpreter.h
index 4b965546408..8bb90365050 100644
--- a/src/Interpreters/MutationsInterpreter.h
+++ b/src/Interpreters/MutationsInterpreter.h
@@ -50,7 +50,7 @@ public:
         bool return_all_columns_ = false,
         bool return_mutated_rows_ = false);
 
-    /// Special case for MergeTree
+    /// Special case for *MergeTree
     MutationsInterpreter(
         MergeTreeData & storage_,
         MergeTreeData::DataPartPtr source_part_,
@@ -123,7 +123,7 @@ public:
     private:
         StoragePtr storage;
 
-        /// Special case for MergeTree.
+        /// Special case for *MergeTree.
         MergeTreeData * data = nullptr;
         MergeTreeData::DataPartPtr part;
     };
diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp
index 35a503c658d..d0b85ee65b8 100644
--- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp
+++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp
@@ -90,7 +90,10 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
     }
 
     new_part_info = MergeTreePartInfo::fromPartName(entry.new_part_name, storage.format_version);
-    commands = std::make_shared<MutationCommands>(storage.queue.getMutationCommands(source_part, new_part_info.mutation));
+    Strings mutation_ids;
+    commands = std::make_shared<MutationCommands>(storage.queue.getMutationCommands(source_part, new_part_info.mutation, mutation_ids));
+    LOG_TRACE(log, "Mutating part {} with mutation commands from {} mutations ({}): {}",
+              entry.new_part_name, commands->size(), fmt::join(mutation_ids, ", "), commands->toString());
 
     /// Once we mutate part, we must reserve space on the same disk, because mutations can possibly create hardlinks.
     /// Can throw an exception.
diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp
index a3fa210ac42..27e5319ed4f 100644
--- a/src/Storages/MergeTree/MutateTask.cpp
+++ b/src/Storages/MergeTree/MutateTask.cpp
@@ -58,7 +58,9 @@ static void splitAndModifyMutationCommands(
     MergeTreeData::DataPartPtr part,
     const MutationCommands & commands,
     MutationCommands & for_interpreter,
-    MutationCommands & for_file_renames)
+    MutationCommands & for_file_renames,
+    const StorageMetadataPtr & table_metadata_snapshot,
+    Poco::Logger * log)
 {
     auto part_columns = part->getColumnsDescription();
 
@@ -142,6 +144,29 @@ static void splitAndModifyMutationCommands(
         {
             if (!mutated_columns.contains(column.name))
             {
+                if (!table_metadata_snapshot->getColumns().has(column.name))
+                {
+                    /// We cannot add the column because there's no such column in table.
+                    /// It's okay if the column was dropped. It may also absent in dropped_columns
+                    /// if the corresponding MUTATE_PART entry was not created yet or was created separately from current MUTATE_PART.
+                    /// But we don't know for sure what happened.
+                    auto part_metadata_version = part->getMetadataVersion();
+                    auto table_metadata_version = table_metadata_snapshot->getMetadataVersion();
+                    if (table_metadata_version <= part_metadata_version)
+                        throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} with metadata version {} contains column {} that is absent "
+                                        "in table {} with metadata version {}",
+                                        part->name, part_metadata_version, column.name,
+                                        part->storage.getStorageID().getNameForLogs(), table_metadata_version);
+
+                    if (part_metadata_version < table_metadata_version)
+                    {
+                        LOG_WARNING(log, "Ignoring column {} from part {} with metadata version {} because there is no such column "
+                                         "in table {} with metadata version {}. Assuming the column was dropped", column.name, part->name,
+                                    part_metadata_version, part->storage.getStorageID().getNameForLogs(), table_metadata_version);
+                        continue;
+                    }
+                }
+
                 for_interpreter.emplace_back(
                     MutationCommand{.type = MutationCommand::Type::READ_COLUMN, .column_name = column.name, .data_type = column.type});
             }
@@ -1776,7 +1801,8 @@ bool MutateTask::prepare()
     context_for_reading->setSetting("allow_asynchronous_read_from_io_pool_for_merge_tree", false);
     context_for_reading->setSetting("max_streams_for_merge_tree_reading", Field(0));
 
-    MutationHelpers::splitAndModifyMutationCommands(ctx->source_part, ctx->commands_for_part, ctx->for_interpreter, ctx->for_file_renames);
+    MutationHelpers::splitAndModifyMutationCommands(ctx->source_part, ctx->commands_for_part, ctx->for_interpreter,
+                                                    ctx->for_file_renames, ctx->metadata_snapshot, ctx->log);
 
     ctx->stage_progress = std::make_unique<MergeStageProgress>(1.0);
 
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp
index 1efb3f6826b..5e01cd96f6b 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp
@@ -99,4 +99,13 @@ std::shared_ptr<const IBackupEntry> ReplicatedMergeTreeMutationEntry::backup() c
     return std::make_shared<BackupEntryFromMemory>(out.str());
 }
 
+
+String ReplicatedMergeTreeMutationEntry::getBlockNumbersForLogs() const
+{
+    WriteBufferFromOwnString out;
+    for (const auto & kv : block_numbers)
+        out << kv.first << " = " << kv.second << "; ";
+    return out.str();
+}
+
 }
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h
index 8c359a57279..09787bd1731 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h
@@ -51,6 +51,8 @@ struct ReplicatedMergeTreeMutationEntry
     bool isAlterMutation() const { return alter_version != -1; }
 
     std::shared_ptr<const IBackupEntry> backup() const;
+
+    String getBlockNumbersForLogs() const;
 };
 
 using ReplicatedMergeTreeMutationEntryPtr = std::shared_ptr<const ReplicatedMergeTreeMutationEntry>;
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
index 1762c7aabe9..9f7ae3222a4 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
@@ -955,13 +955,14 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C
                     const String & partition_id = pair.first;
                     Int64 block_num = pair.second;
                     mutations_by_partition[partition_id].emplace(block_num, &mutation);
-                    LOG_TRACE(log, "Adding mutation {} for partition {} for all block numbers less than {}", entry->znode_name, partition_id, block_num);
                 }
+                LOG_TRACE(log, "Adding mutation {} for {} partitions (data versions: {})",
+                          entry->znode_name, entry->block_numbers.size(), entry->getBlockNumbersForLogs());
 
                 /// Initialize `mutation.parts_to_do`. We cannot use only current_parts + virtual_parts here so we
                 /// traverse all the queue and build correct state of parts_to_do.
                 auto queue_representation = getQueueRepresentation(queue, format_version);
-                mutation.parts_to_do = getPartNamesToMutate(*entry, virtual_parts, queue_representation, format_version);
+                mutation.parts_to_do = getPartNamesToMutate(*entry, current_parts, queue_representation, format_version);
 
                 if (mutation.parts_to_do.size() == 0)
                     some_mutations_are_probably_done = true;
@@ -1801,7 +1802,7 @@ std::map<int64_t, MutationCommands> ReplicatedMergeTreeQueue::getAlterMutationCo
 }
 
 MutationCommands ReplicatedMergeTreeQueue::getMutationCommands(
-    const MergeTreeData::DataPartPtr & part, Int64 desired_mutation_version) const
+    const MergeTreeData::DataPartPtr & part, Int64 desired_mutation_version, Strings & mutation_ids) const
 {
     /// NOTE: If the corresponding mutation is not found, the error is logged (and not thrown as an exception)
     /// to allow recovering from a mutation that cannot be executed. This way you can delete the mutation entry
@@ -1840,6 +1841,8 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands(
     MutationCommands commands;
     for (auto it = begin; it != end; ++it)
     {
+        chassert(mutation_pointer < it->second->entry->znode_name);
+        mutation_ids.push_back(it->second->entry->znode_name);
         const auto & commands_from_entry = it->second->entry->commands;
         commands.insert(commands.end(), commands_from_entry.begin(), commands_from_entry.end());
     }
@@ -2600,7 +2603,7 @@ void ReplicatedMergeTreeQueue::removeCurrentPartsFromMutations()
 {
     std::lock_guard state_lock(state_mutex);
     for (const auto & part_name : current_parts.getParts())
-        removeCoveredPartsFromMutations(part_name, /*remove_part = */ true, /*remove_covered_parts = */ true);
+        removeCoveredPartsFromMutations(part_name, /*remove_part = */ false, /*remove_covered_parts = */ true);
 }
 
 }
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h
index 368f2d4bc1f..3fefc341bbc 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h
@@ -392,7 +392,8 @@ public:
     /// Returns functor which used by MergeTreeMergerMutator to select parts for merge
     ReplicatedMergeTreeMergePredicate getMergePredicate(zkutil::ZooKeeperPtr & zookeeper, PartitionIdsHint && partition_ids_hint);
 
-    MutationCommands getMutationCommands(const MergeTreeData::DataPartPtr & part, Int64 desired_mutation_version) const;
+    MutationCommands getMutationCommands(const MergeTreeData::DataPartPtr & part, Int64 desired_mutation_version,
+                                         Strings & mutation_ids) const;
 
     /// Return mutation commands for part which could be not applied to
     /// it according to part mutation version. Used when we apply alter commands on fly,
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index b5e53950a02..5ca97f06cde 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -5265,12 +5265,12 @@ void StorageReplicatedMergeTree::alter(
             fs::path(zookeeper_path) / "log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential));
 
         PartitionBlockNumbersHolder partition_block_numbers_holder;
+        ReplicatedMergeTreeMutationEntry mutation_entry;
         if (have_mutation)
         {
             delayMutationOrThrowIfNeeded(&partial_shutdown_event, query_context);
             const String mutations_path(fs::path(zookeeper_path) / "mutations");
 
-            ReplicatedMergeTreeMutationEntry mutation_entry;
             mutation_entry.alter_version = new_metadata_version;
             mutation_entry.source_replica = replica_name;
             mutation_entry.commands = std::move(maybe_mutation_commands);
@@ -5322,12 +5322,16 @@ void StorageReplicatedMergeTree::alter(
                 /// ReplicatedMergeTreeMutationEntry record in /mutations
                 String mutation_path = dynamic_cast<const Coordination::CreateResponse &>(*results[mutation_path_idx]).path_created;
                 mutation_znode = mutation_path.substr(mutation_path.find_last_of('/') + 1);
+                LOG_DEBUG(log, "Created log entry {} to update table metadata to version {}, created a mutation {} (data versions: {})",
+                          alter_entry->znode_name, alter_entry->alter_version, *mutation_znode, mutation_entry.getBlockNumbersForLogs());
             }
             else
             {
                 /// ALTER_METADATA record in replication /log
                 String alter_path = dynamic_cast<const Coordination::CreateResponse &>(*results[alter_path_idx]).path_created;
                 alter_entry->znode_name = alter_path.substr(alter_path.find_last_of('/') + 1);
+                LOG_DEBUG(log, "Created log entry {} to update table metadata to version {}",
+                          alter_entry->znode_name, alter_entry->alter_version);
             }
             break;
         }
@@ -6493,7 +6497,8 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, Conte
             const String & path_created =
                 dynamic_cast<const Coordination::CreateResponse *>(responses[1].get())->path_created;
             mutation_entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1);
-            LOG_TRACE(log, "Created mutation with ID {}", mutation_entry.znode_name);
+            LOG_TRACE(log, "Created mutation with ID {} (data versions: {})",
+                      mutation_entry.znode_name, mutation_entry.getBlockNumbersForLogs());
             break;
         }
         else if (rc == Coordination::Error::ZBADVERSION)
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index e279b899a93..cd9dde28fea 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -286,11 +286,11 @@ def get_processlist_with_stacktraces(args):
             -- NOTE: view() here to do JOIN on shards, instead of initiator
             FROM clusterAllReplicas('test_cluster_database_replicated', view(
                 SELECT
+                    p.*,
                     arrayStringConcat(groupArray('Thread ID ' || toString(s.thread_id) || '\n' || arrayStringConcat(arrayMap(
                         x -> concat(addressToLine(x), '::', demangle(addressToSymbol(x))),
                         s.trace), '\n') AS stacktrace
-                    )) AS stacktraces,
-                    p.*
+                    )) AS stacktraces
                 FROM system.processes p
                 JOIN system.stack_trace s USING (query_id)
                 WHERE query NOT LIKE '%system.processes%'
@@ -307,11 +307,11 @@ def get_processlist_with_stacktraces(args):
                 args,
                 """
             SELECT
+                p.*,
                 arrayStringConcat(groupArray('Thread ID ' || toString(s.thread_id) || '\n' || arrayStringConcat(arrayMap(
                     x -> concat(addressToLine(x), '::', demangle(addressToSymbol(x))),
                     s.trace), '\n') AS stacktrace
-                )) AS stacktraces,
-                p.*
+                )) AS stacktraces
             FROM system.processes p
             JOIN system.stack_trace s USING (query_id)
             WHERE query NOT LIKE '%system.processes%'
diff --git a/tests/integration/test_default_compression_codec/test.py b/tests/integration/test_default_compression_codec/test.py
index 5d033ac8f7e..c7c30f5eea4 100644
--- a/tests/integration/test_default_compression_codec/test.py
+++ b/tests/integration/test_default_compression_codec/test.py
@@ -262,6 +262,8 @@ def test_default_codec_multiple(start_cluster):
         )
     )
 
+    node2.query("SYSTEM SYNC REPLICA compression_table_multiple", timeout=15)
+
     # Same codec for all
     assert (
         get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_0")
@@ -330,6 +332,8 @@ def test_default_codec_multiple(start_cluster):
 
     node1.query("OPTIMIZE TABLE compression_table_multiple FINAL")
 
+    node2.query("SYSTEM SYNC REPLICA compression_table_multiple", timeout=15)
+
     assert (
         get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_1")
         == CODECS_MAPPING["Multiple"]
diff --git a/tests/queries/0_stateless/02440_mutations_finalization.reference b/tests/queries/0_stateless/02440_mutations_finalization.reference
new file mode 100644
index 00000000000..a8b9c2acdce
--- /dev/null
+++ b/tests/queries/0_stateless/02440_mutations_finalization.reference
@@ -0,0 +1,5 @@
+0000000000	UPDATE n = 2 WHERE n = 1	['all_0_0_0']	0
+1
+0000000000	UPDATE n = 2 WHERE n = 1	['all_0_0_0']	0
+2
+0000000000	UPDATE n = 2 WHERE n = 1	[]	1
diff --git a/tests/queries/0_stateless/02440_mutations_finalization.sql b/tests/queries/0_stateless/02440_mutations_finalization.sql
new file mode 100644
index 00000000000..796dcde8e4e
--- /dev/null
+++ b/tests/queries/0_stateless/02440_mutations_finalization.sql
@@ -0,0 +1,33 @@
+
+create table mut (n int) engine=ReplicatedMergeTree('/test/02440/{database}/mut', '1') order by tuple();
+set insert_keeper_fault_injection_probability=0;
+insert into mut values (1);
+system stop merges mut;
+alter table mut update n = 2 where n = 1;
+-- it will create MUTATE_PART entry, but will not execute it
+
+select mutation_id, command, parts_to_do_names, is_done from system.mutations where database=currentDatabase() and table='mut';
+
+-- merges (and mutations) will start again after detach/attach, we need to avoid this somehow...
+create table tmp (n int) engine=MergeTree order by tuple() settings index_granularity=1;
+insert into tmp select * from numbers(1000);
+alter table tmp update n = sleepEachRow(1) where 1;
+select sleepEachRow(2) as higher_probablility_of_reproducing_the_issue format Null;
+
+-- it will not execute MUTATE_PART, because another mutation is currently executing (in tmp)
+alter table mut modify setting max_number_of_mutations_for_replica=1;
+detach table mut;
+attach table mut;
+
+-- mutation should not be finished yet
+select * from mut;
+select mutation_id, command, parts_to_do_names, is_done from system.mutations where database=currentDatabase() and table='mut';
+
+alter table mut modify setting max_number_of_mutations_for_replica=100;
+system sync replica mut;
+
+-- and now it should
+select * from mut;
+select mutation_id, command, parts_to_do_names, is_done from system.mutations where database=currentDatabase() and table='mut';
+
+drop table tmp; -- btw, it will check that mutation can be cancelled between blocks on shutdown
diff --git a/tests/queries/0_stateless/02441_alter_delete_and_drop_column.reference b/tests/queries/0_stateless/02441_alter_delete_and_drop_column.reference
new file mode 100644
index 00000000000..e9858167301
--- /dev/null
+++ b/tests/queries/0_stateless/02441_alter_delete_and_drop_column.reference
@@ -0,0 +1,2 @@
+MUTATE_PART	all_0_0_0_1	['all_0_0_0']
+1	2
diff --git a/tests/queries/0_stateless/02441_alter_delete_and_drop_column.sql b/tests/queries/0_stateless/02441_alter_delete_and_drop_column.sql
new file mode 100644
index 00000000000..d274fae1a4f
--- /dev/null
+++ b/tests/queries/0_stateless/02441_alter_delete_and_drop_column.sql
@@ -0,0 +1,20 @@
+
+create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n;
+set insert_keeper_fault_injection_probability=0;
+insert into mut values (1, 2, 3), (10, 20, 30);
+
+system stop merges mut;
+alter table mut delete where n = 10;
+alter table mut drop column k settings alter_sync=0;
+system sync replica mut pull;
+
+-- a funny way to wait for ALTER_METADATA to disappear from the replication queue
+select sleepEachRow(1) from url('http://localhost:8123/?param_tries={1..30}&query=' || encodeURLComponent(
+    'select * from system.replication_queue where database=''' || currentDatabase() || ''' and table=''mut'' and type=''ALTER_METADATA'''
+    ), 'LineAsString', 's String') settings max_threads=1 format Null;
+
+select type, new_part_name, parts_to_merge from system.replication_queue where database=currentDatabase() and table='mut';
+system start merges mut;
+set receive_timeout=30;
+system sync replica mut;
+select * from mut;

From bfcaf95aed8b6f88a02c19c384db9a11a875961d Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Mon, 15 May 2023 18:32:54 +0200
Subject: [PATCH 539/689] Delete unneded files

---
 .../FunctionGenerateRandomStructure.h         | 45 -------------------
 1 file changed, 45 deletions(-)
 delete mode 100644 src/Functions/FunctionGenerateRandomStructure.h

diff --git a/src/Functions/FunctionGenerateRandomStructure.h b/src/Functions/FunctionGenerateRandomStructure.h
deleted file mode 100644
index 1d1bcb1a0a8..00000000000
--- a/src/Functions/FunctionGenerateRandomStructure.h
+++ /dev/null
@@ -1,45 +0,0 @@
-#include <Functions/IFunction.h>
-#include <Interpreters/Context.h>
-
-#include <pcg_random.hpp>
-
-namespace DB
-{
-
-class FunctionGenerateRandomStructure : public IFunction
-{
-public:
-    static constexpr auto name = "generateRandomStructure";
-
-    explicit FunctionGenerateRandomStructure(bool allow_suspicious_lc_types_) : allow_suspicious_lc_types(allow_suspicious_lc_types_)
-    {
-    }
-
-    static FunctionPtr create(ContextPtr context)
-    {
-        return std::make_shared<FunctionGenerateRandomStructure>(context->getSettingsRef().allow_suspicious_low_cardinality_types.value);
-    }
-
-    String getName() const override { return name; }
-
-    size_t getNumberOfArguments() const override { return 0; }
-
-    bool isVariadic() const override { return true; }
-    bool isDeterministic() const override { return false; }
-    bool isDeterministicInScopeOfQuery() const override { return false; }
-    bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
-    ColumnNumbers getArgumentsThatAreAlwaysConstant() const  override { return {0, 1}; }
-    bool useDefaultImplementationForConstants() const override { return false; }
-    bool useDefaultImplementationForNulls() const override { return false; }
-
-    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override;
-
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override;
-
-    static String generateRandomStructure(size_t seed, const ContextPtr & context);
-
-private:
-    bool allow_suspicious_lc_types;
-};
-
-}

From 900aca5f0a9c6c498b18cb1778e545b5f4d951f2 Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Mon, 15 May 2023 18:33:09 +0200
Subject: [PATCH 540/689] Delete unneded files

---
 .../FunctionGenerateRandomStructure.cpp       | 457 ------------------
 1 file changed, 457 deletions(-)
 delete mode 100644 src/Functions/FunctionGenerateRandomStructure.cpp

diff --git a/src/Functions/FunctionGenerateRandomStructure.cpp b/src/Functions/FunctionGenerateRandomStructure.cpp
deleted file mode 100644
index 023a73fe147..00000000000
--- a/src/Functions/FunctionGenerateRandomStructure.cpp
+++ /dev/null
@@ -1,457 +0,0 @@
-#include <Columns/ColumnString.h>
-#include <DataTypes/DataTypeString.h>
-#include <DataTypes/DataTypeFixedString.h>
-#include <Functions/FunctionFactory.h>
-#include <Functions/FunctionHelpers.h>
-#include <Functions/IFunction.h>
-#include <Interpreters/Context.h>
-#include <Common/randomSeed.h>
-
-#include <pcg_random.hpp>
-
-namespace DB
-{
-
-namespace ErrorCodes
-{
-    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
-    extern const int BAD_ARGUMENTS;
-}
-
-class FunctionGenerateRandomStructure : public IFunction
-{
-private:
-    static constexpr std::array<TypeIndex, 29> simple_types
-    {
-        TypeIndex::Int8,
-        TypeIndex::UInt8,
-        TypeIndex::Int16,
-        TypeIndex::UInt16,
-        TypeIndex::Int32,
-        TypeIndex::UInt32,
-        TypeIndex::Int64,
-        TypeIndex::UInt64,
-        TypeIndex::Int128,
-        TypeIndex::UInt128,
-        TypeIndex::Int256,
-        TypeIndex::UInt256,
-        TypeIndex::Float32,
-        TypeIndex::Float64,
-        TypeIndex::Decimal32,
-        TypeIndex::Decimal64,
-        TypeIndex::Decimal128,
-        TypeIndex::Decimal256,
-        TypeIndex::Date,
-        TypeIndex::Date32,
-        TypeIndex::DateTime,
-        TypeIndex::DateTime64,
-        TypeIndex::String,
-        TypeIndex::FixedString,
-        TypeIndex::Enum8,
-        TypeIndex::Enum16,
-        TypeIndex::IPv4,
-        TypeIndex::IPv6,
-        TypeIndex::UUID,
-    };
-
-    static constexpr std::array<TypeIndex, 5> complex_types
-    {
-        TypeIndex::Nullable,
-        TypeIndex::LowCardinality,
-        TypeIndex::Array,
-        TypeIndex::Tuple,
-        TypeIndex::Map,
-    };
-
-    static constexpr std::array<TypeIndex, 22> map_key_types
-    {
-        TypeIndex::Int8,
-        TypeIndex::UInt8,
-        TypeIndex::Int16,
-        TypeIndex::UInt16,
-        TypeIndex::Int32,
-        TypeIndex::UInt32,
-        TypeIndex::Int64,
-        TypeIndex::UInt64,
-        TypeIndex::Int128,
-        TypeIndex::UInt128,
-        TypeIndex::Int256,
-        TypeIndex::UInt256,
-        TypeIndex::Date,
-        TypeIndex::Date32,
-        TypeIndex::DateTime,
-        TypeIndex::String,
-        TypeIndex::FixedString,
-        TypeIndex::IPv4,
-        TypeIndex::Enum8,
-        TypeIndex::Enum16,
-        TypeIndex::UUID,
-        TypeIndex::LowCardinality,
-    };
-
-    static constexpr std::array<TypeIndex, 22> suspicious_lc_types
-    {
-        TypeIndex::Int8,
-        TypeIndex::UInt8,
-        TypeIndex::Int16,
-        TypeIndex::UInt16,
-        TypeIndex::Int32,
-        TypeIndex::UInt32,
-        TypeIndex::Int64,
-        TypeIndex::UInt64,
-        TypeIndex::Int128,
-        TypeIndex::UInt128,
-        TypeIndex::Int256,
-        TypeIndex::UInt256,
-        TypeIndex::Float32,
-        TypeIndex::Float64,
-        TypeIndex::Date,
-        TypeIndex::Date32,
-        TypeIndex::DateTime,
-        TypeIndex::String,
-        TypeIndex::FixedString,
-        TypeIndex::IPv4,
-        TypeIndex::IPv6,
-        TypeIndex::UUID,
-    };
-
-    static constexpr size_t MAX_NUMBER_OF_COLUMNS = 128;
-    static constexpr size_t MAX_TUPLE_ELEMENTS = 16;
-    static constexpr size_t MAX_DATETIME64_PRECISION = 9;
-    static constexpr size_t MAX_DECIMAL32_PRECISION = 9;
-    static constexpr size_t MAX_DECIMAL64_PRECISION = 18;
-    static constexpr size_t MAX_DECIMAL128_PRECISION = 38;
-    static constexpr size_t MAX_DECIMAL256_PRECISION = 76;
-    static constexpr size_t MAX_DEPTH = 32;
-
-public:
-    static constexpr auto name = "generateRandomStructure";
-
-    explicit FunctionGenerateRandomStructure(bool allow_suspicious_lc_types_) : allow_suspicious_lc_types(allow_suspicious_lc_types_)
-    {
-    }
-
-    static FunctionPtr create(ContextPtr context)
-    {
-        return std::make_shared<FunctionGenerateRandomStructure>(context->getSettingsRef().allow_suspicious_low_cardinality_types.value);
-    }
-
-    String getName() const override { return name; }
-
-    size_t getNumberOfArguments() const override { return 0; }
-
-    bool isVariadic() const override { return true; }
-    bool isDeterministic() const override { return false; }
-    bool isDeterministicInScopeOfQuery() const override { return false; }
-    bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
-    ColumnNumbers getArgumentsThatAreAlwaysConstant() const  override { return {0, 1}; }
-    bool useDefaultImplementationForConstants() const override { return false; }
-    bool useDefaultImplementationForNulls() const override { return false; }
-
-    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
-    {
-        if (arguments.size() > 2)
-            throw Exception(
-                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
-                "Number of arguments for function {} doesn't match: passed {}, expected from 0 to 2",
-                getName(), arguments.size());
-
-
-        for (size_t i = 0; i != arguments.size(); ++i)
-        {
-            if (!isUnsignedInteger(arguments[i]) && !arguments[i]->onlyNull())
-            {
-                throw Exception(
-                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "Illegal type {} of the {} argument of function {}, expected unsigned integer or Null",
-                    arguments[i]->getName(),
-                    i + 1,
-                    getName());
-            }
-        }
-
-        return std::make_shared<DataTypeString>();
-    }
-
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
-    {
-        size_t seed = randomSeed();
-        size_t number_of_columns = 0;
-
-        if (!arguments.empty() && !arguments[0].column->onlyNull())
-        {
-            number_of_columns = arguments[0].column->getUInt(0);
-            if (number_of_columns > MAX_NUMBER_OF_COLUMNS)
-                throw Exception(
-                    ErrorCodes::BAD_ARGUMENTS,
-                    "Maximum allowed number of columns is {}, got {}",
-                    MAX_NUMBER_OF_COLUMNS,
-                    number_of_columns);
-        }
-
-        if (arguments.size() > 1 && !arguments[1].column->onlyNull())
-            seed = arguments[1].column->getUInt(0);
-
-        pcg64 rng(seed);
-        if (number_of_columns == 0)
-            number_of_columns = generateNumberOfColumns(rng);
-
-        auto col_res = ColumnString::create();
-        auto & string_column = assert_cast<ColumnString &>(*col_res);
-        auto & chars = string_column.getChars();
-        WriteBufferFromVector buf(chars);
-        writeRandomStructure(rng, number_of_columns, buf, allow_suspicious_lc_types);
-        buf.finalize();
-        chars.push_back(0);
-        string_column.getOffsets().push_back(chars.size());
-        return ColumnConst::create(std::move(col_res), input_rows_count);
-    }
-
-    static String getRandomStructure(size_t seed, const ContextPtr & context)
-    {
-        pcg64 rng(seed);
-        size_t number_of_columns = generateNumberOfColumns(rng);
-        WriteBufferFromOwnString buf;
-        writeRandomStructure(rng, number_of_columns, buf, context->getSettingsRef().allow_suspicious_low_cardinality_types);
-        return buf.str();
-    }
-
-private:
-
-    static size_t generateNumberOfColumns(pcg64 & rng)
-    {
-        return rng() % MAX_NUMBER_OF_COLUMNS + 1;
-    }
-
-    static void writeRandomStructure(pcg64 & rng, size_t number_of_columns, WriteBuffer & buf, bool allow_suspicious_lc_types)
-    {
-        for (size_t i = 0; i != number_of_columns; ++i)
-        {
-            if (i != 0)
-                writeCString(", ", buf);
-            String column_name = "c" + std::to_string(i + 1);
-            writeString(column_name, buf);
-            writeChar(' ', buf);
-            writeRandomType(column_name, rng, buf, allow_suspicious_lc_types);
-        }
-    }
-
-    template <bool allow_complex_types = true>
-    static void writeRandomType(const String & column_name, pcg64 & rng, WriteBuffer & buf, bool allow_suspicious_lc_types, size_t depth = 0)
-    {
-        if (allow_complex_types && depth > MAX_DEPTH)
-            writeRandomType<false>(column_name, rng, buf, depth);
-
-        constexpr auto all_types = getAllTypes<allow_complex_types>();
-        auto type = all_types[rng() % all_types.size()];
-
-        switch (type)
-        {
-            case TypeIndex::UInt8:
-                if (rng() % 2)
-                    writeCString("UInt8", buf);
-                else
-                    writeCString("Bool", buf);
-                return;
-            case TypeIndex::FixedString:
-                writeString("FixedString(" + std::to_string(rng() % MAX_FIXEDSTRING_SIZE_WITHOUT_SUSPICIOUS + 1) + ")", buf);
-                return;
-            case TypeIndex::DateTime64:
-                writeString("DateTime64(" + std::to_string(rng() % MAX_DATETIME64_PRECISION + 1) + ")", buf);
-                return;
-            case TypeIndex::Decimal32:
-                writeString("Decimal32(" + std::to_string(rng() % MAX_DECIMAL32_PRECISION + 1) + ")", buf);
-                return;
-            case TypeIndex::Decimal64:
-                writeString("Decimal64(" + std::to_string(rng() % MAX_DECIMAL64_PRECISION + 1) + ")", buf);
-                return;
-            case TypeIndex::Decimal128:
-                writeString("Decimal128(" + std::to_string(rng() % MAX_DECIMAL128_PRECISION + 1) + ")", buf);
-                return;
-            case TypeIndex::Decimal256:
-                writeString("Decimal256(" + std::to_string(rng() % MAX_DECIMAL256_PRECISION + 1) + ")", buf);
-                return;
-            case TypeIndex::Enum8:
-                writeCString("Enum8(", buf);
-                writeEnumValues(column_name, rng, buf, INT8_MAX);
-                writeChar(')', buf);
-                return;
-            case TypeIndex::Enum16:
-                writeCString("Enum16(", buf);
-                writeEnumValues(column_name, rng, buf, INT16_MAX);
-                writeChar(')', buf);
-                return;
-            case TypeIndex::LowCardinality:
-                writeCString("LowCardinality(", buf);
-                writeLowCardinalityNestedType(rng, buf, allow_suspicious_lc_types);
-                writeChar(')', buf);
-                return;
-            case TypeIndex::Nullable:
-            {
-                writeCString("Nullable(", buf);
-                writeRandomType<false>(column_name, rng, buf, allow_suspicious_lc_types, depth + 1);
-                writeChar(')', buf);
-                return;
-            }
-            case TypeIndex::Array:
-            {
-                writeCString("Array(", buf);
-                writeRandomType(column_name, rng, buf, allow_suspicious_lc_types, depth + 1);
-                writeChar(')', buf);
-                return;
-            }
-            case TypeIndex::Map:
-            {
-                writeCString("Map(", buf);
-                writeMapKeyType(rng, buf, allow_suspicious_lc_types);
-                writeCString(", ", buf);
-                writeRandomType(column_name, rng, buf, allow_suspicious_lc_types, depth + 1);
-                writeChar(')', buf);
-                return;
-            }
-            case TypeIndex::Tuple:
-            {
-                size_t elements = rng() % MAX_TUPLE_ELEMENTS + 1;
-                bool generate_nested = rng() % 2;
-                bool generate_named_tuple = rng() % 2;
-                if (generate_nested)
-                    writeCString("Nested(", buf);
-                else
-                    writeCString("Tuple(", buf);
-
-                for (size_t i = 0; i != elements; ++i)
-                {
-                    if (i != 0)
-                        writeCString(", ", buf);
-
-                    String element_name = "e" + std::to_string(i + 1);
-                    if (generate_named_tuple || generate_nested)
-                    {
-                        writeString(element_name, buf);
-                        writeChar(' ', buf);
-                    }
-                    writeRandomType(element_name, rng, buf, allow_suspicious_lc_types, depth + 1);
-                }
-                writeChar(')', buf);
-                return;
-            }
-            default:
-                writeString(magic_enum::enum_name<TypeIndex>(type), buf);
-                return;
-        }
-    }
-
-    static void writeMapKeyType(pcg64 & rng, WriteBuffer & buf, bool allow_suspicious_lc_types)
-    {
-        TypeIndex type = map_key_types[rng() % map_key_types.size()];
-        if (type == TypeIndex::FixedString)
-        {
-            writeString("FixedString(" + std::to_string(rng() % MAX_FIXEDSTRING_SIZE_WITHOUT_SUSPICIOUS + 1) + ")", buf);
-        }
-        else if (type == TypeIndex::LowCardinality)
-        {
-            writeCString("LowCardinality(", buf);
-            writeLowCardinalityNestedType(rng, buf, allow_suspicious_lc_types);
-            writeChar(')', buf);
-        }
-        else
-        {
-            writeString(magic_enum::enum_name<TypeIndex>(type), buf);
-        }
-    }
-
-    static void writeLowCardinalityNestedType(pcg64 & rng, WriteBuffer & buf, bool allow_suspicious_lc_types)
-    {
-        bool make_nullable = rng() % 2;
-        if (make_nullable)
-            writeCString("Nullable(", buf);
-
-        if (allow_suspicious_lc_types)
-        {
-            TypeIndex type = suspicious_lc_types[rng() % map_key_types.size()];
-            if (type == TypeIndex::FixedString)
-                writeString("FixedString(" + std::to_string(rng() % MAX_FIXEDSTRING_SIZE_WITHOUT_SUSPICIOUS + 1) + ")", buf);
-            else
-                writeString(magic_enum::enum_name<TypeIndex>(type), buf);
-        }
-        else
-        {
-            /// Support only String and FixedString.
-            if (rng() % 2)
-                writeCString("String", buf);
-            else
-                writeString("FixedString(" + std::to_string(rng() % MAX_FIXEDSTRING_SIZE_WITHOUT_SUSPICIOUS + 1) + ")", buf);
-        }
-
-        if (make_nullable)
-            writeChar(')', buf);
-    }
-
-    static void writeEnumValues(const String & column_name, pcg64 & rng, WriteBuffer & buf, ssize_t max_value)
-    {
-        /// Don't generate big enums, because it will lead to really big result
-        /// and slowness of this function, and it can lead to `Max query size exceeded`
-        /// while using this function with generateRandom.
-        size_t num_values = rng() % 16 + 1;
-        std::vector<Int16> values(num_values);
-
-        /// Generate random numbers from range [-(max_value + 1), max_value - num_values + 1]
-        for (Int16 & x : values)
-            x = rng() % (2 * (max_value + 1) - num_values) - max_value - 1;
-        /// Make all numbers unique.
-        std::sort(values.begin(), values.end());
-        for (size_t i = 0; i < num_values; ++i)
-            values[i] += i;
-        std::shuffle(values.begin(), values.end(), rng);
-        for (size_t i = 0; i != num_values; ++i)
-        {
-            if (i != 0)
-                writeCString(", ", buf);
-            writeString("'" + column_name + "V" + std::to_string(values[i]) + "' = " + std::to_string(i), buf);
-        }
-    }
-
-    template <bool allow_complex_types>
-    static constexpr auto getAllTypes()
-    {
-        constexpr size_t complex_types_size = complex_types.size() * allow_complex_types;
-        constexpr size_t result_size = simple_types.size() + complex_types_size;
-        std::array<TypeIndex, result_size> result;
-        size_t index = 0;
-
-        for (size_t i = 0; i != simple_types.size(); ++i, ++index)
-            result[index] = simple_types[i];
-
-        for (size_t i = 0; i != complex_types_size; ++i, ++index)
-            result[index] = complex_types[i];
-
-        return result;
-    }
-
-    bool allow_suspicious_lc_types;
-};
-
-
-REGISTER_FUNCTION(GenerateRandomStructure)
-{
-    factory.registerFunction<FunctionGenerateRandomStructure>(
-        {
-            R"(
-Generates a random table structure.
-This function takes 2 optional constant arguments:
-the number of columns in the result structure (random by default) and random seed (random by default)
-The maximum number of columns is 128.
-The function returns a value of type String.
-)",
-            Documentation::Examples{
-                {"random", "SELECT generateRandomStructure()"},
-                {"with specified number of columns", "SELECT generateRandomStructure(10)"},
-                {"with specified seed", "SELECT generateRandomStructure(10, 42)"},
-            },
-            Documentation::Categories{"Random"}
-        },
-        FunctionFactory::CaseSensitive);
-}
-
-}

From 4137a5e0582041b0f7fcb388156153c4eb5e360a Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Mon, 15 May 2023 18:51:16 +0200
Subject: [PATCH 541/689] use chassert in MergeTreeDeduplicationLog to have
 better log info

---
 src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp
index b843ce6a078..ac03b0be779 100644
--- a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp
+++ b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp
@@ -227,7 +227,7 @@ std::pair<MergeTreePartInfo, bool> MergeTreeDeduplicationLog::addPart(const std:
         return std::make_pair(info, false);
     }
 
-    assert(current_writer != nullptr);
+    chassert(current_writer != nullptr);
 
     /// Create new record
     MergeTreeDeduplicationLogRecord record;
@@ -257,7 +257,7 @@ void MergeTreeDeduplicationLog::dropPart(const MergeTreePartInfo & drop_part_inf
     if (deduplication_window == 0)
         return;
 
-    assert(current_writer != nullptr);
+    chassert(current_writer != nullptr);
 
     for (auto itr = deduplication_map.begin(); itr != deduplication_map.end(); /* no increment here, we erasing from map */)
     {

From 91db14851397cccb166bc96f4d335dca5928640e Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Thu, 4 May 2023 21:28:33 +0000
Subject: [PATCH 542/689] Fix AsynchronousReadIndirectBufferFromRemoteFS
 breaking on short seeks

---
 ...chronousReadIndirectBufferFromRemoteFS.cpp | 28 +++++++++++--------
 ...ynchronousReadIndirectBufferFromRemoteFS.h |  4 ++-
 .../0_stateless/02731_parquet_s3.reference    |  1 +
 .../queries/0_stateless/02731_parquet_s3.sql  |  7 +++++
 4 files changed, 27 insertions(+), 13 deletions(-)
 create mode 100644 tests/queries/0_stateless/02731_parquet_s3.reference
 create mode 100644 tests/queries/0_stateless/02731_parquet_s3.sql

diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
index bf9a476b785..24b7042e459 100644
--- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
+++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
@@ -118,12 +118,7 @@ std::future<IAsynchronousReader::Result> AsynchronousReadIndirectBufferFromRemot
     request.size = size;
     request.offset = file_offset_of_buffer_end;
     request.priority = base_priority + priority;
-
-    if (bytes_to_ignore)
-    {
-        request.ignore = bytes_to_ignore;
-        bytes_to_ignore = 0;
-    }
+    request.ignore = bytes_to_ignore;
     return reader.submit(request);
 }
 
@@ -165,8 +160,7 @@ void AsynchronousReadIndirectBufferFromRemoteFS::setReadUntilPosition(size_t pos
 
 void AsynchronousReadIndirectBufferFromRemoteFS::setReadUntilEnd()
 {
-    read_until_position = impl->getFileSize();
-    impl->setReadUntilPosition(*read_until_position);
+    setReadUntilPosition(impl->getFileSize());
 }
 
 
@@ -228,12 +222,13 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl()
 
         chassert(memory.size() == read_settings.prefetch_buffer_size || memory.size() == read_settings.remote_fs_buffer_size);
         std::tie(size, offset) = impl->readInto(memory.data(), memory.size(), file_offset_of_buffer_end, bytes_to_ignore);
-        bytes_to_ignore = 0;
 
         ProfileEvents::increment(ProfileEvents::RemoteFSUnprefetchedReads);
         ProfileEvents::increment(ProfileEvents::RemoteFSUnprefetchedBytes, size);
     }
 
+    bytes_to_ignore = 0;
+
     chassert(size >= offset);
 
     size_t bytes_read = size - offset;
@@ -269,7 +264,7 @@ off_t AsynchronousReadIndirectBufferFromRemoteFS::seek(off_t offset, int whence)
     }
     else if (whence == SEEK_CUR)
     {
-        new_pos = file_offset_of_buffer_end - (working_buffer.end() - pos) + offset;
+        new_pos = static_cast<size_t>(getPosition()) + offset;
     }
     else
     {
@@ -277,13 +272,15 @@ off_t AsynchronousReadIndirectBufferFromRemoteFS::seek(off_t offset, int whence)
     }
 
     /// Position is unchanged.
-    if (new_pos + (working_buffer.end() - pos) == file_offset_of_buffer_end)
+    if (new_pos == static_cast<size_t>(getPosition()))
         return new_pos;
 
     bool read_from_prefetch = false;
     while (true)
     {
-        if (file_offset_of_buffer_end - working_buffer.size() <= new_pos && new_pos <= file_offset_of_buffer_end)
+        /// The first condition implies bytes_to_ignore = 0.
+        if (!working_buffer.empty() && file_offset_of_buffer_end - working_buffer.size() <= new_pos &&
+            new_pos <= file_offset_of_buffer_end)
         {
             /// Position is still inside the buffer.
             /// Probably it is at the end of the buffer - then we will load data on the following 'next' call.
@@ -320,6 +317,7 @@ off_t AsynchronousReadIndirectBufferFromRemoteFS::seek(off_t offset, int whence)
 
     /// First reset the buffer so the next read will fetch new data to the buffer.
     resetWorkingBuffer();
+    bytes_to_ignore = 0;
 
     if (read_until_position && new_pos > *read_until_position)
     {
@@ -356,6 +354,12 @@ off_t AsynchronousReadIndirectBufferFromRemoteFS::seek(off_t offset, int whence)
 }
 
 
+off_t AsynchronousReadIndirectBufferFromRemoteFS::getPosition()
+{
+    return file_offset_of_buffer_end - available() + bytes_to_ignore;
+}
+
+
 void AsynchronousReadIndirectBufferFromRemoteFS::finalize()
 {
     resetPrefetch(FilesystemPrefetchState::UNNEEDED);
diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h
index 49b44916a46..e8fb3fe248b 100644
--- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h
+++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h
@@ -42,7 +42,7 @@ public:
 
     off_t seek(off_t offset_, int whence) override;
 
-    off_t getPosition() override { return file_offset_of_buffer_end - available(); }
+    off_t getPosition() override;
 
     String getFileName() const override;
 
@@ -89,6 +89,8 @@ private:
 
     std::string current_reader_id;
 
+    /// If nonzero then working_buffer is empty.
+    /// If a prefetch is in flight, the prefetch task has been instructed to ignore this many bytes.
     size_t bytes_to_ignore = 0;
 
     std::optional<size_t> read_until_position;
diff --git a/tests/queries/0_stateless/02731_parquet_s3.reference b/tests/queries/0_stateless/02731_parquet_s3.reference
new file mode 100644
index 00000000000..5a5aaeb0068
--- /dev/null
+++ b/tests/queries/0_stateless/02731_parquet_s3.reference
@@ -0,0 +1 @@
+12639441726720293784
diff --git a/tests/queries/0_stateless/02731_parquet_s3.sql b/tests/queries/0_stateless/02731_parquet_s3.sql
new file mode 100644
index 00000000000..3c3f11f535b
--- /dev/null
+++ b/tests/queries/0_stateless/02731_parquet_s3.sql
@@ -0,0 +1,7 @@
+-- Tags: no-fasttest
+-- Tag no-fasttest: Depends on AWS
+
+-- Reading from s3 a parquet file of size between ~1 MB and ~2 MB was broken at some point.
+insert into function s3(s3_conn, filename='test_02731_parquet_s3.parquet') select cityHash64(number) from numbers(170000) settings s3_truncate_on_insert=1;
+
+select sum(*) from s3(s3_conn, filename='test_02731_parquet_s3.parquet') settings remote_filesystem_read_method='threadpool', remote_filesystem_read_prefetch=1;

From 65bc702b0bfa2dc01c76fc1ba10007eff980fdd7 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Mon, 15 May 2023 20:02:30 +0200
Subject: [PATCH 543/689] fix

---
 src/Storages/MergeTree/MutateTask.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp
index 27e5319ed4f..f7de6ed3d22 100644
--- a/src/Storages/MergeTree/MutateTask.cpp
+++ b/src/Storages/MergeTree/MutateTask.cpp
@@ -144,7 +144,7 @@ static void splitAndModifyMutationCommands(
         {
             if (!mutated_columns.contains(column.name))
             {
-                if (!table_metadata_snapshot->getColumns().has(column.name))
+                if (!table_metadata_snapshot->getColumns().has(column.name) && !part->storage.getVirtuals().contains(column.name))
                 {
                     /// We cannot add the column because there's no such column in table.
                     /// It's okay if the column was dropped. It may also absent in dropped_columns

From cbc15bf35a61d86e2c38e467d8b0f2210aec8230 Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Mon, 15 May 2023 23:13:17 +0200
Subject: [PATCH 544/689] Add `DynamicResourceManager` and `FairPolicy` into
 scheduling subsystem (#49671)

* Add `DynamicResourceManager` and `FairPolicy` into scheduling subsystem

* fix test

* fix tidy build
---
 src/IO/Resource/DynamicResourceManager.cpp    | 234 ++++++++++++++++++
 src/IO/Resource/DynamicResourceManager.h      |  93 +++++++
 src/IO/Resource/FairPolicy.cpp                |  13 +
 src/IO/Resource/FairPolicy.h                  | 232 +++++++++++++++++
 src/IO/Resource/registerResourceManagers.cpp  |   2 +
 src/IO/Resource/registerSchedulerNodes.cpp    |   2 +
 .../tests/gtest_resource_class_fair.cpp       | 187 ++++++++++++++
 .../gtest_resource_manager_hierarchical.cpp   | 116 +++++++++
 src/Interpreters/Context.cpp                  |   2 +-
 9 files changed, 880 insertions(+), 1 deletion(-)
 create mode 100644 src/IO/Resource/DynamicResourceManager.cpp
 create mode 100644 src/IO/Resource/DynamicResourceManager.h
 create mode 100644 src/IO/Resource/FairPolicy.cpp
 create mode 100644 src/IO/Resource/FairPolicy.h
 create mode 100644 src/IO/Resource/tests/gtest_resource_class_fair.cpp
 create mode 100644 src/IO/Resource/tests/gtest_resource_manager_hierarchical.cpp

diff --git a/src/IO/Resource/DynamicResourceManager.cpp b/src/IO/Resource/DynamicResourceManager.cpp
new file mode 100644
index 00000000000..49e12984e33
--- /dev/null
+++ b/src/IO/Resource/DynamicResourceManager.cpp
@@ -0,0 +1,234 @@
+#include <IO/Resource/DynamicResourceManager.h>
+
+#include <IO/SchedulerNodeFactory.h>
+#include <IO/ResourceManagerFactory.h>
+#include <IO/ISchedulerQueue.h>
+
+#include <Common/Exception.h>
+#include <Common/StringUtils/StringUtils.h>
+
+#include <map>
+#include <tuple>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int RESOURCE_ACCESS_DENIED;
+    extern const int RESOURCE_NOT_FOUND;
+    extern const int INVALID_SCHEDULER_NODE;
+}
+
+DynamicResourceManager::State::State(EventQueue * event_queue, const Poco::Util::AbstractConfiguration & config)
+    : classifiers(config)
+{
+    Poco::Util::AbstractConfiguration::Keys keys;
+    const String config_prefix = "resources";
+    config.keys(config_prefix, keys);
+
+    // Create resource for every element under <resources> tag
+    for (const auto & key : keys)
+    {
+        resources.emplace(key, std::make_shared<Resource>(key, event_queue, config, config_prefix + "." + key));
+    }
+}
+
+DynamicResourceManager::State::Resource::Resource(
+    const String & name,
+    EventQueue * event_queue,
+    const Poco::Util::AbstractConfiguration & config,
+    const std::string & config_prefix)
+{
+    Poco::Util::AbstractConfiguration::Keys keys;
+    config.keys(config_prefix, keys);
+
+    // Sort nodes by path to create parents before children
+    std::map<String, String> path2key;
+    for (const auto & key : keys)
+    {
+        if (!startsWith(key, "node"))
+            continue;
+        String path = config.getString(config_prefix + "." + key + "[@path]", "");
+        if (path.empty())
+            throw Exception(ErrorCodes::INVALID_SCHEDULER_NODE, "Attribute 'path' must be specified in all nodes for resource '{}'", name);
+        if (path[0] != '/')
+            throw Exception(ErrorCodes::INVALID_SCHEDULER_NODE, "Path must start with '/' for resource '{}'", name);
+        if (auto [_, inserted] = path2key.emplace(path, key); !inserted)
+            throw Exception(ErrorCodes::INVALID_SCHEDULER_NODE, "Duplicate path '{}' for resource '{}'", path, name);
+    }
+
+    // Create nodes
+    bool has_root = false;
+    for (auto [path, key] : path2key)
+    {
+        // Validate path
+        size_t slash = path.rfind('/');
+        if (slash == String::npos)
+            throw Exception(ErrorCodes::INVALID_SCHEDULER_NODE, "Invalid scheduler node path '{}' for resource '{}'", path, name);
+
+        // Create node
+        String basename = path.substr(slash + 1); // root name is empty string
+        auto [iter, _] = nodes.emplace(path, Node(basename, event_queue, config, config_prefix + "." + key));
+        if (path == "/")
+        {
+            has_root = true;
+            continue;
+        }
+
+        // Attach created node to parent (if not root)
+        // NOTE: resource root is attached to the scheduler using event queue for thread-safety
+        String parent_path = path.substr(0, slash);
+        if (parent_path.empty())
+            parent_path = "/";
+        if (auto parent = nodes.find(parent_path); parent != nodes.end())
+            parent->second.ptr->attachChild(iter->second.ptr);
+        else
+            throw Exception(ErrorCodes::INVALID_SCHEDULER_NODE, "Parent node doesn't exist for path '{}' for resource '{}'", path, name);
+    }
+
+    if (!has_root)
+        throw Exception(ErrorCodes::INVALID_SCHEDULER_NODE, "undefined root node path '/' for resource '{}'", name);
+}
+
+DynamicResourceManager::State::Resource::~Resource()
+{
+    // NOTE: we should rely on `attached_to` and cannot use `parent`,
+    // NOTE: because `parent` can be `nullptr` in case attachment is still in event queue
+    if (attached_to != nullptr)
+    {
+        ISchedulerNode * root = nodes.find("/")->second.ptr.get();
+        attached_to->event_queue->enqueue([scheduler = attached_to, root]
+        {
+            scheduler->removeChild(root);
+        });
+    }
+}
+
+DynamicResourceManager::State::Node::Node(const String & name, EventQueue * event_queue, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix)
+    : type(config.getString(config_prefix + ".type", "fifo"))
+    , ptr(SchedulerNodeFactory::instance().get(type, event_queue, config, config_prefix))
+{
+    ptr->basename = name;
+}
+
+bool DynamicResourceManager::State::Resource::equals(const DynamicResourceManager::State::Resource & o) const
+{
+    if (nodes.size() != o.nodes.size())
+        return false;
+
+    for (const auto & [path, o_node] : o.nodes)
+    {
+        auto iter = nodes.find(path);
+        if (iter == nodes.end())
+            return false;
+        if (!iter->second.equals(o_node))
+            return false;
+    }
+
+    return true;
+}
+
+bool DynamicResourceManager::State::Node::equals(const DynamicResourceManager::State::Node & o) const
+{
+    if (type != o.type)
+        return false;
+    return ptr->equals(o.ptr.get());
+}
+
+DynamicResourceManager::Classifier::Classifier(const DynamicResourceManager::StatePtr & state_, const String & classifier_name)
+    : state(state_)
+{
+    // State is immutable, but nodes are mutable and thread-safe
+    // So it's safe to obtain node pointers w/o lock
+    for (auto [resource_name, path] : state->classifiers.get(classifier_name))
+    {
+        if (auto resource_iter = state->resources.find(resource_name); resource_iter != state->resources.end())
+        {
+            const auto & resource = resource_iter->second;
+            if (auto node_iter = resource->nodes.find(path); node_iter != resource->nodes.end())
+            {
+                if (auto * queue = dynamic_cast<ISchedulerQueue *>(node_iter->second.ptr.get()))
+                    resources.emplace(resource_name, ResourceLink{.queue = queue});
+                else
+                    throw Exception(ErrorCodes::RESOURCE_NOT_FOUND, "Unable to access non-queue node at path '{}' for resource '{}'", path, resource_name);
+            }
+            else
+                throw Exception(ErrorCodes::RESOURCE_NOT_FOUND, "Path '{}' for resource '{}' does not exist", path, resource_name);
+        }
+        else
+            resources.emplace(resource_name, ResourceLink{}); // resource not configured yet - use unlimited resource
+    }
+}
+
+ResourceLink DynamicResourceManager::Classifier::get(const String & resource_name)
+{
+    if (auto iter = resources.find(resource_name); iter != resources.end())
+        return iter->second;
+    else
+        throw Exception(ErrorCodes::RESOURCE_ACCESS_DENIED, "Access denied to resource '{}'", resource_name);
+}
+
+DynamicResourceManager::DynamicResourceManager()
+    : state(new State())
+{
+    scheduler.start();
+}
+
+void DynamicResourceManager::updateConfiguration(const Poco::Util::AbstractConfiguration & config)
+{
+    StatePtr new_state = std::make_shared<State>(scheduler.event_queue, config);
+
+    std::lock_guard lock{mutex};
+
+    // Resource update leads to loss of runtime data of nodes and may lead to temporary violation of constraints (e.g. limits)
+    // Try to minimise this by reusing "equal" resources (initialized with the same configuration).
+    for (auto & [name, new_resource] : new_state->resources)
+    {
+        if (auto iter = state->resources.find(name); iter != state->resources.end()) // Resource update
+        {
+            State::ResourcePtr old_resource = iter->second;
+            if (old_resource->equals(*new_resource))
+                new_resource = old_resource; // Rewrite with older version to avoid loss of runtime data
+        }
+    }
+
+    // Commit new state
+    // NOTE: dtor will detach from scheduler old resources that are not in use currently
+    state = new_state;
+
+    // Attach new and updated resources to the scheduler
+    for (auto & [name, resource] : new_state->resources)
+    {
+        const SchedulerNodePtr & root = resource->nodes.find("/")->second.ptr;
+        if (root->parent == nullptr)
+        {
+            resource->attached_to = &scheduler;
+            scheduler.event_queue->enqueue([this, root]
+            {
+                scheduler.attachChild(root);
+            });
+        }
+    }
+
+    // NOTE: after mutex unlock `state` became available for Classifier(s) and must be immutable
+}
+
+ClassifierPtr DynamicResourceManager::acquire(const String & classifier_name)
+{
+    // Acquire a reference to the current state
+    StatePtr state_;
+    {
+        std::lock_guard lock{mutex};
+        state_ = state;
+    }
+
+    return std::make_shared<Classifier>(state_, classifier_name);
+}
+
+void registerDynamicResourceManager(ResourceManagerFactory & factory)
+{
+    factory.registerMethod<DynamicResourceManager>("dynamic");
+}
+
+}
diff --git a/src/IO/Resource/DynamicResourceManager.h b/src/IO/Resource/DynamicResourceManager.h
new file mode 100644
index 00000000000..aa1147f1fb2
--- /dev/null
+++ b/src/IO/Resource/DynamicResourceManager.h
@@ -0,0 +1,93 @@
+#pragma once
+
+#include <IO/IResourceManager.h>
+#include <IO/SchedulerRoot.h>
+#include <IO/Resource/ClassifiersConfig.h>
+
+#include <mutex>
+
+namespace DB
+{
+
+/*
+ * Implementation of `IResourceManager` supporting arbitrary dynamic hierarchy of scheduler nodes.
+ * All resources are controlled by single root `SchedulerRoot`.
+ *
+ * State of manager is set of resources attached to the scheduler. States are referenced by classifiers.
+ * Classifiers are used (1) to access resources and (2) to keep shared ownership of resources with pending
+ * resource requests. This allows `ResourceRequest` and `ResourceLink` to hold raw pointers as long as
+ * `ClassifierPtr` is acquired and held.
+ *
+ * Manager can update configuration after initialization. During update, new version of resources are also
+ * attached to scheduler, so multiple version can coexist for a short perid. This will violate constraints
+ * (e.g. in-fly-limit), because different version have independent nodes to impose constraints, the same
+ * violation will apply to fairness. Old version exists as long as there is at least one classifier
+ * instance referencing it. Classifiers are typically attached to queries and will be destructed with them.
+ */
+class DynamicResourceManager : public IResourceManager
+{
+public:
+    DynamicResourceManager();
+    void updateConfiguration(const Poco::Util::AbstractConfiguration & config) override;
+    ClassifierPtr acquire(const String & classifier_name) override;
+
+private:
+    /// Holds everything required to work with one specific configuration
+    struct State
+    {
+        struct Node
+        {
+            String type;
+            SchedulerNodePtr ptr;
+
+            Node(
+                const String & name,
+                EventQueue * event_queue,
+                const Poco::Util::AbstractConfiguration & config,
+                const std::string & config_prefix);
+            bool equals(const Node & o) const;
+        };
+
+        struct Resource
+        {
+            std::unordered_map<String, Node> nodes; // by path
+            SchedulerRoot * attached_to = nullptr;
+
+            Resource(
+                const String & name,
+                EventQueue * event_queue,
+                const Poco::Util::AbstractConfiguration & config,
+                const std::string & config_prefix);
+            ~Resource(); // unregisters resource from scheduler
+            bool equals(const Resource & o) const;
+        };
+
+        using ResourcePtr = std::shared_ptr<Resource>;
+
+        std::unordered_map<String, ResourcePtr> resources; // by name
+        ClassifiersConfig classifiers;
+
+        State() = default;
+        explicit State(EventQueue * event_queue, const Poco::Util::AbstractConfiguration & config);
+    };
+
+    using StatePtr = std::shared_ptr<State>;
+
+    /// Created per query, holds State used by that query
+    class Classifier : public IClassifier
+    {
+    public:
+        Classifier(const StatePtr & state_, const String & classifier_name);
+        ResourceLink get(const String & resource_name) override;
+    private:
+        std::unordered_map<String, ResourceLink> resources; // accessible resources by names
+        StatePtr state; // hold state to avoid ResourceLink invalidation due to resource deregistration from SchedulerRoot
+    };
+
+private:
+    SchedulerRoot scheduler;
+    std::mutex mutex;
+    StatePtr state;
+};
+
+}
diff --git a/src/IO/Resource/FairPolicy.cpp b/src/IO/Resource/FairPolicy.cpp
new file mode 100644
index 00000000000..248ff04cbd7
--- /dev/null
+++ b/src/IO/Resource/FairPolicy.cpp
@@ -0,0 +1,13 @@
+#include <IO/Resource/FairPolicy.h>
+
+#include <IO/SchedulerNodeFactory.h>
+
+namespace DB
+{
+
+void registerFairPolicy(SchedulerNodeFactory & factory)
+{
+    factory.registerMethod<FairPolicy>("fair");
+}
+
+}
diff --git a/src/IO/Resource/FairPolicy.h b/src/IO/Resource/FairPolicy.h
new file mode 100644
index 00000000000..9c0c78f057c
--- /dev/null
+++ b/src/IO/Resource/FairPolicy.h
@@ -0,0 +1,232 @@
+#pragma once
+
+#include <IO/ISchedulerQueue.h>
+#include <IO/SchedulerRoot.h>
+
+#include <Common/Stopwatch.h>
+
+#include <algorithm>
+#include <unordered_map>
+#include <vector>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int INVALID_SCHEDULER_NODE;
+}
+
+/*
+ * Scheduler node that implements weight-based fair scheduling policy.
+ * Based on Start-time Fair Queueing (SFQ) algorithm.
+ *
+ * Algorithm description.
+ * Virtual runtime (total consumed cost divided by child weight) is tracked for every child.
+ * Active child with minimum vruntime is selected to be dequeued next. On activation, initial vruntime
+ * of a child is set to vruntime of "start" of the last request. This guarantees immediate processing
+ * of at least single request of newly activated children and thus best isolation and scheduling latency.
+ */
+class FairPolicy : public ISchedulerNode
+{
+    /// Scheduling state of a child
+    struct Item
+    {
+        ISchedulerNode * child = nullptr;
+        double vruntime = 0; /// total consumed cost divided by child weight
+
+        /// For min-heap by vruntime
+        bool operator<(const Item & rhs) const noexcept
+        {
+            return vruntime > rhs.vruntime;
+        }
+    };
+
+public:
+    explicit FairPolicy(EventQueue * event_queue_, const Poco::Util::AbstractConfiguration & config = emptyConfig(), const String & config_prefix = {})
+        : ISchedulerNode(event_queue_, config, config_prefix)
+    {}
+
+    bool equals(ISchedulerNode * other) override
+    {
+        if (auto * o = dynamic_cast<FairPolicy *>(other))
+            return true;
+        return false;
+    }
+
+    void attachChild(const SchedulerNodePtr & child) override
+    {
+        // Take ownership
+        if (auto [it, inserted] = children.emplace(child->basename, child); !inserted)
+            throw Exception(
+                ErrorCodes::INVALID_SCHEDULER_NODE,
+                "Can't add another child with the same path: {}",
+                it->second->getPath());
+
+        // Attach
+        child->setParent(this);
+
+        // At first attach as inactive child.
+        // Inactive attached child must have `info.parent.idx` equal it's index inside `items` array.
+        // This is needed to avoid later scanning through inactive `items` in O(N). Important optimization.
+        // NOTE: vruntime must be equal to `system_vruntime` for fairness.
+        child->info.parent.idx = items.size();
+        items.emplace_back(Item{child.get(), system_vruntime});
+
+        // Activate child if it is not empty
+        if (child->isActive())
+            activateChildImpl(items.size() - 1);
+    }
+
+    void removeChild(ISchedulerNode * child) override
+    {
+        if (auto iter = children.find(child->basename); iter != children.end())
+        {
+            SchedulerNodePtr removed = iter->second;
+
+            // Deactivate: detach is not very common operation, so we can afford O(N) here
+            size_t child_idx = 0;
+            [[ maybe_unused ]] bool found = false;
+            for (; child_idx != items.size(); child_idx++)
+            {
+                if (items[child_idx].child == removed.get())
+                {
+                    found = true;
+                    break;
+                }
+            }
+            assert(found);
+            if (child_idx < heap_size) // Detach of active child requires deactivation at first
+            {
+                heap_size--;
+                std::swap(items[child_idx], items[heap_size]);
+                // Element was removed from inside of heap -- heap must be rebuilt
+                std::make_heap(items.begin(), items.begin() + heap_size);
+                child_idx = heap_size;
+            }
+
+            // Now detach inactive child
+            if (child_idx != items.size() - 1)
+            {
+                std::swap(items[child_idx], items.back());
+                items[child_idx].child->info.parent.idx = child_idx;
+            }
+            items.pop_back();
+
+            // Detach
+            removed->setParent(nullptr);
+
+            // Get rid of ownership
+            children.erase(iter);
+        }
+    }
+
+    ISchedulerNode * getChild(const String & child_name) override
+    {
+        if (auto iter = children.find(child_name); iter != children.end())
+            return iter->second.get();
+        else
+            return nullptr;
+    }
+
+    std::pair<ResourceRequest *, bool> dequeueRequest() override
+    {
+        if (heap_size == 0)
+            return {nullptr, false};
+
+        // Recursively pull request from child
+        auto [request, child_active] = items.front().child->dequeueRequest();
+        assert(request != nullptr);
+        std::pop_heap(items.begin(), items.begin() + heap_size);
+        Item & current = items[heap_size - 1];
+
+        // SFQ fairness invariant: system vruntime equals last served request start-time
+        assert(current.vruntime >= system_vruntime);
+        system_vruntime = current.vruntime;
+
+        // By definition vruntime is amount of consumed resource (cost) divided by weight
+        current.vruntime += double(request->cost) / current.child->info.weight;
+        max_vruntime = std::max(max_vruntime, current.vruntime);
+
+        if (child_active) // Put active child back in heap after vruntime update
+        {
+            std::push_heap(items.begin(), items.begin() + heap_size);
+        }
+        else // Deactivate child if it is empty, but remember it's vruntime for latter activations
+        {
+            heap_size--;
+
+            // Store index of this inactive child in `parent.idx`
+            // This enables O(1) search of inactive children instead of O(n)
+            current.child->info.parent.idx = heap_size;
+        }
+
+        // Reset any difference between children on busy period end
+        if (heap_size == 0)
+        {
+            // Reset vtime to zero to avoid floating-point error accumulation,
+            // but do not reset too often, because it's O(N)
+            UInt64 ns = clock_gettime_ns();
+            if (last_reset_ns + 1000000000 < ns)
+            {
+                last_reset_ns = ns;
+                for (Item & item : items)
+                    item.vruntime = 0;
+                max_vruntime = 0;
+            }
+            system_vruntime = max_vruntime;
+        }
+
+        return {request, heap_size > 0};
+    }
+
+    bool isActive() override
+    {
+        return heap_size > 0;
+    }
+
+    void activateChild(ISchedulerNode * child) override
+    {
+        // Find this child; this is O(1), thanks to inactive index we hold in `parent.idx`
+        activateChildImpl(child->info.parent.idx);
+    }
+
+private:
+    void activateChildImpl(size_t inactive_idx)
+    {
+        bool activate_parent = heap_size == 0;
+
+        if (heap_size != inactive_idx)
+        {
+            std::swap(items[heap_size], items[inactive_idx]);
+            items[inactive_idx].child->info.parent.idx = inactive_idx;
+        }
+
+        // Newly activated child should have at least `system_vruntime` to keep fairness
+        items[heap_size].vruntime = std::max(system_vruntime, items[heap_size].vruntime);
+        heap_size++;
+        std::push_heap(items.begin(), items.begin() + heap_size);
+
+        // Recursive activation
+        if (activate_parent && parent)
+            parent->activateChild(this);
+    }
+
+private:
+    /// Beginning of `items` vector is heap of active children: [0; `heap_size`).
+    /// Next go inactive children in unsorted order.
+    /// NOTE: we have to track vruntime of inactive children for max-min fairness.
+    std::vector<Item> items;
+    size_t heap_size = 0;
+
+    /// Last request vruntime
+    double system_vruntime = 0;
+    double max_vruntime = 0;
+    UInt64 last_reset_ns = 0;
+
+    /// All children with ownership
+    std::unordered_map<String, SchedulerNodePtr> children; // basename -> child
+};
+
+}
diff --git a/src/IO/Resource/registerResourceManagers.cpp b/src/IO/Resource/registerResourceManagers.cpp
index 0a394e3f0cd..5217bcdfbec 100644
--- a/src/IO/Resource/registerResourceManagers.cpp
+++ b/src/IO/Resource/registerResourceManagers.cpp
@@ -4,11 +4,13 @@
 namespace DB
 {
 
+void registerDynamicResourceManager(ResourceManagerFactory &);
 void registerStaticResourceManager(ResourceManagerFactory &);
 
 void registerResourceManagers()
 {
     auto & factory = ResourceManagerFactory::instance();
+    registerDynamicResourceManager(factory);
     registerStaticResourceManager(factory);
 }
 
diff --git a/src/IO/Resource/registerSchedulerNodes.cpp b/src/IO/Resource/registerSchedulerNodes.cpp
index 1b58b3981c2..896f96d7f50 100644
--- a/src/IO/Resource/registerSchedulerNodes.cpp
+++ b/src/IO/Resource/registerSchedulerNodes.cpp
@@ -8,6 +8,7 @@ namespace DB
 {
 
 void registerPriorityPolicy(SchedulerNodeFactory &);
+void registerFairPolicy(SchedulerNodeFactory &);
 void registerSemaphoreConstraint(SchedulerNodeFactory &);
 void registerFifoQueue(SchedulerNodeFactory &);
 
@@ -17,6 +18,7 @@ void registerSchedulerNodes()
 
     // ISchedulerNode
     registerPriorityPolicy(factory);
+    registerFairPolicy(factory);
 
     // ISchedulerConstraint
     registerSemaphoreConstraint(factory);
diff --git a/src/IO/Resource/tests/gtest_resource_class_fair.cpp b/src/IO/Resource/tests/gtest_resource_class_fair.cpp
new file mode 100644
index 00000000000..89ec2ac7c32
--- /dev/null
+++ b/src/IO/Resource/tests/gtest_resource_class_fair.cpp
@@ -0,0 +1,187 @@
+#include <gtest/gtest.h>
+
+#include <IO/Resource/tests/ResourceTest.h>
+
+#include <IO/Resource/FairPolicy.h>
+
+using namespace DB;
+
+using ResourceTest = ResourceTestClass;
+
+TEST(IOResourceFairPolicy, Factory)
+{
+    ResourceTest t;
+
+    Poco::AutoPtr cfg = new Poco::Util::XMLConfiguration();
+    SchedulerNodePtr fair = SchedulerNodeFactory::instance().get("fair", /* event_queue = */ nullptr, *cfg, "");
+    EXPECT_TRUE(dynamic_cast<FairPolicy *>(fair.get()) != nullptr);
+}
+
+TEST(IOResourceFairPolicy, FairnessWeights)
+{
+    ResourceTest t;
+
+    t.add<FairPolicy>("/");
+    t.add<FifoQueue>("/A", "<weight>1.0</weight>");
+    t.add<FifoQueue>("/B", "<weight>3.0</weight>");
+
+    t.enqueue("/A", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/B", {10, 10, 10, 10, 10, 10, 10, 10});
+
+    t.dequeue(4);
+    t.consumed("A", 10);
+    t.consumed("B", 30);
+
+    t.dequeue(4);
+    t.consumed("A", 10);
+    t.consumed("B", 30);
+
+    t.dequeue();
+    t.consumed("A", 60);
+    t.consumed("B", 20);
+}
+
+TEST(IOResourceFairPolicy, Activation)
+{
+    ResourceTest t;
+
+    t.add<FairPolicy>("/");
+    t.add<FifoQueue>("/A");
+    t.add<FifoQueue>("/B");
+    t.add<FifoQueue>("/C");
+
+    t.enqueue("/A", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/B", {10});
+    t.enqueue("/C", {10, 10});
+
+    t.dequeue(3);
+    t.consumed("A", 10);
+    t.consumed("B", 10);
+    t.consumed("C", 10);
+
+    t.dequeue(4);
+    t.consumed("A", 30);
+    t.consumed("B", 0);
+    t.consumed("C", 10);
+
+    t.enqueue("/B", {10, 10});
+    t.dequeue(1);
+    t.consumed("B", 10);
+
+    t.enqueue("/C", {10, 10});
+    t.dequeue(1);
+    t.consumed("C", 10);
+
+    t.dequeue(2); // A B or B A
+    t.consumed("A", 10);
+    t.consumed("B", 10);
+}
+
+TEST(IOResourceFairPolicy, FairnessMaxMin)
+{
+    ResourceTest t;
+
+    t.add<FairPolicy>("/");
+    t.add<FifoQueue>("/A");
+    t.add<FifoQueue>("/B");
+
+    t.enqueue("/A", {10, 10}); // make sure A is never empty
+
+    for (int i = 0; i < 10; i++)
+    {
+        t.enqueue("/A", {10, 10, 10, 10});
+        t.enqueue("/B", {10, 10});
+
+        t.dequeue(6);
+        t.consumed("A", 40);
+        t.consumed("B", 20);
+    }
+
+    t.dequeue(2);
+    t.consumed("A", 20);
+}
+
+TEST(IOResourceFairPolicy, HierarchicalFairness)
+{
+    ResourceTest t;
+
+    t.add<FairPolicy>("/");
+    t.add<FairPolicy>("/X");
+    t.add<FairPolicy>("/Y");
+    t.add<FifoQueue>("/X/A");
+    t.add<FifoQueue>("/X/B");
+    t.add<FifoQueue>("/Y/C");
+    t.add<FifoQueue>("/Y/D");
+
+    t.enqueue("/X/A", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/X/B", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/Y/C", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/Y/D", {10, 10, 10, 10, 10, 10, 10, 10});
+    for (int i = 0; i < 4; i++)
+    {
+        t.dequeue(8);
+        t.consumed("A", 20);
+        t.consumed("B", 20);
+        t.consumed("C", 20);
+        t.consumed("D", 20);
+    }
+
+    t.enqueue("/X/A", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/X/A", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/Y/C", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/Y/D", {10, 10, 10, 10, 10, 10, 10, 10});
+    for (int i = 0; i < 4; i++)
+    {
+        t.dequeue(8);
+        t.consumed("A", 40);
+        t.consumed("C", 20);
+        t.consumed("D", 20);
+    }
+
+    t.enqueue("/X/B", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/X/B", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/Y/C", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/Y/D", {10, 10, 10, 10, 10, 10, 10, 10});
+    for (int i = 0; i < 4; i++)
+    {
+        t.dequeue(8);
+        t.consumed("B", 40);
+        t.consumed("C", 20);
+        t.consumed("D", 20);
+    }
+
+    t.enqueue("/X/A", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/X/B", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/Y/C", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/Y/C", {10, 10, 10, 10, 10, 10, 10, 10});
+    for (int i = 0; i < 4; i++)
+    {
+        t.dequeue(8);
+        t.consumed("A", 20);
+        t.consumed("B", 20);
+        t.consumed("C", 40);
+    }
+
+    t.enqueue("/X/A", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/X/B", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/Y/D", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/Y/D", {10, 10, 10, 10, 10, 10, 10, 10});
+    for (int i = 0; i < 4; i++)
+    {
+        t.dequeue(8);
+        t.consumed("A", 20);
+        t.consumed("B", 20);
+        t.consumed("D", 40);
+    }
+
+    t.enqueue("/X/A", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/X/A", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/Y/D", {10, 10, 10, 10, 10, 10, 10, 10});
+    t.enqueue("/Y/D", {10, 10, 10, 10, 10, 10, 10, 10});
+    for (int i = 0; i < 4; i++)
+    {
+        t.dequeue(8);
+        t.consumed("A", 40);
+        t.consumed("D", 40);
+    }
+}
diff --git a/src/IO/Resource/tests/gtest_resource_manager_hierarchical.cpp b/src/IO/Resource/tests/gtest_resource_manager_hierarchical.cpp
new file mode 100644
index 00000000000..b113da31d59
--- /dev/null
+++ b/src/IO/Resource/tests/gtest_resource_manager_hierarchical.cpp
@@ -0,0 +1,116 @@
+#include <gtest/gtest.h>
+
+#include <IO/Resource/tests/ResourceTest.h>
+
+#include <IO/Resource/DynamicResourceManager.h>
+#include <Poco/Util/XMLConfiguration.h>
+
+using namespace DB;
+
+using ResourceTest = ResourceTestManager<DynamicResourceManager>;
+using TestGuard = ResourceTest::Guard;
+
+TEST(IOResourceDynamicResourceManager, Smoke)
+{
+    ResourceTest t;
+
+    t.update(R"CONFIG(
+        <clickhouse>
+            <resources>
+                <res1>
+                    <node path="/"><type>inflight_limit</type><max_requests>10</max_requests></node>
+                    <node path="/fair"><type>fair</type></node>
+                    <node path="/fair/A"><type>fifo</type></node>
+                    <node path="/fair/B"><type>fifo</type><weight>3</weight></node>
+                </res1>
+            </resources>
+            <classifiers>
+                <A><res1>/fair/A</res1></A>
+                <B><res1>/fair/B</res1></B>
+            </classifiers>
+        </clickhouse>
+    )CONFIG");
+
+    ClassifierPtr cA = t.manager->acquire("A");
+    ClassifierPtr cB = t.manager->acquire("B");
+
+    for (int i = 0; i < 10; i++)
+    {
+        ResourceGuard gA(cA->get("res1"), ResourceGuard::PostponeLocking);
+        gA.lock();
+        gA.setFailure();
+        gA.unlock();
+
+        ResourceGuard gB(cB->get("res1"));
+    }
+}
+
+TEST(IOResourceDynamicResourceManager, Fairness)
+{
+    constexpr size_t T = 3; // threads per queue
+    int N = 100; // requests per thread
+    ResourceTest t(2 * T + 1);
+
+    t.update(R"CONFIG(
+        <clickhouse>
+            <resources>
+                <res1>
+                    <node path="/">           <type>inflight_limit</type><max_requests>1</max_requests></node>
+                    <node path="/fair">       <type>fair</type></node>
+                    <node path="/fair/A">     <type>fifo</type></node>
+                    <node path="/fair/B">     <type>fifo</type></node>
+                    <node path="/fair/leader"><type>fifo</type></node>
+                </res1>
+            </resources>
+            <classifiers>
+                <A><res1>/fair/A</res1></A>
+                <B><res1>/fair/B</res1></B>
+                <leader><res1>/fair/leader</res1></leader>
+            </classifiers>
+        </clickhouse>
+    )CONFIG");
+
+
+    // Total cost for A and B cannot differ for more than 1 (every request has cost equal to 1).
+    // Requests from A use `value = 1` and from B `value = -1` is used.
+    std::atomic<Int64> unfairness = 0;
+    auto fairness_diff = [&] (Int64 value)
+    {
+        Int64 cur_unfairness = unfairness.fetch_add(value, std::memory_order_relaxed) + value;
+        EXPECT_NEAR(cur_unfairness, 0, 1);
+    };
+
+    for (int thr = 0; thr < T; thr++)
+    {
+        t.threads.emplace_back([&]
+        {
+            ClassifierPtr c = t.manager->acquire("A");
+            ResourceLink link = c->get("res1");
+            t.startBusyPeriod(link, 1, N);
+            for (int req = 0; req < N; req++)
+            {
+                TestGuard g(t, link, 1);
+                fairness_diff(1);
+            }
+        });
+    }
+
+    for (int thr = 0; thr < T; thr++)
+    {
+        t.threads.emplace_back([&]
+        {
+            ClassifierPtr c = t.manager->acquire("B");
+            ResourceLink link = c->get("res1");
+            t.startBusyPeriod(link, 1, N);
+            for (int req = 0; req < N; req++)
+            {
+                TestGuard g(t, link, 1);
+                fairness_diff(-1);
+            }
+        });
+    }
+
+    ClassifierPtr c = t.manager->acquire("leader");
+    ResourceLink link = c->get("res1");
+    t.blockResource(link);
+}
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index e222b8655aa..b4bdb7cf233 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -1220,7 +1220,7 @@ ResourceManagerPtr Context::getResourceManager() const
 {
     auto lock = getLock();
     if (!shared->resource_manager)
-        shared->resource_manager = ResourceManagerFactory::instance().get(getConfigRef().getString("resource_manager", "static"));
+        shared->resource_manager = ResourceManagerFactory::instance().get(getConfigRef().getString("resource_manager", "dynamic"));
     return shared->resource_manager;
 }
 

From 11b94a626aca1cf346db8c797bbce672a72f0703 Mon Sep 17 00:00:00 2001
From: FFFFFFFHHHHHHH <75292180+FFFFFFFHHHHHHH@users.noreply.github.com>
Date: Tue, 16 May 2023 05:20:29 +0800
Subject: [PATCH 545/689]  Fix aggregate function kolmogorovSmirnovTest
 (#49768)

---
 .../AggregateFunctionKolmogorovSmirnovTest.h    | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.h b/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.h
index 33a9966ee2c..5629de31c88 100644
--- a/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.h
+++ b/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.h
@@ -43,6 +43,7 @@ struct KolmogorovSmirnov : public StatisticalSample<Float64, Float64>
         Float64 now_s = 0;
         UInt64 pos_x = 0;
         UInt64 pos_y = 0;
+        UInt64 pos_tmp;
         UInt64 n1 = x.size();
         UInt64 n2 = y.size();
 
@@ -65,14 +66,22 @@ struct KolmogorovSmirnov : public StatisticalSample<Float64, Float64>
                     now_s -= n2_d;
                     ++pos_y;
                 }
-                max_s = std::max(max_s, now_s);
-                min_s = std::min(min_s, now_s);
             }
             else
             {
-                now_s += n1_d;
-                ++pos_x;
+                pos_tmp = pos_x + 1;
+                while (pos_tmp < x.size() && unlikely(fabs(x[pos_tmp] - x[pos_x]) <= tol))
+                    pos_tmp++;
+                now_s += n1_d * (pos_tmp - pos_x);
+                pos_x = pos_tmp;
+                pos_tmp = pos_y + 1;
+                while (pos_tmp < y.size() && unlikely(fabs(y[pos_tmp] - y[pos_y]) <= tol))
+                    pos_tmp++;
+                now_s -= n2_d * (pos_tmp - pos_y);
+                pos_y = pos_tmp;
             }
+            max_s = std::max(max_s, now_s);
+            min_s = std::min(min_s, now_s);
         }
         now_s += n1_d * (x.size() - pos_x) - n2_d * (y.size() - pos_y);
         min_s = std::min(min_s, now_s);

From 29aa96037758df616ec6253289e218abd7212626 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 16 May 2023 09:07:35 +0200
Subject: [PATCH 546/689] refine docs for regexp tree dictionary

---
 docs/en/sql-reference/dictionaries/index.md | 107 ++++++++++++++++----
 1 file changed, 89 insertions(+), 18 deletions(-)

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index 5801b7866cb..ad5a65df994 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -2197,13 +2197,13 @@ Result:
 └─────────────────────────────────┴───────┘
 ```
 
-## RegExp Tree Dictionary {#regexp-tree-dictionary}
+## RegExpTree Dictionary {#regexp-tree-dictionary}
 
-Regexp Tree dictionary stores multiple trees of regular expressions with attributions. Users can retrieve strings in the dictionary. If a string matches the root of the regexp tree, we will collect the corresponding attributes of the matched root and continue to walk the children. If any of the children matches the string, we will collect attributes and rewrite the old ones if conflicts occur, then continue the traverse until we reach leaf nodes.
+RegExpTree Dictionary is designed to store multiple regular expressions in a dictionary, and query if a string could match one or multiple regular expressions. In some senarioes, for example, User Agent Parser, this data structure is very useful. We can use it in both local and cloud environment.
 
-Example of the ddl query for creating Regexp Tree dictionary:
+### Use RegExpTree Dictionary in local environment
 
-<CloudDetails />
+In local environment, we create RegexpTree dictionary by a yaml file:
 
 ```sql
 create dictionary regexp_dict
@@ -2218,11 +2218,9 @@ LAYOUT(regexp_tree)
 ...
 ```
 
-**Source**
+The dictionary source `YAMLRegExpTree` represents the structure of a Regexp Tree. For example:
 
-We introduce a type of source called `YAMLRegExpTree` representing the structure of Regexp Tree dictionary. An Example of a valid yaml config is like:
-
-```xml
+```yaml
 - regexp: 'Linux/(\d+[\.\d]*).+tlinux'
   name: 'TencentOS'
   version: '\1'
@@ -2240,17 +2238,15 @@ We introduce a type of source called `YAMLRegExpTree` representing the structure
       version: '10'
 ```
 
-The key `regexp` represents the regular expression of a tree node. The name of key is same as the dictionary key. The `name` and `version` is user-defined attributions in the dicitionary. The `versions` (which can be any name that not appear in attributions or the key) indicates the children nodes of this tree.
+This config consists of a list of RegExpTree nodes. Each node has following structure:
 
-**Back Reference**
+- **regexp** means the regular expression of this node.
+- **user defined attributions** is a list of dictionary attributions defined in the dictionary structure. In this case, we have two attributions: `name` and `version`. The first nodes have the both attributions. The second node only have `name` attribution, because the `version` is defined in the children nodes.
+  - The value of an attribution could contain a **back reference** which refers to a capture group of the matched regular expression. Reference number ranges from 1 to 9 and writes as `$1` or `\1`. During the query execution, the back reference in the value will be replaced by the matched capture group.
+- **children nodes** is the secondary layer of the RegExpTree nodes, which also contains a list of RegExpTree nodes. If a string matches a regexp node in the first layer, the dictionary will check if the string matches the children nodes of it. If it matches, we assign the attributions of the matching nodes. If two or more nodes define the same attribution, chilren nodes have more priority.
+  - the name of **children nodes** in yaml files can be arbitrary.
 
-The value of an attribution could contain a back reference which refers to a capture group of the matched regular expression. Reference number ranges from 1 to 9 and writes as `$1` or `\1`.
-
-During the query execution, the back reference in the value will be replaced by the matched capture group.
-
-**Query**
-
-Due to the specialty of Regexp Tree dictionary, we only allow functions `dictGet`, `dictGetOrDefault` and `dictGetOrNull` work with it.
+Due to the specialty of Regexp Tree dictionary, we only allow functions `dictGet`, `dictGetOrDefault` and `dictGetOrNull`.
 
 Example:
 
@@ -2260,12 +2256,87 @@ SELECT dictGet('regexp_dict', ('name', 'version'), '31/tclwebkit1024');
 
 Result:
 
-```
+```text
 ┌─dictGet('regexp_dict', ('name', 'version'), '31/tclwebkit1024')─┐
 │ ('Andriod','12')                                                │
 └─────────────────────────────────────────────────────────────────┘
 ```
 
+Explain:
+
+In this case, we match the regular expression `\d+/tclwebkit(?:\d+[\.\d]*)` in the first layer, so the dictionary will continue to look into the children nodes in the second layer and find it matches `3[12]/tclwebkit`. As a result, the value of `name` is `Andriod` defined in the first layer and the value of `version` is `12` defined in the second layer.
+
+### Use RegExpTree Dictionary on cloud
+
+We have shown how RegExpTree work in the local enviroument, but we cannot use `YAMLRegExpTree` on cloud. If we have a local yaml file, we can use this file to create RegExpTree Dictionary in the local enviroment, then dump this dictionary to a csv file by `dictionary` table function and [INTO OUTFILE](../../statements/select/into-outfile.md) clause.
+
+```sql
+select * from dictionary(regexp_dict) into outfile('regexp_dict.csv')
+```
+
+The content of csv file is:
+
+```text
+1,0,"Linux/(\d+[\.\d]*).+tlinux","['version','name']","['\\1','TencentOS']"
+2,0,"(\d+)/tclwebkit(\d+[\.\d]*)","['comment','version','name']","['test $1 and $2','$1','Andriod']"
+3,2,"33/tclwebkit","['version']","['13']"
+4,2,"3[12]/tclwebkit","['version']","['12']"
+5,2,"3[12]/tclwebkit","['version']","['11']"
+6,2,"3[12]/tclwebkit","['version']","['10']"
+```
+
+The schema of dumped file is always
+
+- `id UInt64` represents the identify number of the RegexpTree node.
+- `parent_id UInt64` represents the id of the parent of a node.
+- `regexp String` represents the regular expression string.
+- `keys Array(String)` represents the names of user defined attributions.
+- `values Array(String)` represents the values of user defined attributions.
+
+On the cloud, we can create a table `regexp_dictionary_source_table` with the above table structure.
+
+```sql
+CREATE TABLE regexp_dictionary_source_table
+(
+    id UInt64,
+    parent_id UInt64,
+    regexp String,
+    keys   Array(String),
+    values Array(String)
+) ENGINE=Memory;
+```
+
+Then update the local CSV by
+
+```bash
+clickhouse client \
+    --host MY_HOST \
+    --secure \
+    --password MY_PASSWORD \
+    --query "
+    insert into regexp_dictionary_source_table 
+    select * from input ('id UInt64, parent_id UInt64, regexp String, keys Array(String), values Array(String)') 
+    FORMAT CSV" < regexp_dict.csv
+```
+
+You can see how to [Insert Local Files](https://clickhouse.com/docs/en/integrations/data-ingestion/insert-local-files) for more details. After we initialize the source table, we can create a RegexpTree by table source:
+
+``` sql
+create dictionary regexp_dict
+(
+    regexp String,
+    name String,
+    version String
+PRIMARY KEY(regexp)
+SOURCE(CLICKHOUSE(TABLE 'regexp_dictionary_source_table'))
+LIFETIME(0)
+LAYOUT(regexp_tree);
+```
+
+### Use RegexpTree Dictionary as a UA Parser
+
+With a powerful yaml configure file, we can use RepexpTree dictionary as a UA parser. We support [uap-core](https://github.com/ua-parser/uap-core) and demostrate how to use it in the functional test [02504_regexp_dictionary_ua_parser](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh)
+
 ## Embedded Dictionaries {#embedded-dictionaries}
 
 <SelfManaged />

From e4e473ef30ff4a6ca3f198e3da6d21b1ff85ccbd Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 16 May 2023 11:22:14 +0200
Subject: [PATCH 547/689] Update docs/en/sql-reference/dictionaries/index.md

Co-authored-by: Sergei Trifonov <svtrifonov@gmail.com>
---
 docs/en/sql-reference/dictionaries/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index ad5a65df994..1ab6370c977 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -2199,7 +2199,7 @@ Result:
 
 ## RegExpTree Dictionary {#regexp-tree-dictionary}
 
-RegExpTree Dictionary is designed to store multiple regular expressions in a dictionary, and query if a string could match one or multiple regular expressions. In some senarioes, for example, User Agent Parser, this data structure is very useful. We can use it in both local and cloud environment.
+RegExpTree Dictionary is designed to store multiple regular expressions in a dictionary, and query if a string could match one or multiple regular expressions. In some scenarios, for example, User Agent Parser, this data structure is very useful. We can use it in both local and cloud environments.
 
 ### Use RegExpTree Dictionary in local environment
 

From 31b8e3c4892ff03340314274ab715506aff0e95b Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 16 May 2023 11:22:24 +0200
Subject: [PATCH 548/689] Update docs/en/sql-reference/dictionaries/index.md

Co-authored-by: Sergei Trifonov <svtrifonov@gmail.com>
---
 docs/en/sql-reference/dictionaries/index.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index 1ab6370c977..b50ad4bf365 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -2241,9 +2241,9 @@ The dictionary source `YAMLRegExpTree` represents the structure of a Regexp Tree
 This config consists of a list of RegExpTree nodes. Each node has following structure:
 
 - **regexp** means the regular expression of this node.
-- **user defined attributions** is a list of dictionary attributions defined in the dictionary structure. In this case, we have two attributions: `name` and `version`. The first nodes have the both attributions. The second node only have `name` attribution, because the `version` is defined in the children nodes.
+- **user defined attributions** is a list of dictionary attributions defined in the dictionary structure. In this case, we have two attributions: `name` and `version`. The first nodes have both attributions. The second node only has `name` attribution, because the `version` is defined in the children nodes.
   - The value of an attribution could contain a **back reference** which refers to a capture group of the matched regular expression. Reference number ranges from 1 to 9 and writes as `$1` or `\1`. During the query execution, the back reference in the value will be replaced by the matched capture group.
-- **children nodes** is the secondary layer of the RegExpTree nodes, which also contains a list of RegExpTree nodes. If a string matches a regexp node in the first layer, the dictionary will check if the string matches the children nodes of it. If it matches, we assign the attributions of the matching nodes. If two or more nodes define the same attribution, chilren nodes have more priority.
+- **children nodes** is the secondary layer of the RegExpTree nodes, which also contains a list of RegExpTree nodes. If a string matches a regexp node in the first layer, the dictionary will check if the string matches the children nodes of it. If it matches, we assign the attributions of the matching nodes. If two or more nodes define the same attribution, children nodes have more priority.
   - the name of **children nodes** in yaml files can be arbitrary.
 
 Due to the specialty of Regexp Tree dictionary, we only allow functions `dictGet`, `dictGetOrDefault` and `dictGetOrNull`.

From ed5906f15ded79823b9b4988cca6b335a6908100 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 16 May 2023 11:22:31 +0200
Subject: [PATCH 549/689] Update docs/en/sql-reference/dictionaries/index.md

Co-authored-by: Sergei Trifonov <svtrifonov@gmail.com>
---
 docs/en/sql-reference/dictionaries/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index b50ad4bf365..086c0b5c0ed 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -2335,7 +2335,7 @@ LAYOUT(regexp_tree);
 
 ### Use RegexpTree Dictionary as a UA Parser
 
-With a powerful yaml configure file, we can use RepexpTree dictionary as a UA parser. We support [uap-core](https://github.com/ua-parser/uap-core) and demostrate how to use it in the functional test [02504_regexp_dictionary_ua_parser](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh)
+With a powerful yaml configure file, we can use RepexpTree dictionary as a UA parser. We support [uap-core](https://github.com/ua-parser/uap-core) and demonstrate how to use it in the functional test [02504_regexp_dictionary_ua_parser](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh)
 
 ## Embedded Dictionaries {#embedded-dictionaries}
 

From a40d86b921ac3a519b7c9d65b41afbdf6667d2b8 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 16 May 2023 11:22:42 +0200
Subject: [PATCH 550/689] Update docs/en/sql-reference/dictionaries/index.md

Co-authored-by: Sergei Trifonov <svtrifonov@gmail.com>
---
 docs/en/sql-reference/dictionaries/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index 086c0b5c0ed..66f661cce60 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -2268,7 +2268,7 @@ In this case, we match the regular expression `\d+/tclwebkit(?:\d+[\.\d]*)` in t
 
 ### Use RegExpTree Dictionary on cloud
 
-We have shown how RegExpTree work in the local enviroument, but we cannot use `YAMLRegExpTree` on cloud. If we have a local yaml file, we can use this file to create RegExpTree Dictionary in the local enviroment, then dump this dictionary to a csv file by `dictionary` table function and [INTO OUTFILE](../../statements/select/into-outfile.md) clause.
+We have shown how RegExpTree work in the local environment, but we cannot use `YAMLRegExpTree` in the cloud. If we have a local yaml file, we can use this file to create RegExpTree Dictionary in the local environment, then dump this dictionary to a csv file by the `dictionary` table function and [INTO OUTFILE](../../statements/select/into-outfile.md) clause.
 
 ```sql
 select * from dictionary(regexp_dict) into outfile('regexp_dict.csv')

From a4f84cecfde581b37a37589239040612e7022935 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 16 May 2023 13:05:36 +0200
Subject: [PATCH 551/689] Retry connection expired in
 test_rename_column/test.py

---
 tests/integration/test_rename_column/test.py | 32 ++++++++++++++++++--
 1 file changed, 29 insertions(+), 3 deletions(-)

diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py
index 6bab0a28259..a664f1050e1 100644
--- a/tests/integration/test_rename_column/test.py
+++ b/tests/integration/test_rename_column/test.py
@@ -225,7 +225,9 @@ def select(
 def rename_column(
     node, table_name, name, new_name, iterations=1, ignore_exception=False
 ):
-    for i in range(iterations):
+    i = 0
+    while True:
+        i += 1
         try:
             node.query(
                 "ALTER TABLE {table_name} RENAME COLUMN {name} to {new_name}".format(
@@ -233,14 +235,22 @@ def rename_column(
                 )
             )
         except QueryRuntimeException as ex:
+            if 'Coordination::Exception' in str(ex):
+                continue
+
             if not ignore_exception:
                 raise
 
+        if i >= iterations:
+            break
+
 
 def rename_column_on_cluster(
     node, table_name, name, new_name, iterations=1, ignore_exception=False
 ):
-    for i in range(iterations):
+    i = 0
+    while True:
+        i += 1
         try:
             node.query(
                 "ALTER TABLE {table_name} ON CLUSTER test_cluster RENAME COLUMN {name} to {new_name}".format(
@@ -248,12 +258,21 @@ def rename_column_on_cluster(
                 )
             )
         except QueryRuntimeException as ex:
+            if 'Coordination::Exception' in str(ex):
+                continue
+
             if not ignore_exception:
                 raise
 
+        if i >= iterations:
+            break
+
+
 
 def alter_move(node, table_name, iterations=1, ignore_exception=False):
-    for i in range(iterations):
+    i = 0
+    while True:
+        i += 1
         move_part = random.randint(0, 99)
         move_volume = "external"
         try:
@@ -263,9 +282,16 @@ def alter_move(node, table_name, iterations=1, ignore_exception=False):
                 )
             )
         except QueryRuntimeException as ex:
+            if 'Coordination::Exception' in str(ex):
+                continue
+
             if not ignore_exception:
                 raise
 
+        if i >= iterations:
+            break
+
+
 
 def test_rename_parallel_same_node(started_cluster):
     table_name = "test_rename_parallel_same_node"

From b3fc7d1a86264bb7c62b6b10026d5701e91753e1 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Tue, 16 May 2023 11:15:15 +0000
Subject: [PATCH 552/689] Automatic style fix

---
 tests/integration/test_rename_column/test.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py
index a664f1050e1..8dc57cf08ff 100644
--- a/tests/integration/test_rename_column/test.py
+++ b/tests/integration/test_rename_column/test.py
@@ -235,7 +235,7 @@ def rename_column(
                 )
             )
         except QueryRuntimeException as ex:
-            if 'Coordination::Exception' in str(ex):
+            if "Coordination::Exception" in str(ex):
                 continue
 
             if not ignore_exception:
@@ -258,7 +258,7 @@ def rename_column_on_cluster(
                 )
             )
         except QueryRuntimeException as ex:
-            if 'Coordination::Exception' in str(ex):
+            if "Coordination::Exception" in str(ex):
                 continue
 
             if not ignore_exception:
@@ -268,7 +268,6 @@ def rename_column_on_cluster(
             break
 
 
-
 def alter_move(node, table_name, iterations=1, ignore_exception=False):
     i = 0
     while True:
@@ -282,7 +281,7 @@ def alter_move(node, table_name, iterations=1, ignore_exception=False):
                 )
             )
         except QueryRuntimeException as ex:
-            if 'Coordination::Exception' in str(ex):
+            if "Coordination::Exception" in str(ex):
                 continue
 
             if not ignore_exception:
@@ -292,7 +291,6 @@ def alter_move(node, table_name, iterations=1, ignore_exception=False):
             break
 
 
-
 def test_rename_parallel_same_node(started_cluster):
     table_name = "test_rename_parallel_same_node"
     drop_table(nodes, table_name)

From d4ea3ea045a250c86859ebfcff953b13822f0a23 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Tue, 16 May 2023 13:49:32 +0200
Subject: [PATCH 553/689] Fix

---
 src/Interpreters/Cache/Metadata.cpp | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp
index a97b10ffbfa..01cdc7f1d1b 100644
--- a/src/Interpreters/Cache/Metadata.cpp
+++ b/src/Interpreters/Cache/Metadata.cpp
@@ -1,6 +1,7 @@
 #include <Interpreters/Cache/Metadata.h>
 #include <Interpreters/Cache/FileCache.h>
 #include <Interpreters/Cache/FileSegment.h>
+#include "Common/Exception.h"
 #include <Common/logger_useful.h>
 #include <filesystem>
 
@@ -257,8 +258,6 @@ void CacheMetadata::doCleanup()
         }
 
         locked_metadata->markAsRemoved();
-        erase(it);
-        LOG_DEBUG(log, "Key {} is removed from metadata", cleanup_key);
 
         try
         {
@@ -272,9 +271,17 @@ void CacheMetadata::doCleanup()
         }
         catch (...)
         {
-            tryLogCurrentException(__PRETTY_FUNCTION__);
+            LOG_ERROR(log, "Error while removing key {}: {}", cleanup_key, getCurrentExceptionMessage(false));
             chassert(false);
         }
+
+        /// Remove key from metadata AFTER deleting key directory, because otherwise key lock is
+        /// released before we delete directory from fs and there might be a race:
+        /// a key, which we just removed, can be added back to cache before we start removing key directory,
+        /// which makes key directory either non-empty (and we get exception in try catch above)
+        /// or we removed directory while another thread thinks it exists.
+        erase(it);
+        LOG_DEBUG(log, "Key {} is removed from metadata", cleanup_key);
     }
 }
 

From 9a2645a72904e1146e98ca58c0274bbe034a79ef Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Tue, 16 May 2023 14:09:38 +0200
Subject: [PATCH 554/689] Fixed clang build

---
 src/Backups/BackupCoordinationLocal.h   | 2 +-
 src/Backups/BackupCoordinationRemote.h  | 2 +-
 src/Backups/BackupsWorker.cpp           | 6 +++---
 src/Backups/IBackupCoordination.h       | 2 +-
 src/Backups/IRestoreCoordination.h      | 2 +-
 src/Backups/RestoreCoordinationLocal.h  | 2 +-
 src/Backups/RestoreCoordinationRemote.h | 2 +-
 7 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h
index a7b05fbb83c..60fcc014720 100644
--- a/src/Backups/BackupCoordinationLocal.h
+++ b/src/Backups/BackupCoordinationLocal.h
@@ -22,7 +22,7 @@ public:
     BackupCoordinationLocal(bool plain_backup_);
     ~BackupCoordinationLocal() override;
 
-    void setStage(const String & new_stage, const String & message = "") override;
+    void setStage(const String & new_stage, const String & message) override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
diff --git a/src/Backups/BackupCoordinationRemote.h b/src/Backups/BackupCoordinationRemote.h
index 5671079fa27..949dd9c9bf0 100644
--- a/src/Backups/BackupCoordinationRemote.h
+++ b/src/Backups/BackupCoordinationRemote.h
@@ -33,7 +33,7 @@ public:
 
     ~BackupCoordinationRemote() override;
 
-    void setStage(const String & new_stage, const String & message = "") override;
+    void setStage(const String & new_stage, const String & message) override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp
index 720ca994a40..0a6482fb7de 100644
--- a/src/Backups/BackupsWorker.cpp
+++ b/src/Backups/BackupsWorker.cpp
@@ -368,7 +368,7 @@ void BackupsWorker::doBackup(
 
             /// Wait until all the hosts have written their backup entries.
             backup_coordination->waitForStage(Stage::COMPLETED);
-            backup_coordination->setStage(Stage::COMPLETED);
+            backup_coordination->setStage(Stage::COMPLETED,"");
         }
         else
         {
@@ -386,7 +386,7 @@ void BackupsWorker::doBackup(
             writeBackupEntries(backup, std::move(backup_entries), backup_id, backup_coordination, backup_settings.internal);
 
             /// We have written our backup entries, we need to tell other hosts (they could be waiting for it).
-            backup_coordination->setStage(Stage::COMPLETED);
+            backup_coordination->setStage(Stage::COMPLETED,"");
         }
 
         size_t num_files = 0;
@@ -709,7 +709,7 @@ void BackupsWorker::doRestore(
 
             /// Wait until all the hosts have written their backup entries.
             restore_coordination->waitForStage(Stage::COMPLETED);
-            restore_coordination->setStage(Stage::COMPLETED);
+            restore_coordination->setStage(Stage::COMPLETED,"");
         }
         else
         {
diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h
index 68a13ab7846..75d9202374b 100644
--- a/src/Backups/IBackupCoordination.h
+++ b/src/Backups/IBackupCoordination.h
@@ -21,7 +21,7 @@ public:
     virtual ~IBackupCoordination() = default;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    virtual void setStage(const String & new_stage, const String & message = "") = 0;
+    virtual void setStage(const String & new_stage, const String & message) = 0;
     virtual void setError(const Exception & exception) = 0;
     virtual Strings waitForStage(const String & stage_to_wait) = 0;
     virtual Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) = 0;
diff --git a/src/Backups/IRestoreCoordination.h b/src/Backups/IRestoreCoordination.h
index b4df9491c4c..2f9e8d171f6 100644
--- a/src/Backups/IRestoreCoordination.h
+++ b/src/Backups/IRestoreCoordination.h
@@ -18,7 +18,7 @@ public:
     virtual ~IRestoreCoordination() = default;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    virtual void setStage(const String & new_stage, const String & message = "") = 0;
+    virtual void setStage(const String & new_stage, const String & message) = 0;
     virtual void setError(const Exception & exception) = 0;
     virtual Strings waitForStage(const String & stage_to_wait) = 0;
     virtual Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) = 0;
diff --git a/src/Backups/RestoreCoordinationLocal.h b/src/Backups/RestoreCoordinationLocal.h
index 2240a25ef3d..e27f0d1ef88 100644
--- a/src/Backups/RestoreCoordinationLocal.h
+++ b/src/Backups/RestoreCoordinationLocal.h
@@ -19,7 +19,7 @@ public:
     ~RestoreCoordinationLocal() override;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    void setStage(const String & new_stage, const String & message = "") override;
+    void setStage(const String & new_stage, const String & message) override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
diff --git a/src/Backups/RestoreCoordinationRemote.h b/src/Backups/RestoreCoordinationRemote.h
index 989b1c1b727..eb0fcff9c2d 100644
--- a/src/Backups/RestoreCoordinationRemote.h
+++ b/src/Backups/RestoreCoordinationRemote.h
@@ -26,7 +26,7 @@ public:
     ~RestoreCoordinationRemote() override;
 
     /// Sets the current stage and waits for other hosts to come to this stage too.
-    void setStage(const String & new_stage, const String & message = "") override;
+    void setStage(const String & new_stage, const String & message) override;
     void setError(const Exception & exception) override;
     Strings waitForStage(const String & stage_to_wait) override;
     Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;

From fd2731845c12d0ee82ebf2fb8b8aada523303581 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sat, 15 Apr 2023 20:43:06 +0200
Subject: [PATCH 555/689] Simplify interface of IBackupWriter: Remove
 supportNativeCopy() function.

---
 src/Backups/BackupIO.cpp                      | 22 +++---
 src/Backups/BackupIO.h                        | 29 +++++---
 src/Backups/BackupIO_Disk.cpp                 | 54 +++++++-------
 src/Backups/BackupIO_Disk.h                   |  6 +-
 src/Backups/BackupIO_File.cpp                 | 47 ++++++------
 src/Backups/BackupIO_File.h                   |  7 +-
 src/Backups/BackupIO_S3.cpp                   | 52 +++++++-------
 src/Backups/BackupIO_S3.h                     | 24 +------
 src/Backups/BackupImpl.cpp                    | 72 ++++++++-----------
 .../test_backup_restore_new/test.py           | 12 ----
 .../test_backup_restore_s3/test.py            |  6 +-
 11 files changed, 146 insertions(+), 185 deletions(-)

diff --git a/src/Backups/BackupIO.cpp b/src/Backups/BackupIO.cpp
index f78e6df23a8..7b269bd965f 100644
--- a/src/Backups/BackupIO.cpp
+++ b/src/Backups/BackupIO.cpp
@@ -4,29 +4,30 @@
 #include <IO/WriteBufferFromFileBase.h>
 #include <IO/SeekableReadBuffer.h>
 #include <Interpreters/Context.h>
+#include <Common/logger_useful.h>
 
 
 namespace DB
 {
 
-namespace ErrorCodes
+IBackupReader::IBackupReader(Poco::Logger * log_) : log(log_)
 {
-    extern const int NOT_IMPLEMENTED;
 }
 
 void IBackupReader::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path,
                                    WriteMode write_mode, const WriteSettings & write_settings)
 {
+    LOG_TRACE(log, "Copying file {} through buffers", file_name);
     auto read_buffer = readFile(file_name);
     auto write_buffer = destination_disk->writeFile(destination_path, std::min<size_t>(size, DBMS_DEFAULT_BUFFER_SIZE), write_mode, write_settings);
     copyData(*read_buffer, *write_buffer, size);
     write_buffer->finalize();
 }
 
-IBackupWriter::IBackupWriter(const ContextPtr & context_)
-    : read_settings(context_->getBackupReadSettings())
-    , has_throttling(static_cast<bool>(context_->getBackupsThrottler()))
-{}
+IBackupWriter::IBackupWriter(const ContextPtr & context_, Poco::Logger * log_)
+    : log(log_), read_settings(context_->getBackupReadSettings())
+{
+}
 
 void IBackupWriter::copyDataToFile(const CreateReadBufferFunction & create_read_buffer, UInt64 offset, UInt64 size, const String & dest_file_name)
 {
@@ -38,9 +39,12 @@ void IBackupWriter::copyDataToFile(const CreateReadBufferFunction & create_read_
     write_buffer->finalize();
 }
 
-void IBackupWriter::copyFileNative(
-    DiskPtr /* src_disk */, const String & /* src_file_name */, UInt64 /* src_offset */, UInt64 /* src_size */, const String & /* dest_file_name */)
+void IBackupWriter::copyFileFromDisk(
+    DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name)
 {
-    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Native copy not implemented for backup writer");
+    LOG_TRACE(log, "Copying file {} through buffers", src_file_name);
+    auto create_read_buffer = [this, src_disk, src_file_name] { return src_disk->readFile(src_file_name, read_settings); };
+    copyDataToFile(create_read_buffer, src_offset, src_size, dest_file_name);
 }
+
 }
diff --git a/src/Backups/BackupIO.h b/src/Backups/BackupIO.h
index aef9c14e83e..dae13422bf2 100644
--- a/src/Backups/BackupIO.h
+++ b/src/Backups/BackupIO.h
@@ -15,6 +15,8 @@ class WriteBuffer;
 class IBackupReader /// BackupReaderFile, BackupReaderDisk
 {
 public:
+    explicit IBackupReader(Poco::Logger * log_);
+
     virtual ~IBackupReader() = default;
     virtual bool fileExists(const String & file_name) = 0;
     virtual UInt64 getFileSize(const String & file_name) = 0;
@@ -22,6 +24,9 @@ public:
     virtual void copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path,
                                 WriteMode write_mode, const WriteSettings & write_settings);
     virtual DataSourceDescription getDataSourceDescription() const = 0;
+
+protected:
+    Poco::Logger * const log;
 };
 
 /// Represents operations of storing to disk or uploading for writing a backup.
@@ -30,29 +35,31 @@ class IBackupWriter /// BackupWriterFile, BackupWriterDisk
 public:
     using CreateReadBufferFunction = std::function<std::unique_ptr<SeekableReadBuffer>()>;
 
-    explicit IBackupWriter(const ContextPtr & context_);
+    IBackupWriter(const ContextPtr & context_, Poco::Logger * log_);
 
     virtual ~IBackupWriter() = default;
     virtual bool fileExists(const String & file_name) = 0;
     virtual UInt64 getFileSize(const String & file_name) = 0;
     virtual bool fileContentsEqual(const String & file_name, const String & expected_file_contents) = 0;
+
     virtual std::unique_ptr<WriteBuffer> writeFile(const String & file_name) = 0;
+
+    virtual void copyDataToFile(const CreateReadBufferFunction & create_read_buffer, UInt64 offset, UInt64 size, const String & dest_file_name);
+
+    /// copyFileFromDisk() can be much faster than copyDataToFile()
+    /// (especially for S3 where it can use CopyObject to copy objects inside S3 instead of downloading and uploading them).
+    virtual void copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name);
+
     virtual void removeFile(const String & file_name) = 0;
     virtual void removeFiles(const Strings & file_names) = 0;
-    virtual DataSourceDescription getDataSourceDescription() const = 0;
-    virtual void copyDataToFile(const CreateReadBufferFunction & create_read_buffer, UInt64 offset, UInt64 size, const String & dest_file_name);
-    virtual bool supportNativeCopy(DataSourceDescription /* data_source_description */) const { return false; }
 
-    /// Copy file using native copy (optimized for S3 to use CopyObject)
-    ///
-    /// NOTE: It still may fall back to copyDataToFile() if native copy is not possible:
-    /// - different buckets
-    /// - throttling had been requested
-    virtual void copyFileNative(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name);
+    virtual DataSourceDescription getDataSourceDescription() const = 0;
 
 protected:
+    Poco::Logger * const log;
+
+    /// These read settings are used to read from the source disk in copyFileFromDisk().
     const ReadSettings read_settings;
-    const bool has_throttling;
 };
 
 }
diff --git a/src/Backups/BackupIO_Disk.cpp b/src/Backups/BackupIO_Disk.cpp
index 10d7a572f6b..643922cf3d0 100644
--- a/src/Backups/BackupIO_Disk.cpp
+++ b/src/Backups/BackupIO_Disk.cpp
@@ -3,6 +3,7 @@
 #include <Disks/IDisk.h>
 #include <IO/ReadBufferFromFileBase.h>
 #include <IO/WriteBufferFromFileBase.h>
+#include <Interpreters/Context.h>
 
 
 namespace DB
@@ -14,7 +15,7 @@ namespace ErrorCodes
 }
 
 BackupReaderDisk::BackupReaderDisk(const DiskPtr & disk_, const String & path_)
-    : disk(disk_), path(path_), log(&Poco::Logger::get("BackupReaderDisk"))
+    : IBackupReader(&Poco::Logger::get("BackupReaderDisk")), disk(disk_), path(path_)
 {
 }
 
@@ -38,22 +39,29 @@ std::unique_ptr<SeekableReadBuffer> BackupReaderDisk::readFile(const String & fi
 void BackupReaderDisk::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path,
                                       WriteMode write_mode, const WriteSettings & write_settings)
 {
-    if (write_mode == WriteMode::Rewrite)
+    if ((write_mode == WriteMode::Rewrite) && (destination_disk->getDataSourceDescription() == getDataSourceDescription()))
     {
-        LOG_TRACE(log, "Copying {}/{} from disk {} to {} by the disk", path, file_name, disk->getName(), destination_disk->getName());
+        /// Use more optimal way.
+        LOG_TRACE(log, "Copying file {} using {} disk", file_name, toString(destination_disk->getDataSourceDescription().type));
         disk->copyFile(path / file_name, *destination_disk, destination_path, write_settings);
         return;
     }
 
-    LOG_TRACE(log, "Copying {}/{} from disk {} to {} through buffers", path, file_name, disk->getName(), destination_disk->getName());
+    /// Fallback to copy through buffers.
     IBackupReader::copyFileToDisk(file_name, size, destination_disk, destination_path, write_mode, write_settings);
 }
 
+DataSourceDescription BackupReaderDisk::getDataSourceDescription() const
+{
+    return disk->getDataSourceDescription();
+}
+
 
 BackupWriterDisk::BackupWriterDisk(const DiskPtr & disk_, const String & path_, const ContextPtr & context_)
-    : IBackupWriter(context_)
+    : IBackupWriter(context_, &Poco::Logger::get("BackupWriterDisk"))
     , disk(disk_)
     , path(path_)
+    , has_throttling(static_cast<bool>(context_->getBackupsThrottler()))
 {
 }
 
@@ -115,31 +123,25 @@ DataSourceDescription BackupWriterDisk::getDataSourceDescription() const
     return disk->getDataSourceDescription();
 }
 
-DataSourceDescription BackupReaderDisk::getDataSourceDescription() const
+void BackupWriterDisk::copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name)
 {
-    return disk->getDataSourceDescription();
-}
-
-bool BackupWriterDisk::supportNativeCopy(DataSourceDescription data_source_description) const
-{
-    return data_source_description == disk->getDataSourceDescription();
-}
-
-void BackupWriterDisk::copyFileNative(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name)
-{
-    if (!src_disk)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot natively copy data to disk without source disk");
-
-    if (has_throttling || (src_offset != 0) || (src_size != src_disk->getFileSize(src_file_name)))
+    /// IDisk::copyFile() can copy to the same disk only, and it cannot do the throttling.
+    if (!has_throttling && (getDataSourceDescription() == src_disk->getDataSourceDescription()))
     {
-        auto create_read_buffer = [this, src_disk, src_file_name] { return src_disk->readFile(src_file_name, read_settings); };
-        copyDataToFile(create_read_buffer, src_offset, src_size, dest_file_name);
-        return;
+        /// IDisk::copyFile() can copy a file as a whole only.
+        if ((src_offset == 0) && (src_size == src_disk->getFileSize(src_file_name)))
+        {
+            /// Use more optimal way.
+            LOG_TRACE(log, "Copying file {} using {} disk", src_file_name, toString(src_disk->getDataSourceDescription().type));
+            auto dest_file_path = path / dest_file_name;
+            disk->createDirectories(dest_file_path.parent_path());
+            src_disk->copyFile(src_file_name, *disk, dest_file_path);
+            return;
+        }
     }
 
-    auto file_path = path / dest_file_name;
-    disk->createDirectories(file_path.parent_path());
-    src_disk->copyFile(src_file_name, *disk, file_path);
+    /// Fallback to copy through buffers.
+    IBackupWriter::copyFileFromDisk(src_disk, src_file_name, src_offset, src_size, dest_file_name);
 }
 
 }
diff --git a/src/Backups/BackupIO_Disk.h b/src/Backups/BackupIO_Disk.h
index be34847000d..884282ea4e0 100644
--- a/src/Backups/BackupIO_Disk.h
+++ b/src/Backups/BackupIO_Disk.h
@@ -25,7 +25,6 @@ public:
 private:
     DiskPtr disk;
     std::filesystem::path path;
-    Poco::Logger * log;
 };
 
 class BackupWriterDisk : public IBackupWriter
@@ -38,16 +37,15 @@ public:
     UInt64 getFileSize(const String & file_name) override;
     bool fileContentsEqual(const String & file_name, const String & expected_file_contents) override;
     std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
+    void copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name) override;
     void removeFile(const String & file_name) override;
     void removeFiles(const Strings & file_names) override;
     DataSourceDescription getDataSourceDescription() const override;
 
-    bool supportNativeCopy(DataSourceDescription data_source_description) const override;
-    void copyFileNative(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name) override;
-
 private:
     DiskPtr disk;
     std::filesystem::path path;
+    const bool has_throttling;
 };
 
 }
diff --git a/src/Backups/BackupIO_File.cpp b/src/Backups/BackupIO_File.cpp
index d4c9d0cb210..b569e65284b 100644
--- a/src/Backups/BackupIO_File.cpp
+++ b/src/Backups/BackupIO_File.cpp
@@ -5,6 +5,7 @@
 #include <IO/copyData.h>
 #include <Common/filesystemHelpers.h>
 #include <Common/logger_useful.h>
+#include <Interpreters/Context.h>
 
 
 namespace fs = std::filesystem;
@@ -12,7 +13,9 @@ namespace fs = std::filesystem;
 
 namespace DB
 {
-BackupReaderFile::BackupReaderFile(const String & path_) : path(path_), log(&Poco::Logger::get("BackupReaderFile"))
+
+BackupReaderFile::BackupReaderFile(const String & path_)
+    : IBackupReader(&Poco::Logger::get("BackupReaderFile")), path(path_)
 {
 }
 
@@ -39,19 +42,20 @@ void BackupReaderFile::copyFileToDisk(const String & file_name, size_t size, Dis
     if (destination_disk->getDataSourceDescription() == getDataSourceDescription())
     {
         /// Use more optimal way.
-        LOG_TRACE(log, "Copying {}/{} to disk {} locally", path, file_name, destination_disk->getName());
+        LOG_TRACE(log, "Copying file {} locally", file_name);
         fs::copy(path / file_name, fullPath(destination_disk, destination_path), fs::copy_options::overwrite_existing);
         return;
     }
 
-    LOG_TRACE(log, "Copying {}/{} to disk {} through buffers", path, file_name, destination_disk->getName());
+    /// Fallback to copy through buffers.
     IBackupReader::copyFileToDisk(path / file_name, size, destination_disk, destination_path, write_mode, write_settings);
 }
 
 
 BackupWriterFile::BackupWriterFile(const String & path_, const ContextPtr & context_)
-    : IBackupWriter(context_)
+    : IBackupWriter(context_, &Poco::Logger::get("BackupWriterFile"))
     , path(path_)
+    , has_throttling(static_cast<bool>(context_->getBackupsThrottler()))
 {
 }
 
@@ -141,29 +145,26 @@ DataSourceDescription BackupReaderFile::getDataSourceDescription() const
 }
 
 
-bool BackupWriterFile::supportNativeCopy(DataSourceDescription data_source_description) const
+void BackupWriterFile::copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name)
 {
-    return data_source_description == getDataSourceDescription();
-}
-
-void BackupWriterFile::copyFileNative(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name)
-{
-    std::string abs_source_path;
-    if (src_disk)
-        abs_source_path = fullPath(src_disk, src_file_name);
-    else
-        abs_source_path = fs::absolute(src_file_name);
-
-    if (has_throttling || (src_offset != 0) || (src_size != fs::file_size(abs_source_path)))
+    /// std::filesystem::copy() can copy from the filesystem only, and it cannot do the throttling.
+    if (!has_throttling && (getDataSourceDescription() == src_disk->getDataSourceDescription()))
     {
-        auto create_read_buffer = [this, abs_source_path] { return createReadBufferFromFileBase(abs_source_path, read_settings); };
-        copyDataToFile(create_read_buffer, src_offset, src_size, dest_file_name);
-        return;
+        std::string abs_source_path = fullPath(src_disk, src_file_name);
+        /// std::filesystem::copy() can copy a file as a whole only.
+        if ((src_offset == 0) && (src_size == fs::file_size(abs_source_path)))
+        {
+            /// Use more optimal way.
+            LOG_TRACE(log, "Copying file {} locally", src_file_name);
+            auto abs_dest_path = path / dest_file_name;
+            fs::create_directories(abs_dest_path.parent_path());
+            fs::copy(abs_source_path, abs_dest_path, fs::copy_options::overwrite_existing);
+            return;
+        }
     }
 
-    auto file_path = path / dest_file_name;
-    fs::create_directories(file_path.parent_path());
-    fs::copy(abs_source_path, file_path, fs::copy_options::overwrite_existing);
+    /// Fallback to copy through buffers.
+    IBackupWriter::copyFileFromDisk(src_disk, src_file_name, src_offset, src_size, dest_file_name);
 }
 
 }
diff --git a/src/Backups/BackupIO_File.h b/src/Backups/BackupIO_File.h
index d4b6e13d546..1f0a247c321 100644
--- a/src/Backups/BackupIO_File.h
+++ b/src/Backups/BackupIO_File.h
@@ -22,27 +22,26 @@ public:
 
 private:
     std::filesystem::path path;
-    Poco::Logger * log;
 };
 
 class BackupWriterFile : public IBackupWriter
 {
 public:
-    explicit BackupWriterFile(const String & path_, const ContextPtr & context_);
+    BackupWriterFile(const String & path_, const ContextPtr & context_);
     ~BackupWriterFile() override;
 
     bool fileExists(const String & file_name) override;
     UInt64 getFileSize(const String & file_name) override;
     bool fileContentsEqual(const String & file_name, const String & expected_file_contents) override;
     std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
+    void copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name) override;
     void removeFile(const String & file_name) override;
     void removeFiles(const Strings & file_names) override;
     DataSourceDescription getDataSourceDescription() const override;
-    bool supportNativeCopy(DataSourceDescription data_source_description) const override;
-    void copyFileNative(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name) override;
 
 private:
     std::filesystem::path path;
+    const bool has_throttling;
 };
 
 }
diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp
index 84dba63ae4e..043075d55ec 100644
--- a/src/Backups/BackupIO_S3.cpp
+++ b/src/Backups/BackupIO_S3.cpp
@@ -102,11 +102,11 @@ namespace
 
 BackupReaderS3::BackupReaderS3(
     const S3::URI & s3_uri_, const String & access_key_id_, const String & secret_access_key_, const ContextPtr & context_)
-    : s3_uri(s3_uri_)
+    : IBackupReader(&Poco::Logger::get("BackupReaderS3"))
+    , s3_uri(s3_uri_)
     , client(makeS3Client(s3_uri_, access_key_id_, secret_access_key_, context_))
     , read_settings(context_->getReadSettings())
     , request_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString()).request_settings)
-    , log(&Poco::Logger::get("BackupReaderS3"))
 {
     request_settings.max_single_read_retries = context_->getSettingsRef().s3_max_single_read_retries; // FIXME: Avoid taking value for endpoint
 }
@@ -141,8 +141,6 @@ std::unique_ptr<SeekableReadBuffer> BackupReaderS3::readFile(const String & file
 void BackupReaderS3::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path,
                                     WriteMode write_mode, const WriteSettings & write_settings)
 {
-    LOG_TRACE(log, "Copying {} to disk {}", file_name, destination_disk->getName());
-
     copyS3FileToDisk(
         client,
         s3_uri.bucket,
@@ -162,11 +160,10 @@ void BackupReaderS3::copyFileToDisk(const String & file_name, size_t size, DiskP
 
 BackupWriterS3::BackupWriterS3(
     const S3::URI & s3_uri_, const String & access_key_id_, const String & secret_access_key_, const ContextPtr & context_)
-    : IBackupWriter(context_)
+    : IBackupWriter(context_, &Poco::Logger::get("BackupWriterS3"))
     , s3_uri(s3_uri_)
     , client(makeS3Client(s3_uri_, access_key_id_, secret_access_key_, context_))
     , request_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString()).request_settings)
-    , log(&Poco::Logger::get("BackupWriterS3"))
 {
     request_settings.updateFromSettings(context_->getSettingsRef());
     request_settings.max_single_read_retries = context_->getSettingsRef().s3_max_single_read_retries; // FIXME: Avoid taking value for endpoint
@@ -177,30 +174,31 @@ DataSourceDescription BackupWriterS3::getDataSourceDescription() const
     return DataSourceDescription{DataSourceType::S3, s3_uri.endpoint, false, false};
 }
 
-bool BackupWriterS3::supportNativeCopy(DataSourceDescription data_source_description) const
+void BackupWriterS3::copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name)
 {
-    return getDataSourceDescription() == data_source_description;
-}
-
-void BackupWriterS3::copyFileNative(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name)
-{
-    if (!src_disk)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot natively copy data to disk without source disk");
-
-    auto objects = src_disk->getStorageObjects(src_file_name);
-    if (objects.size() > 1)
+    /// copyS3File() can copy to another S3 bucket, but it requires the same S3 URI endpoint.
+    /// We don't check `has_throttling` here (compare with BackupWriterDisk::copyFileFromDisk()) because
+    /// copyS3File() almost doesn't use network so the throttling is not needed.
+    if (getDataSourceDescription() == src_disk->getDataSourceDescription())
     {
-        auto create_read_buffer = [this, src_disk, src_file_name] { return src_disk->readFile(src_file_name, read_settings); };
-        copyDataToFile(create_read_buffer, src_offset, src_size, dest_file_name);
-    }
-    else
-    {
-        auto object_storage = src_disk->getObjectStorage();
-        std::string src_bucket = object_storage->getObjectsNamespace();
-        auto file_path = fs::path(s3_uri.key) / dest_file_name;
-        copyS3File(client, src_bucket, objects[0].remote_path, src_offset, src_size, s3_uri.bucket, file_path, request_settings, {},
-                   threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupWriterS3"));
+        /// A single file can be represented as multiple objects in S3 bucket.
+        /// However copyS3File() can copy only a single file into a single file.
+        auto objects = src_disk->getStorageObjects(src_file_name);
+        if (objects.size() == 1)
+        {
+            /// Use more optimal way.
+            LOG_TRACE(log, "Copying file {} using native copy", src_file_name);
+            auto object_storage = src_disk->getObjectStorage();
+            std::string src_bucket = object_storage->getObjectsNamespace();
+            auto file_path = fs::path(s3_uri.key) / dest_file_name;
+            copyS3File(client, src_bucket, objects[0].remote_path, src_offset, src_size, s3_uri.bucket, file_path, request_settings, {},
+                       threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupWriterS3"));
+            return;
+        }
     }
+
+    /// Fallback to BackupWriterS3::copyDataToFile().
+    IBackupWriter::copyFileFromDisk(src_disk, src_file_name, src_offset, src_size, dest_file_name);
 }
 
 void BackupWriterS3::copyDataToFile(
diff --git a/src/Backups/BackupIO_S3.h b/src/Backups/BackupIO_S3.h
index c32a6b48660..7d53d30e8d6 100644
--- a/src/Backups/BackupIO_S3.h
+++ b/src/Backups/BackupIO_S3.h
@@ -32,7 +32,6 @@ private:
     std::shared_ptr<S3::Client> client;
     ReadSettings read_settings;
     S3Settings::RequestSettings request_settings;
-    Poco::Logger * log;
 };
 
 
@@ -48,37 +47,18 @@ public:
     std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
 
     void copyDataToFile(const CreateReadBufferFunction & create_read_buffer, UInt64 offset, UInt64 size, const String & dest_file_name) override;
+    void copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name) override;
+    DataSourceDescription getDataSourceDescription() const override;
 
     void removeFile(const String & file_name) override;
     void removeFiles(const Strings & file_names) override;
 
-    DataSourceDescription getDataSourceDescription() const override;
-    bool supportNativeCopy(DataSourceDescription data_source_description) const override;
-    void copyFileNative(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name) override;
-
 private:
-    void copyObjectImpl(
-        const String & src_bucket,
-        const String & src_key,
-        const String & dst_bucket,
-        const String & dst_key,
-        size_t size,
-        const std::optional<ObjectAttributes> & metadata = std::nullopt) const;
-
-    void copyObjectMultipartImpl(
-        const String & src_bucket,
-        const String & src_key,
-        const String & dst_bucket,
-        const String & dst_key,
-        size_t size,
-        const std::optional<ObjectAttributes> & metadata = std::nullopt) const;
-
     void removeFilesBatch(const Strings & file_names);
 
     S3::URI s3_uri;
     std::shared_ptr<S3::Client> client;
     S3Settings::RequestSettings request_settings;
-    Poco::Logger * log;
     std::optional<bool> supports_batch_delete;
 };
 
diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp
index a8c6e16b268..55fb6dbfe03 100644
--- a/src/Backups/BackupImpl.cpp
+++ b/src/Backups/BackupImpl.cpp
@@ -805,72 +805,56 @@ void BackupImpl::writeFile(const BackupFileInfo & info, BackupEntryPtr entry)
     if (writing_finalized)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Backup is already finalized");
 
-    std::string from_file_name = "memory buffer";
-    if (auto fname = entry->getFilePath(); !fname.empty())
-        from_file_name = "file " + fname;
-
+    bool should_check_lock_file = false;
     {
         std::lock_guard lock{mutex};
         ++num_files;
         total_size += info.size;
+        if (!num_entries)
+            should_check_lock_file = true;
     }
 
+    auto src_disk = entry->tryGetDiskIfExists();
+    auto src_file_path = entry->getFilePath();
+    String src_file_desc = src_file_path.empty() ? "memory buffer" : ("file " + src_file_path);
+
     if (info.data_file_name.empty())
     {
-        LOG_TRACE(log, "Writing backup for file {} from {}: skipped, {}", info.data_file_name, from_file_name, !info.size ? "empty" : "base backup has it");
+        LOG_TRACE(log, "Writing backup for file {} from {}: skipped, {}", info.data_file_name, src_file_desc, !info.size ? "empty" : "base backup has it");
         return;
     }
 
     if (!coordination->startWritingFile(info.data_file_index))
     {
-        LOG_TRACE(log, "Writing backup for file {} from {}: skipped, data file #{} is already being written", info.data_file_name, from_file_name, info.data_file_index);
+        LOG_TRACE(log, "Writing backup for file {} from {}: skipped, data file #{} is already being written", info.data_file_name, src_file_desc, info.data_file_index);
         return;
     }
 
-    LOG_TRACE(log, "Writing backup for file {} from {}: data file #{}", info.data_file_name, from_file_name, info.data_file_index);
+    if (!should_check_lock_file)
+        checkLockFile(true);
 
-    auto writer_description = writer->getDataSourceDescription();
-    auto reader_description = entry->getDataSourceDescription();
+    /// NOTE: `mutex` must be unlocked during copying otherwise writing will be in one thread maximum and hence slow.
 
-    /// We need to copy whole file without archive, we can do it faster
-    /// if source and destination are compatible
-    if (!use_archive && writer->supportNativeCopy(reader_description))
+    if (use_archive)
     {
-        /// Should be much faster than writing data through server.
-        LOG_TRACE(log, "Will copy file {} using native copy", info.data_file_name);
-
-        /// NOTE: `mutex` must be unlocked here otherwise writing will be in one thread maximum and hence slow.
-
-        writer->copyFileNative(entry->tryGetDiskIfExists(), entry->getFilePath(), info.base_size, info.size - info.base_size, info.data_file_name);
+        LOG_TRACE(log, "Writing backup for file {} from {}: data file #{}, adding to archive", info.data_file_name, src_file_desc, info.data_file_index);
+        auto out = archive_writer->writeFile(info.data_file_name);
+        auto read_buffer = entry->getReadBuffer();
+        if (info.base_size != 0)
+            read_buffer->seek(info.base_size, SEEK_SET);
+        copyData(*read_buffer, *out);
+        out->finalize();
+    }
+    else if (src_disk)
+    {
+        LOG_TRACE(log, "Writing backup for file {} from {} (disk {}): data file #{}", info.data_file_name, src_file_desc, src_disk->getName(), info.data_file_index);
+        writer->copyFileFromDisk(src_disk, src_file_path, info.base_size, info.size - info.base_size, info.data_file_name);
     }
     else
     {
-        bool has_entries = false;
-        {
-            std::lock_guard lock{mutex};
-            has_entries = num_entries > 0;
-        }
-        if (!has_entries)
-            checkLockFile(true);
-
-        if (use_archive)
-        {
-            LOG_TRACE(log, "Adding file {} to archive", info.data_file_name);
-            auto out = archive_writer->writeFile(info.data_file_name);
-            auto read_buffer = entry->getReadBuffer();
-            if (info.base_size != 0)
-                read_buffer->seek(info.base_size, SEEK_SET);
-            copyData(*read_buffer, *out);
-            out->finalize();
-        }
-        else
-        {
-            LOG_TRACE(log, "Will copy file {}", info.data_file_name);
-            auto create_read_buffer = [entry] { return entry->getReadBuffer(); };
-
-            /// NOTE: `mutex` must be unlocked here otherwise writing will be in one thread maximum and hence slow.
-            writer->copyDataToFile(create_read_buffer, info.base_size, info.size - info.base_size, info.data_file_name);
-        }
+        LOG_TRACE(log, "Writing backup for file {} from {}: data file #{}", info.data_file_name, src_file_desc, info.data_file_index);
+        auto create_read_buffer = [entry] { return entry->getReadBuffer(); };
+        writer->copyDataToFile(create_read_buffer, info.base_size, info.size - info.base_size, info.data_file_name);
     }
 
     {
diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py
index 18f6ada6821..c19cca4126a 100644
--- a/tests/integration/test_backup_restore_new/test.py
+++ b/tests/integration/test_backup_restore_new/test.py
@@ -158,8 +158,6 @@ def test_restore_table(engine):
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
     instance.query(f"BACKUP TABLE test.table TO {backup_name}")
 
-    assert instance.contains_in_log("using native copy")
-
     instance.query("DROP TABLE test.table")
     assert instance.query("EXISTS test.table") == "0\n"
 
@@ -200,8 +198,6 @@ def test_restore_table_under_another_name():
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
     instance.query(f"BACKUP TABLE test.table TO {backup_name}")
 
-    assert instance.contains_in_log("using native copy")
-
     assert instance.query("EXISTS test.table2") == "0\n"
 
     instance.query(f"RESTORE TABLE test.table AS test.table2 FROM {backup_name}")
@@ -215,8 +211,6 @@ def test_backup_table_under_another_name():
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
     instance.query(f"BACKUP TABLE test.table AS test.table2 TO {backup_name}")
 
-    assert instance.contains_in_log("using native copy")
-
     assert instance.query("EXISTS test.table2") == "0\n"
 
     instance.query(f"RESTORE TABLE test.table2 FROM {backup_name}")
@@ -245,8 +239,6 @@ def test_incremental_backup():
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
     instance.query(f"BACKUP TABLE test.table TO {backup_name}")
 
-    assert instance.contains_in_log("using native copy")
-
     instance.query("INSERT INTO test.table VALUES (65, 'a'), (66, 'b')")
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "102\t5081\n"
@@ -524,8 +516,6 @@ def test_file_engine():
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
     instance.query(f"BACKUP TABLE test.table TO {backup_name}")
 
-    assert instance.contains_in_log("using native copy")
-
     instance.query("DROP TABLE test.table")
     assert instance.query("EXISTS test.table") == "0\n"
 
@@ -540,8 +530,6 @@ def test_database():
 
     instance.query(f"BACKUP DATABASE test TO {backup_name}")
 
-    assert instance.contains_in_log("using native copy")
-
     instance.query("DROP DATABASE test")
     instance.query(f"RESTORE DATABASE test FROM {backup_name}")
 
diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py
index e42754dd2a3..6e9b6b8569e 100644
--- a/tests/integration/test_backup_restore_s3/test.py
+++ b/tests/integration/test_backup_restore_s3/test.py
@@ -141,7 +141,7 @@ def test_backup_to_s3_native_copy():
         f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', 'minio123')"
     )
     check_backup_and_restore(storage_policy, backup_destination)
-    assert node.contains_in_log("BackupImpl.*using native copy")
+    assert node.contains_in_log("BackupWriterS3.*using native copy")
     assert node.contains_in_log("copyS3FileToDisk.*using native copy")
     assert node.contains_in_log(
         f"copyS3File: Single operation copy has completed. Bucket: root, Key: data/backups/{backup_name}"
@@ -155,7 +155,7 @@ def test_backup_to_s3_native_copy_other_bucket():
         f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', 'minio123')"
     )
     check_backup_and_restore(storage_policy, backup_destination)
-    assert node.contains_in_log("BackupImpl.*using native copy")
+    assert node.contains_in_log("BackupWriterS3.*using native copy")
     assert node.contains_in_log("copyS3FileToDisk.*using native copy")
     assert node.contains_in_log(
         f"copyS3File: Single operation copy has completed. Bucket: root, Key: data/backups/{backup_name}"
@@ -167,7 +167,7 @@ def test_backup_to_s3_native_copy_multipart():
     backup_name = new_backup_name()
     backup_destination = f"S3('http://minio1:9001/root/data/backups/multipart/{backup_name}', 'minio', 'minio123')"
     check_backup_and_restore(storage_policy, backup_destination, size=1000000)
-    assert node.contains_in_log("BackupImpl.*using native copy")
+    assert node.contains_in_log("BackupWriterS3.*using native copy")
     assert node.contains_in_log("copyS3FileToDisk.*using native copy")
     assert node.contains_in_log(
         f"copyS3File: Multipart upload has completed. Bucket: root, Key: data/backups/multipart/{backup_name}/"

From 69114cb550527443e555875a91f9fe1c2731e532 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sat, 22 Apr 2023 13:48:03 +0200
Subject: [PATCH 556/689] Add function getBlobPath() to IDisk interface to
 allow copying to/from disks which are not built on top of IObjectStorage.

---
 src/Backups/BackupIO_S3.cpp                    | 14 ++++++--------
 src/Disks/DiskEncrypted.h                      | 12 ++++++++++++
 src/Disks/DiskLocal.cpp                        | 11 +++++++++++
 src/Disks/DiskLocal.h                          |  3 +++
 src/Disks/FakeDiskTransaction.h                |  8 ++------
 src/Disks/IDisk.cpp                            | 10 ----------
 src/Disks/IDisk.h                              | 16 ++++++++++------
 src/Disks/IDiskTransaction.h                   |  8 +++-----
 src/Disks/ObjectStorages/DiskObjectStorage.cpp | 16 ++++++++++------
 src/Disks/ObjectStorages/DiskObjectStorage.h   |  7 ++-----
 .../DiskObjectStorageTransaction.cpp           | 11 +++++------
 .../DiskObjectStorageTransaction.h             |  6 +-----
 .../ObjectStorages/S3/copyS3FileToDisk.cpp     | 18 ++++++++++++++----
 13 files changed, 79 insertions(+), 61 deletions(-)

diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp
index 043075d55ec..901e9de7576 100644
--- a/src/Backups/BackupIO_S3.cpp
+++ b/src/Backups/BackupIO_S3.cpp
@@ -181,17 +181,15 @@ void BackupWriterS3::copyFileFromDisk(DiskPtr src_disk, const String & src_file_
     /// copyS3File() almost doesn't use network so the throttling is not needed.
     if (getDataSourceDescription() == src_disk->getDataSourceDescription())
     {
-        /// A single file can be represented as multiple objects in S3 bucket.
-        /// However copyS3File() can copy only a single file into a single file.
-        auto objects = src_disk->getStorageObjects(src_file_name);
-        if (objects.size() == 1)
+        /// getBlobPath() can return std::nullopt if the file is stored as multiple objects in S3 bucket.
+        /// In this case we can't use the native copy.
+        if (auto blob_path = src_disk->getBlobPath(src_file_name))
         {
             /// Use more optimal way.
             LOG_TRACE(log, "Copying file {} using native copy", src_file_name);
-            auto object_storage = src_disk->getObjectStorage();
-            std::string src_bucket = object_storage->getObjectsNamespace();
-            auto file_path = fs::path(s3_uri.key) / dest_file_name;
-            copyS3File(client, src_bucket, objects[0].remote_path, src_offset, src_size, s3_uri.bucket, file_path, request_settings, {},
+            const auto & [src_bucket, src_key] = *blob_path;
+            auto dest_key = fs::path(s3_uri.key) / dest_file_name;
+            copyS3File(client, src_bucket, src_key, src_offset, src_size, s3_uri.bucket, dest_key, request_settings, {},
                        threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupWriterS3"));
             return;
         }
diff --git a/src/Disks/DiskEncrypted.h b/src/Disks/DiskEncrypted.h
index 8e824a1f7e5..d9da320e505 100644
--- a/src/Disks/DiskEncrypted.h
+++ b/src/Disks/DiskEncrypted.h
@@ -131,6 +131,18 @@ public:
         WriteMode mode,
         const WriteSettings & settings) override;
 
+    std::optional<std::pair<String, String>> getBlobPath(const String & path) const override
+    {
+        auto wrapped_path = wrappedPath(path);
+        return delegate->getBlobPath(wrapped_path);
+    }
+
+    void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) override
+    {
+        auto wrapped_path = wrappedPath(path);
+        delegate->writeFileUsingBlobWritingFunction(wrapped_path, mode, std::move(write_blob_function));
+    }
+
     void removeFile(const String & path) override
     {
         auto wrapped_path = wrappedPath(path);
diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp
index 1abecb7af4e..a40368fae88 100644
--- a/src/Disks/DiskLocal.cpp
+++ b/src/Disks/DiskLocal.cpp
@@ -328,6 +328,17 @@ DiskLocal::writeFile(const String & path, size_t buf_size, WriteMode mode, const
         fs::path(disk_path) / path, buf_size, flags, settings.local_throttler);
 }
 
+std::optional<std::pair<String, String>> DiskLocal::getBlobPath(const String & path) const
+{
+    return std::make_pair(fs::path(disk_path) / path, "");
+}
+
+void DiskLocal::writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function)
+{
+    auto blob_path = std::make_pair(fs::path(disk_path) / path, "");
+    std::move(write_blob_function)(blob_path, mode, {});
+}
+
 void DiskLocal::removeFile(const String & path)
 {
     auto fs_path = fs::path(disk_path) / path;
diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h
index 3112198aab3..b838654925d 100644
--- a/src/Disks/DiskLocal.h
+++ b/src/Disks/DiskLocal.h
@@ -81,6 +81,9 @@ public:
         WriteMode mode,
         const WriteSettings & settings) override;
 
+    std::optional<std::pair<String, String>> getBlobPath(const String & path) const override;
+    void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) override;
+
     void removeFile(const String & path) override;
     void removeFileIfExists(const String & path) override;
     void removeDirectory(const String & path) override;
diff --git a/src/Disks/FakeDiskTransaction.h b/src/Disks/FakeDiskTransaction.h
index 3a7ea4473b6..2cf540444be 100644
--- a/src/Disks/FakeDiskTransaction.h
+++ b/src/Disks/FakeDiskTransaction.h
@@ -69,13 +69,9 @@ public:
         return disk.writeFile(path, buf_size, mode, settings);
     }
 
-    void writeFileUsingCustomWriteObject(
-        const String & path,
-        WriteMode mode,
-        std::function<size_t(const StoredObject & object, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>
-            custom_write_object_function) override
+    void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) override
     {
-        disk.writeFileUsingCustomWriteObject(path, mode, std::move(custom_write_object_function));
+        disk.writeFileUsingBlobWritingFunction(path, mode, std::move(write_blob_function));
     }
 
     void removeFile(const std::string & path) override
diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp
index 22197760d88..9f39c8242f5 100644
--- a/src/Disks/IDisk.cpp
+++ b/src/Disks/IDisk.cpp
@@ -35,16 +35,6 @@ void IDisk::copyFile(const String & from_file_path, IDisk & to_disk, const Strin
     out->finalize();
 }
 
-void IDisk::writeFileUsingCustomWriteObject(
-    const String &, WriteMode, std::function<size_t(const StoredObject &, WriteMode, const std::optional<ObjectAttributes> &)>)
-{
-    throw Exception(
-        ErrorCodes::NOT_IMPLEMENTED,
-        "Method `writeFileUsingCustomWriteObject()` is not implemented for disk: {}",
-        getDataSourceDescription().type);
-}
-
-
 DiskTransactionPtr IDisk::createTransaction()
 {
     return std::make_shared<FakeDiskTransaction>(*this);
diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h
index b0a57afe169..238f2258c3d 100644
--- a/src/Disks/IDisk.h
+++ b/src/Disks/IDisk.h
@@ -209,14 +209,18 @@ public:
         WriteMode mode = WriteMode::Rewrite,
         const WriteSettings & settings = {}) = 0;
 
-    /// Write a file using a custom function to write an object to the disk's object storage.
+    /// Returns the path to a blob representing a specified file.
+    /// The meaning of the returned path depends on disk's type.
+    /// For DiskLocal it the absolute path to the file and for DiskObjectStorage it's the name of a namespace
+    /// combined with StoredObject::absolute_path.
+    virtual std::optional<std::pair<String, String>> getBlobPath(const String & path) const = 0;
+
+    using WriteBlobFunction = std::function<size_t(const std::pair<String, String> & blob_path, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>;
+
+    /// Write a file using a custom function to write a blob representing the file.
     /// This method is alternative to writeFile(), the difference is that writeFile() calls IObjectStorage::writeObject()
     /// to write an object to the object storage while this method allows to specify a callback for that.
-    virtual void writeFileUsingCustomWriteObject(
-        const String & path,
-        WriteMode mode,
-        std::function<size_t(const StoredObject & object, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>
-            custom_write_object_function);
+    virtual void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) = 0;
 
     /// Remove file. Throws exception if file doesn't exists or it's a directory.
     /// Return whether file was finally removed. (For remote disks it is not always removed).
diff --git a/src/Disks/IDiskTransaction.h b/src/Disks/IDiskTransaction.h
index 2edbe858c06..376d7bd78e6 100644
--- a/src/Disks/IDiskTransaction.h
+++ b/src/Disks/IDiskTransaction.h
@@ -68,12 +68,10 @@ public:
         const WriteSettings & settings = {},
         bool autocommit = true) = 0;
 
+    using WriteBlobFunction = std::function<size_t(const std::pair<String, String> & blob_path, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>;
+
     /// Write a file using a custom function to write an object to the disk's object storage.
-    virtual void writeFileUsingCustomWriteObject(
-        const String & path,
-        WriteMode mode,
-        std::function<size_t(const StoredObject & object, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>
-            custom_write_object_function) = 0;
+    virtual void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) = 0;
 
     /// Remove file. Throws exception if file doesn't exists or it's a directory.
     virtual void removeFile(const std::string & path) = 0;
diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp
index f832ba5b7b6..b01f1b327e7 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp
@@ -579,15 +579,19 @@ std::unique_ptr<WriteBufferFromFileBase> DiskObjectStorage::writeFile(
     return result;
 }
 
-void DiskObjectStorage::writeFileUsingCustomWriteObject(
-    const String & path,
-    WriteMode mode,
-    std::function<size_t(const StoredObject & object, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>
-        custom_write_object_function)
+std::optional<std::pair<String, String>> DiskObjectStorage::getBlobPath(const String & path) const
+{
+    auto objects = getStorageObjects(path);
+    if (objects.size() != 1)
+        return {};
+    return std::make_pair(object_storage->getObjectsNamespace(), objects[0].absolute_path);
+}
+
+void DiskObjectStorage::writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function)
 {
     LOG_TEST(log, "Write file: {}", path);
     auto transaction = createObjectStorageTransaction();
-    return transaction->writeFileUsingCustomWriteObject(path, mode, std::move(custom_write_object_function));
+    return transaction->writeFileUsingBlobWritingFunction(path, mode, std::move(write_blob_function));
 }
 
 void DiskObjectStorage::applyNewSettings(
diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h
index 4372bc75950..97751edc3f5 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorage.h
+++ b/src/Disks/ObjectStorages/DiskObjectStorage.h
@@ -149,11 +149,8 @@ public:
         WriteMode mode,
         const WriteSettings & settings) override;
 
-    void writeFileUsingCustomWriteObject(
-        const String & path,
-        WriteMode mode,
-        std::function<size_t(const StoredObject & object, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>
-            custom_write_object_function) override;
+    std::optional<std::pair<String, String>> getBlobPath(const String & path) const override;
+    void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) override;
 
     void copy(const String & from_path, const std::shared_ptr<IDisk> & to_disk, const String & to_path) override;
 
diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
index 6cafc35f8fa..f578bcb9772 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
+++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
@@ -669,11 +669,8 @@ std::unique_ptr<WriteBufferFromFileBase> DiskObjectStorageTransaction::writeFile
 }
 
 
-void DiskObjectStorageTransaction::writeFileUsingCustomWriteObject(
-    const String & path,
-    WriteMode mode,
-    std::function<size_t(const StoredObject & object, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>
-        custom_write_object_function)
+void DiskObjectStorageTransaction::writeFileUsingBlobWritingFunction(
+    const String & path, WriteMode mode, WriteBlobFunction && write_blob_function)
 {
     /// This function is a simplified and adapted version of DiskObjectStorageTransaction::writeFile().
     auto blob_name = object_storage.generateBlobNameForPath(path);
@@ -694,8 +691,10 @@ void DiskObjectStorageTransaction::writeFileUsingCustomWriteObject(
 
     operations_to_execute.emplace_back(std::move(write_operation));
 
+    auto blob_path = std::make_pair(object_storage.getObjectsNamespace(), object.absolute_path);
+
     /// We always use mode Rewrite because we simulate append using metadata and different files
-    size_t object_size = std::move(custom_write_object_function)(object, WriteMode::Rewrite, object_attributes);
+    size_t object_size = std::move(write_blob_function)(blob_path, WriteMode::Rewrite, object_attributes);
 
     /// Create metadata (see create_metadata_callback in DiskObjectStorageTransaction::writeFile()).
     if (mode == WriteMode::Rewrite)
diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h
index 080a3e42057..a4cb0ed3739 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h
+++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h
@@ -100,11 +100,7 @@ public:
         bool autocommit = true) override;
 
     /// Write a file using a custom function to write an object to the disk's object storage.
-    void writeFileUsingCustomWriteObject(
-        const String & path,
-        WriteMode mode,
-        std::function<size_t(const StoredObject & object, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>
-            custom_write_object_function) override;
+    void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) override;
 
     void removeFile(const std::string & path) override;
     void removeFileIfExists(const std::string & path) override;
diff --git a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp
index f89415f789a..dea3266ad7a 100644
--- a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp
+++ b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp
@@ -50,18 +50,28 @@ void copyS3FileToDisk(
 
     String dest_bucket = destination_disk->getObjectStorage()->getObjectsNamespace();
 
-    auto custom_write_object = [&](const StoredObject & object_, WriteMode write_mode_, const std::optional<ObjectAttributes> & object_attributes_) -> size_t
+    auto write_blob_function = [&](const std::pair<String, String> & blob_path_, WriteMode write_mode_, const std::optional<ObjectAttributes> & object_attributes_) -> size_t
     {
         /// Object storage always uses mode `Rewrite` because it simulates append using metadata and different files.
         chassert(write_mode_ == WriteMode::Rewrite);
 
-        copyS3File(s3_client, src_bucket, src_key, *src_offset, *src_size, dest_bucket, /* dest_key= */ object_.remote_path,
-                   request_settings, object_attributes_, scheduler, /* for_disk_s3= */ true);
+        copyS3File(
+            s3_client,
+            src_bucket,
+            src_key,
+            *src_offset,
+            *src_size,
+            /* dest_bucket= */ blob_path_.first,
+            /* dest_key= */ blob_path_.second,
+            request_settings,
+            object_attributes_,
+            scheduler,
+            /* for_disk_s3= */ true);
 
         return *src_size;
     };
 
-    destination_disk->writeFileUsingCustomWriteObject(destination_path, write_mode, custom_write_object);
+    destination_disk->writeFileUsingBlobWritingFunction(destination_path, write_mode, write_blob_function);
 }
 
 }

From 101aa6eff09a06f3751f3f06c7fa22afb4d93134 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sat, 22 Apr 2023 17:51:14 +0200
Subject: [PATCH 557/689] Add function copyS3FileFromDisk().

---
 src/Backups/BackupIO_S3.cpp                   |  24 +----
 .../ObjectStorages/S3/copyS3FileToDisk.cpp    | 102 ++++++++++++------
 .../ObjectStorages/S3/copyS3FileToDisk.h      |  20 +++-
 .../test_backup_restore_s3/test.py            |   6 +-
 4 files changed, 94 insertions(+), 58 deletions(-)

diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp
index 901e9de7576..d2861500159 100644
--- a/src/Backups/BackupIO_S3.cpp
+++ b/src/Backups/BackupIO_S3.cpp
@@ -176,27 +176,9 @@ DataSourceDescription BackupWriterS3::getDataSourceDescription() const
 
 void BackupWriterS3::copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name)
 {
-    /// copyS3File() can copy to another S3 bucket, but it requires the same S3 URI endpoint.
-    /// We don't check `has_throttling` here (compare with BackupWriterDisk::copyFileFromDisk()) because
-    /// copyS3File() almost doesn't use network so the throttling is not needed.
-    if (getDataSourceDescription() == src_disk->getDataSourceDescription())
-    {
-        /// getBlobPath() can return std::nullopt if the file is stored as multiple objects in S3 bucket.
-        /// In this case we can't use the native copy.
-        if (auto blob_path = src_disk->getBlobPath(src_file_name))
-        {
-            /// Use more optimal way.
-            LOG_TRACE(log, "Copying file {} using native copy", src_file_name);
-            const auto & [src_bucket, src_key] = *blob_path;
-            auto dest_key = fs::path(s3_uri.key) / dest_file_name;
-            copyS3File(client, src_bucket, src_key, src_offset, src_size, s3_uri.bucket, dest_key, request_settings, {},
-                       threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupWriterS3"));
-            return;
-        }
-    }
-
-    /// Fallback to BackupWriterS3::copyDataToFile().
-    IBackupWriter::copyFileFromDisk(src_disk, src_file_name, src_offset, src_size, dest_file_name);
+    copyS3FileFromDisk(src_disk, src_file_name, src_offset, src_size,
+        client, s3_uri.bucket, fs::path(s3_uri.key) / dest_file_name, read_settings, request_settings,
+        threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupWriterS3"));
 }
 
 void BackupWriterS3::copyDataToFile(
diff --git a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp
index dea3266ad7a..e43d88b2519 100644
--- a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp
+++ b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp
@@ -13,7 +13,7 @@ namespace DB
 {
 
 void copyS3FileToDisk(
-    const std::shared_ptr<const S3::Client> & s3_client,
+    const std::shared_ptr<const S3::Client> & src_s3_client,
     const String & src_bucket,
     const String & src_key,
     const std::optional<String> & version_id,
@@ -31,47 +31,85 @@ void copyS3FileToDisk(
         src_offset = 0;
 
     if (!src_size)
-        src_size = S3::getObjectSize(*s3_client, src_bucket, src_key, version_id.value_or(""), request_settings) - *src_offset;
+        src_size = S3::getObjectSize(*src_s3_client, src_bucket, src_key, version_id.value_or(""), request_settings) - *src_offset;
 
     auto destination_data_source_description = destination_disk->getDataSourceDescription();
-    if (destination_data_source_description != DataSourceDescription{DataSourceType::S3, s3_client->getInitialEndpoint(), false, false})
+    if (destination_data_source_description == DataSourceDescription{DataSourceType::S3, src_s3_client->getInitialEndpoint(), false, false})
     {
-        LOG_TRACE(&Poco::Logger::get("copyS3FileToDisk"), "Copying {} to disk {} through buffers", src_key, destination_disk->getName());
-        ReadBufferFromS3 read_buffer{s3_client, src_bucket, src_key, {}, request_settings, read_settings};
-        if (*src_offset)
-            read_buffer.seek(*src_offset, SEEK_SET);
-        auto write_buffer = destination_disk->writeFile(destination_path, std::min<size_t>(*src_size, DBMS_DEFAULT_BUFFER_SIZE), write_mode, write_settings);
-        copyData(read_buffer, *write_buffer, *src_size);
-        write_buffer->finalize();
+        /// Use native copy, the more optimal way.
+        LOG_TRACE(&Poco::Logger::get("copyS3FileToDisk"), "Copying {} to disk {} using native copy", src_key, destination_disk->getName());
+        auto write_blob_function = [&](const std::pair<String, String> & blob_path_, WriteMode write_mode_, const std::optional<ObjectAttributes> & object_attributes_) -> size_t
+        {
+            /// Object storage always uses mode `Rewrite` because it simulates append using metadata and different files.
+            chassert(write_mode_ == WriteMode::Rewrite);
+
+            copyS3File(
+                src_s3_client,
+                src_bucket,
+                src_key,
+                *src_offset,
+                *src_size,
+                /* dest_bucket= */ blob_path_.first,
+                /* dest_key= */ blob_path_.second,
+                request_settings,
+                object_attributes_,
+                scheduler,
+                /* for_disk_s3= */ true);
+
+            return *src_size;
+        };
+
+        destination_disk->writeFileUsingBlobWritingFunction(destination_path, write_mode, write_blob_function);
         return;
     }
 
-    LOG_TRACE(&Poco::Logger::get("copyS3FileToDisk"), "Copying {} to disk {} using native copy", src_key, destination_disk->getName());
+    /// Fallback to copy through buffers.
+    LOG_TRACE(&Poco::Logger::get("copyS3FileToDisk"), "Copying {} to disk {} through buffers", src_key, destination_disk->getName());
+    ReadBufferFromS3 read_buffer{src_s3_client, src_bucket, src_key, {}, request_settings, read_settings};
+    if (*src_offset)
+        read_buffer.seek(*src_offset, SEEK_SET);
+    auto write_buffer = destination_disk->writeFile(destination_path, std::min<size_t>(*src_size, DBMS_DEFAULT_BUFFER_SIZE), write_mode, write_settings);
+    copyData(read_buffer, *write_buffer, *src_size);
+    write_buffer->finalize();
+}
 
-    String dest_bucket = destination_disk->getObjectStorage()->getObjectsNamespace();
+void copyS3FileFromDisk(
+    DiskPtr src_disk,
+    const String & src_path,
+    std::optional<size_t> src_offset,
+    std::optional<size_t> src_size,
+    const std::shared_ptr<const S3::Client> & dest_s3_client,
+    const String & dest_bucket,
+    const String & dest_key,
+    const ReadSettings & read_settings,
+    const S3Settings::RequestSettings & request_settings,
+    ThreadPoolCallbackRunner<void> scheduler)
+{
+    if (!src_offset)
+        src_offset = 0;
 
-    auto write_blob_function = [&](const std::pair<String, String> & blob_path_, WriteMode write_mode_, const std::optional<ObjectAttributes> & object_attributes_) -> size_t
+    if (!src_size)
+        src_size = src_disk->getFileSize(src_path) - *src_offset;
+
+    auto source_data_source_description = src_disk->getDataSourceDescription();
+    if (source_data_source_description == DataSourceDescription{DataSourceType::S3, dest_s3_client->getInitialEndpoint(), false, false})
     {
-        /// Object storage always uses mode `Rewrite` because it simulates append using metadata and different files.
-        chassert(write_mode_ == WriteMode::Rewrite);
+        /// getBlobPath() can return std::nullopt if the file is stored as multiple objects in S3 bucket.
+        /// In this case we can't use native copy.
+        if (auto blob_path = src_disk->getBlobPath(src_path))
+        {
+            /// Use native copy, the more optimal way.
+            LOG_TRACE(&Poco::Logger::get("copyS3FileFromDisk"), "Copying file {} to S3 using native copy", src_path);
+            const auto & [src_bucket, src_key] = *blob_path;
+            copyS3File(dest_s3_client, src_bucket, src_key, *src_offset, *src_size, dest_bucket, dest_key, request_settings, {}, scheduler);
+            return;
+        }
+    }
 
-        copyS3File(
-            s3_client,
-            src_bucket,
-            src_key,
-            *src_offset,
-            *src_size,
-            /* dest_bucket= */ blob_path_.first,
-            /* dest_key= */ blob_path_.second,
-            request_settings,
-            object_attributes_,
-            scheduler,
-            /* for_disk_s3= */ true);
-
-        return *src_size;
-    };
-
-    destination_disk->writeFileUsingBlobWritingFunction(destination_path, write_mode, write_blob_function);
+    /// Fallback to copy through buffers.
+    LOG_TRACE(&Poco::Logger::get("copyS3FileFromDisk"), "Copying {} to S3 through buffers", src_path);
+    auto create_read_buffer = [src_disk, &src_path, &read_settings] { return src_disk->readFile(src_path, read_settings); };
+    copyDataToS3File(create_read_buffer, *src_offset, *src_size, dest_s3_client, dest_bucket, dest_key, request_settings, {}, scheduler);
 }
 
 }
diff --git a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.h b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.h
index 21c92ec9623..78caf2f50c8 100644
--- a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.h
+++ b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.h
@@ -13,11 +13,11 @@ namespace DB
 {
 
 /// Copies an object from S3 bucket to a disk of any type.
-/// Depending on the disk the function can either do copying though buffers
+/// Depending on the disk the function can either do copying through buffers
 /// (i.e. download the object by portions and then write those portions to the specified disk),
 /// or perform a server-side copy.
 void copyS3FileToDisk(
-    const std::shared_ptr<const S3::Client> & s3_client,
+    const std::shared_ptr<const S3::Client> & src_s3_client,
     const String & src_bucket,
     const String & src_key,
     const std::optional<String> & version_id,
@@ -31,6 +31,22 @@ void copyS3FileToDisk(
     const S3Settings::RequestSettings & request_settings = {},
     ThreadPoolCallbackRunner<void> scheduler = {});
 
+/// Copies an object from a disk of any type to S3 bucket.
+/// Depending on the disk the function can either do copying through buffers
+/// (i.e. read the object by portions and then upload those portions to the specified disk),
+/// or perform a server-side copy.
+void copyS3FileFromDisk(
+    DiskPtr src_disk,
+    const String & src_path,
+    std::optional<size_t> src_offset,
+    std::optional<size_t> src_size,
+    const std::shared_ptr<const S3::Client> & dest_s3_client,
+    const String & dest_bucket,
+    const String & dest_key,
+    const ReadSettings & read_settings = {},
+    const S3Settings::RequestSettings & request_settings = {},
+    ThreadPoolCallbackRunner<void> scheduler = {});
+
 }
 
 #endif
diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py
index 6e9b6b8569e..b5ac34f0b46 100644
--- a/tests/integration/test_backup_restore_s3/test.py
+++ b/tests/integration/test_backup_restore_s3/test.py
@@ -141,7 +141,7 @@ def test_backup_to_s3_native_copy():
         f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', 'minio123')"
     )
     check_backup_and_restore(storage_policy, backup_destination)
-    assert node.contains_in_log("BackupWriterS3.*using native copy")
+    assert node.contains_in_log("copyS3FileFromDisk.*using native copy")
     assert node.contains_in_log("copyS3FileToDisk.*using native copy")
     assert node.contains_in_log(
         f"copyS3File: Single operation copy has completed. Bucket: root, Key: data/backups/{backup_name}"
@@ -155,7 +155,7 @@ def test_backup_to_s3_native_copy_other_bucket():
         f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', 'minio123')"
     )
     check_backup_and_restore(storage_policy, backup_destination)
-    assert node.contains_in_log("BackupWriterS3.*using native copy")
+    assert node.contains_in_log("copyS3FileFromDisk.*using native copy")
     assert node.contains_in_log("copyS3FileToDisk.*using native copy")
     assert node.contains_in_log(
         f"copyS3File: Single operation copy has completed. Bucket: root, Key: data/backups/{backup_name}"
@@ -167,7 +167,7 @@ def test_backup_to_s3_native_copy_multipart():
     backup_name = new_backup_name()
     backup_destination = f"S3('http://minio1:9001/root/data/backups/multipart/{backup_name}', 'minio', 'minio123')"
     check_backup_and_restore(storage_policy, backup_destination, size=1000000)
-    assert node.contains_in_log("BackupWriterS3.*using native copy")
+    assert node.contains_in_log("copyS3FileFromDisk.*using native copy")
     assert node.contains_in_log("copyS3FileToDisk.*using native copy")
     assert node.contains_in_log(
         f"copyS3File: Multipart upload has completed. Bucket: root, Key: data/backups/multipart/{backup_name}/"

From bc880db5d925a4ff57565d85b541e4d6c17c07d7 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sun, 23 Apr 2023 10:50:45 +0200
Subject: [PATCH 558/689] Add functions to read/write encrypted files from
 IDisk.

---
 src/Disks/DiskEncrypted.h | 23 +++++++++++++++++++++++
 src/Disks/IDisk.cpp       | 15 +++++++++++++++
 src/Disks/IDisk.h         | 14 ++++++++++++++
 3 files changed, 52 insertions(+)

diff --git a/src/Disks/DiskEncrypted.h b/src/Disks/DiskEncrypted.h
index d9da320e505..9c9e61275ce 100644
--- a/src/Disks/DiskEncrypted.h
+++ b/src/Disks/DiskEncrypted.h
@@ -198,6 +198,29 @@ public:
         delegate->removeSharedFileIfExists(wrapped_path, flag);
     }
 
+    std::unique_ptr<ReadBufferFromFileBase> readEncryptedFile(
+        const String & path, const ReadSettings & settings) const override
+    {
+        auto wrapped_path = wrappedPath(path);
+        return delegate->readFile(wrapped_path, settings);
+    }
+
+    std::unique_ptr<WriteBufferFromFileBase> writeEncryptedFile(
+        const String & path,
+        size_t buf_size,
+        WriteMode mode,
+        const WriteSettings & settings) const override
+    {
+        auto wrapped_path = wrappedPath(path);
+        return delegate->writeFile(wrapped_path, buf_size, mode, settings);
+    }
+
+    size_t getEncryptedFileSize(const String & path) const override
+    {
+        auto wrapped_path = wrappedPath(path);
+        return delegate->getFileSize(wrapped_path);
+    }
+
     void setLastModified(const String & path, const Poco::Timestamp & timestamp) override
     {
         auto wrapped_path = wrappedPath(path);
diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp
index 9f39c8242f5..54e09b09d2f 100644
--- a/src/Disks/IDisk.cpp
+++ b/src/Disks/IDisk.cpp
@@ -52,6 +52,21 @@ void IDisk::removeSharedFiles(const RemoveBatchRequest & files, bool keep_all_ba
     }
 }
 
+std::unique_ptr<ReadBufferFromFileBase> IDisk::readEncryptedFile(const String &, const ReadSettings &) const
+{
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "File encryption is not implemented for disk of type {}", getDataSourceDescription().type);
+}
+
+std::unique_ptr<WriteBufferFromFileBase> IDisk::writeEncryptedFile(const String &, size_t, WriteMode, const WriteSettings &) const
+{
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "File encryption is not implemented for disk of type {}", getDataSourceDescription().type);
+}
+
+size_t IDisk::getEncryptedFileSize(const String &) const
+{
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "File encryption is not implemented for disk of type {}", getDataSourceDescription().type);
+}
+
 
 using ResultsCollector = std::vector<std::future<void>>;
 
diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h
index 238f2258c3d..b98d8a74308 100644
--- a/src/Disks/IDisk.h
+++ b/src/Disks/IDisk.h
@@ -251,6 +251,20 @@ public:
     /// Second bool param is a flag to remove (true) or keep (false) shared data on S3
     virtual void removeSharedFileIfExists(const String & path, bool /* keep_shared_data */) { removeFileIfExists(path); }
 
+    /// Reads a file from an encrypted disk without decrypting it.
+    virtual std::unique_ptr<ReadBufferFromFileBase> readEncryptedFile(
+        const String & path, const ReadSettings & settings = ReadSettings{}) const;
+
+    /// Writes an already encrypted file to an encrypted disk.
+    virtual std::unique_ptr<WriteBufferFromFileBase> writeEncryptedFile(
+        const String & path,
+        size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
+        WriteMode mode = WriteMode::Rewrite,
+        const WriteSettings & settings = {}) const;
+
+    /// Returns the size of encrypted file on an encrypted disk.
+    virtual size_t getEncryptedFileSize(const String & path) const;
+
     virtual const String & getCacheName() const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There is no cache"); }
 
     virtual bool supportsCache() const { return false; }

From cc50fcc60a4102b194f30b56896d045390833d8d Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sun, 23 Apr 2023 12:25:46 +0200
Subject: [PATCH 559/689] Remove the 'temporary_file_' argument from
 BackupEntryFromImmutableFile's constructor.

---
 src/Backups/BackupEntryFromAppendOnlyFile.cpp   |  5 ++---
 src/Backups/BackupEntryFromAppendOnlyFile.h     |  3 +--
 src/Backups/BackupEntryFromImmutableFile.cpp    |  4 +---
 src/Backups/BackupEntryFromImmutableFile.h      |  5 +----
 src/Backups/BackupEntryWrappedWith.h            |  8 +++++++-
 .../MergeTree/DataPartStorageOnDiskBase.cpp     | 10 +++++++---
 src/Storages/StorageLog.cpp                     | 17 +++++++++--------
 src/Storages/StorageStripeLog.cpp               | 17 +++++++++--------
 8 files changed, 37 insertions(+), 32 deletions(-)

diff --git a/src/Backups/BackupEntryFromAppendOnlyFile.cpp b/src/Backups/BackupEntryFromAppendOnlyFile.cpp
index 5384a69d890..83117f686bf 100644
--- a/src/Backups/BackupEntryFromAppendOnlyFile.cpp
+++ b/src/Backups/BackupEntryFromAppendOnlyFile.cpp
@@ -10,9 +10,8 @@ BackupEntryFromAppendOnlyFile::BackupEntryFromAppendOnlyFile(
     const String & file_path_,
     const ReadSettings & settings_,
     const std::optional<UInt64> & file_size_,
-    const std::optional<UInt128> & checksum_,
-    const std::shared_ptr<TemporaryFileOnDisk> & temporary_file_)
-    : BackupEntryFromImmutableFile(disk_, file_path_, settings_, file_size_, checksum_, temporary_file_)
+    const std::optional<UInt128> & checksum_)
+    : BackupEntryFromImmutableFile(disk_, file_path_, settings_, file_size_, checksum_)
     , limit(BackupEntryFromImmutableFile::getSize())
 {
 }
diff --git a/src/Backups/BackupEntryFromAppendOnlyFile.h b/src/Backups/BackupEntryFromAppendOnlyFile.h
index b0cee38c6be..b7a39c935a9 100644
--- a/src/Backups/BackupEntryFromAppendOnlyFile.h
+++ b/src/Backups/BackupEntryFromAppendOnlyFile.h
@@ -18,8 +18,7 @@ public:
         const String & file_path_,
         const ReadSettings & settings_,
         const std::optional<UInt64> & file_size_ = {},
-        const std::optional<UInt128> & checksum_ = {},
-        const std::shared_ptr<TemporaryFileOnDisk> & temporary_file_ = {});
+        const std::optional<UInt128> & checksum_ = {});
 
     UInt64 getSize() const override { return limit; }
     std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override;
diff --git a/src/Backups/BackupEntryFromImmutableFile.cpp b/src/Backups/BackupEntryFromImmutableFile.cpp
index 48783a3bb63..790ea567496 100644
--- a/src/Backups/BackupEntryFromImmutableFile.cpp
+++ b/src/Backups/BackupEntryFromImmutableFile.cpp
@@ -13,14 +13,12 @@ BackupEntryFromImmutableFile::BackupEntryFromImmutableFile(
     const String & file_path_,
     const ReadSettings & settings_,
     const std::optional<UInt64> & file_size_,
-    const std::optional<UInt128> & checksum_,
-    const std::shared_ptr<TemporaryFileOnDisk> & temporary_file_)
+    const std::optional<UInt128> & checksum_)
     : disk(disk_)
     , file_path(file_path_)
     , settings(settings_)
     , file_size(file_size_)
     , checksum(checksum_)
-    , temporary_file_on_disk(temporary_file_)
 {
 }
 
diff --git a/src/Backups/BackupEntryFromImmutableFile.h b/src/Backups/BackupEntryFromImmutableFile.h
index 66f1fade294..4f2f902d31e 100644
--- a/src/Backups/BackupEntryFromImmutableFile.h
+++ b/src/Backups/BackupEntryFromImmutableFile.h
@@ -7,7 +7,6 @@
 
 namespace DB
 {
-class TemporaryFileOnDisk;
 class IDisk;
 using DiskPtr = std::shared_ptr<IDisk>;
 
@@ -22,8 +21,7 @@ public:
         const String & file_path_,
         const ReadSettings & settings_,
         const std::optional<UInt64> & file_size_ = {},
-        const std::optional<UInt128> & checksum_ = {},
-        const std::shared_ptr<TemporaryFileOnDisk> & temporary_file_ = {});
+        const std::optional<UInt128> & checksum_ = {});
 
     ~BackupEntryFromImmutableFile() override;
 
@@ -43,7 +41,6 @@ private:
     mutable std::optional<UInt64> file_size TSA_GUARDED_BY(get_file_size_mutex);
     mutable std::mutex get_file_size_mutex;
     const std::optional<UInt128> checksum;
-    const std::shared_ptr<TemporaryFileOnDisk> temporary_file_on_disk;
 };
 
 }
diff --git a/src/Backups/BackupEntryWrappedWith.h b/src/Backups/BackupEntryWrappedWith.h
index 97244650b6b..da3b70e9ba9 100644
--- a/src/Backups/BackupEntryWrappedWith.h
+++ b/src/Backups/BackupEntryWrappedWith.h
@@ -27,11 +27,17 @@ private:
     T custom_value;
 };
 
+template <typename T>
+BackupEntryPtr wrapBackupEntryWith(BackupEntryPtr && backup_entry, const T & custom_value)
+{
+    return std::make_shared<BackupEntryWrappedWith<T>>(std::move(backup_entry), custom_value);
+}
+
 template <typename T>
 void wrapBackupEntriesWith(std::vector<std::pair<String, BackupEntryPtr>> & backup_entries, const T & custom_value)
 {
     for (auto & [_, backup_entry] : backup_entries)
-        backup_entry = std::make_shared<BackupEntryWrappedWith<T>>(std::move(backup_entry), custom_value);
+        backup_entry = wrapBackupEntryWith(std::move(backup_entry), custom_value);
 }
 
 }
diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
index ec00cc3d2b9..4df490f41fe 100644
--- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
+++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
@@ -9,6 +9,7 @@
 #include <Storages/MergeTree/localBackup.h>
 #include <Backups/BackupEntryFromSmallFile.h>
 #include <Backups/BackupEntryFromImmutableFile.h>
+#include <Backups/BackupEntryWrappedWith.h>
 #include <Disks/SingleDiskVolume.h>
 #include <Storages/MergeTree/IMergeTreeDataPart.h>
 
@@ -392,9 +393,12 @@ void DataPartStorageOnDiskBase::backup(
             file_hash = {it->second.file_hash.first, it->second.file_hash.second};
         }
 
-        backup_entries.emplace_back(
-            filepath_in_backup,
-            std::make_unique<BackupEntryFromImmutableFile>(disk, filepath_on_disk, read_settings, file_size, file_hash, temp_dir_owner));
+        BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromImmutableFile>(disk, filepath_on_disk, read_settings, file_size, file_hash);
+
+        if (temp_dir_owner)
+            backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
+
+        backup_entries.emplace_back(filepath_in_backup, std::move(backup_entry));
     }
 }
 
diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp
index 8264d67aaba..19887d6695e 100644
--- a/src/Storages/StorageLog.cpp
+++ b/src/Storages/StorageLog.cpp
@@ -28,6 +28,7 @@
 #include <Backups/BackupEntriesCollector.h>
 #include <Backups/BackupEntryFromAppendOnlyFile.h>
 #include <Backups/BackupEntryFromSmallFile.h>
+#include <Backups/BackupEntryWrappedWith.h>
 #include <Backups/IBackup.h>
 #include <Backups/RestorerFromBackup.h>
 #include <Disks/TemporaryFileOnDisk.h>
@@ -951,10 +952,10 @@ void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, c
         String data_file_name = fileName(data_file.path);
         String hardlink_file_path = temp_dir / data_file_name;
         disk->createHardLink(data_file.path, hardlink_file_path);
-        backup_entries_collector.addBackupEntry(
-            data_path_in_backup_fs / data_file_name,
-            std::make_unique<BackupEntryFromAppendOnlyFile>(
-                disk, hardlink_file_path, read_settings, file_checker.getFileSize(data_file.path), std::nullopt, temp_dir_owner));
+        BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromAppendOnlyFile>(
+            disk, hardlink_file_path, read_settings, file_checker.getFileSize(data_file.path));
+        backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
+        backup_entries_collector.addBackupEntry(data_path_in_backup_fs / data_file_name, std::move(backup_entry));
     }
 
     /// __marks.mrk
@@ -964,10 +965,10 @@ void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, c
         String marks_file_name = fileName(marks_file_path);
         String hardlink_file_path = temp_dir / marks_file_name;
         disk->createHardLink(marks_file_path, hardlink_file_path);
-        backup_entries_collector.addBackupEntry(
-            data_path_in_backup_fs / marks_file_name,
-            std::make_unique<BackupEntryFromAppendOnlyFile>(
-                disk, hardlink_file_path, read_settings, file_checker.getFileSize(marks_file_path), std::nullopt, temp_dir_owner));
+        BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromAppendOnlyFile>(
+            disk, hardlink_file_path, read_settings, file_checker.getFileSize(marks_file_path));
+        backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
+        backup_entries_collector.addBackupEntry(data_path_in_backup_fs / marks_file_name, std::move(backup_entry));
     }
 
     /// sizes.json
diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp
index d54725b8b39..ddb55c119c4 100644
--- a/src/Storages/StorageStripeLog.cpp
+++ b/src/Storages/StorageStripeLog.cpp
@@ -32,6 +32,7 @@
 #include <Backups/BackupEntriesCollector.h>
 #include <Backups/BackupEntryFromAppendOnlyFile.h>
 #include <Backups/BackupEntryFromSmallFile.h>
+#include <Backups/BackupEntryWrappedWith.h>
 #include <Backups/IBackup.h>
 #include <Backups/RestorerFromBackup.h>
 #include <Disks/TemporaryFileOnDisk.h>
@@ -551,10 +552,10 @@ void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collec
         String data_file_name = fileName(data_file_path);
         String hardlink_file_path = temp_dir / data_file_name;
         disk->createHardLink(data_file_path, hardlink_file_path);
-        backup_entries_collector.addBackupEntry(
-            data_path_in_backup_fs / data_file_name,
-            std::make_unique<BackupEntryFromAppendOnlyFile>(
-                disk, hardlink_file_path, read_settings, file_checker.getFileSize(data_file_path), std::nullopt, temp_dir_owner));
+        BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromAppendOnlyFile>(
+            disk, hardlink_file_path, read_settings, file_checker.getFileSize(data_file_path));
+        backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
+        backup_entries_collector.addBackupEntry(data_path_in_backup_fs / data_file_name, std::move(backup_entry));
     }
 
     /// index.mrk
@@ -563,10 +564,10 @@ void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collec
         String index_file_name = fileName(index_file_path);
         String hardlink_file_path = temp_dir / index_file_name;
         disk->createHardLink(index_file_path, hardlink_file_path);
-        backup_entries_collector.addBackupEntry(
-            data_path_in_backup_fs / index_file_name,
-            std::make_unique<BackupEntryFromAppendOnlyFile>(
-                disk, hardlink_file_path, read_settings, file_checker.getFileSize(index_file_path), std::nullopt, temp_dir_owner));
+        BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromAppendOnlyFile>(
+            disk, hardlink_file_path, read_settings, file_checker.getFileSize(index_file_path));
+        backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
+        backup_entries_collector.addBackupEntry(data_path_in_backup_fs / index_file_name, std::move(backup_entry));
     }
 
     /// sizes.json

From c92219f01b29c54c6a83b7b2ea63565ff67b7681 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Tue, 25 Apr 2023 19:44:03 +0200
Subject: [PATCH 560/689] BACKUP now writes encrypted data for tables on
 encrypted disks.

---
 src/Backups/BackupCoordinationRemote.cpp      |   2 +
 src/Backups/BackupEntryFromAppendOnlyFile.h   |   3 +-
 src/Backups/BackupEntryFromImmutableFile.cpp  |  28 ++---
 src/Backups/BackupEntryFromImmutableFile.h    |  12 +-
 src/Backups/BackupEntryFromMemory.h           |   7 --
 src/Backups/BackupEntryFromSmallFile.cpp      |  25 +++-
 src/Backups/BackupEntryFromSmallFile.h        |  18 ++-
 src/Backups/BackupEntryWrappedWith.h          |   5 +-
 src/Backups/BackupFileInfo.cpp                |   2 +
 src/Backups/BackupFileInfo.h                  |   3 +
 src/Backups/BackupIO.cpp                      |  43 ++++---
 src/Backups/BackupIO.h                        |  30 +++--
 src/Backups/BackupIO_Disk.cpp                 |  46 +++----
 src/Backups/BackupIO_Disk.h                   |  19 +--
 src/Backups/BackupIO_File.cpp                 | 107 ++++++++--------
 src/Backups/BackupIO_File.h                   |  18 +--
 src/Backups/BackupIO_S3.cpp                   | 106 ++++++++++------
 src/Backups/BackupIO_S3.h                     |  24 ++--
 src/Backups/BackupImpl.cpp                    |  47 +++++--
 src/Backups/BackupImpl.h                      |   2 +
 src/Backups/IBackupEntriesLazyBatch.cpp       |  20 +--
 src/Backups/IBackupEntry.h                    |   9 +-
 src/Common/ErrorCodes.cpp                     |   1 +
 src/Disks/DiskEncrypted.h                     |  24 ++--
 src/Disks/DiskLocal.cpp                       |  32 +++--
 src/Disks/DiskLocal.h                         |   3 +-
 src/Disks/DiskType.cpp                        |   5 +
 src/Disks/DiskType.h                          |   1 +
 src/Disks/IDisk.h                             |  27 ++--
 src/Disks/IDiskTransaction.h                  |   2 +-
 .../ObjectStorages/DiskObjectStorage.cpp      |  11 +-
 src/Disks/ObjectStorages/DiskObjectStorage.h  |   2 +-
 .../DiskObjectStorageTransaction.cpp          |   6 +-
 .../ObjectStorages/S3/copyS3FileToDisk.cpp    | 117 ------------------
 .../ObjectStorages/S3/copyS3FileToDisk.h      |  52 --------
 src/Storages/StorageLog.cpp                   |   1 +
 src/Storages/StorageMemory.cpp                |   1 +
 src/Storages/StorageStripeLog.cpp             |   1 +
 38 files changed, 413 insertions(+), 449 deletions(-)
 delete mode 100644 src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp
 delete mode 100644 src/Disks/ObjectStorages/S3/copyS3FileToDisk.h

diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp
index c1ef353c4eb..9a2fdf5dd6b 100644
--- a/src/Backups/BackupCoordinationRemote.cpp
+++ b/src/Backups/BackupCoordinationRemote.cpp
@@ -115,6 +115,7 @@ namespace
                 writeBinary(info.checksum, out);
                 writeBinary(info.base_size, out);
                 writeBinary(info.base_checksum, out);
+                writeBinary(info.encrypted_by_disk, out);
                 /// We don't store `info.data_file_name` and `info.data_file_index` because they're determined automalically
                 /// after reading file infos for all the hosts (see the class BackupCoordinationFileInfos).
             }
@@ -136,6 +137,7 @@ namespace
                 readBinary(info.checksum, in);
                 readBinary(info.base_size, in);
                 readBinary(info.base_checksum, in);
+                readBinary(info.encrypted_by_disk, in);
             }
             return res;
         }
diff --git a/src/Backups/BackupEntryFromAppendOnlyFile.h b/src/Backups/BackupEntryFromAppendOnlyFile.h
index b7a39c935a9..7c57e55923e 100644
--- a/src/Backups/BackupEntryFromAppendOnlyFile.h
+++ b/src/Backups/BackupEntryFromAppendOnlyFile.h
@@ -11,7 +11,6 @@ namespace DB
 class BackupEntryFromAppendOnlyFile : public BackupEntryFromImmutableFile
 {
 public:
-
     /// The constructor is allowed to not set `file_size_` or `checksum_`, in that case it will be calculated from the data.
     BackupEntryFromAppendOnlyFile(
         const DiskPtr & disk_,
@@ -23,6 +22,8 @@ public:
     UInt64 getSize() const override { return limit; }
     std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override;
 
+    bool isFromImmutableFile() const override { return false; }
+
 private:
     const UInt64 limit;
 };
diff --git a/src/Backups/BackupEntryFromImmutableFile.cpp b/src/Backups/BackupEntryFromImmutableFile.cpp
index 790ea567496..7545134f638 100644
--- a/src/Backups/BackupEntryFromImmutableFile.cpp
+++ b/src/Backups/BackupEntryFromImmutableFile.cpp
@@ -16,9 +16,10 @@ BackupEntryFromImmutableFile::BackupEntryFromImmutableFile(
     const std::optional<UInt128> & checksum_)
     : disk(disk_)
     , file_path(file_path_)
+    , data_source_description(disk->getDataSourceDescription())
     , settings(settings_)
-    , file_size(file_size_)
-    , checksum(checksum_)
+    , file_size(data_source_description.is_encrypted ? std::optional<UInt64>{} : file_size_)
+    , checksum(data_source_description.is_encrypted ? std::optional<UInt128>{} : checksum_)
 {
 }
 
@@ -28,24 +29,21 @@ UInt64 BackupEntryFromImmutableFile::getSize() const
 {
     std::lock_guard lock{get_file_size_mutex};
     if (!file_size)
-        file_size = disk->getFileSize(file_path);
+    {
+        if (data_source_description.is_encrypted)
+            file_size = disk->getEncryptedFileSize(file_path);
+        else
+            file_size = disk->getFileSize(file_path);
+    }
     return *file_size;
 }
 
 std::unique_ptr<SeekableReadBuffer> BackupEntryFromImmutableFile::getReadBuffer() const
 {
-    return disk->readFile(file_path, settings);
-}
-
-
-DataSourceDescription BackupEntryFromImmutableFile::getDataSourceDescription() const
-{
-    return disk->getDataSourceDescription();
-}
-
-String BackupEntryFromImmutableFile::getFilePath() const
-{
-    return file_path;
+    if (data_source_description.is_encrypted)
+        return disk->readEncryptedFile(file_path, settings);
+    else
+        return disk->readFile(file_path, settings);
 }
 
 }
diff --git a/src/Backups/BackupEntryFromImmutableFile.h b/src/Backups/BackupEntryFromImmutableFile.h
index 4f2f902d31e..f2801b67df6 100644
--- a/src/Backups/BackupEntryFromImmutableFile.h
+++ b/src/Backups/BackupEntryFromImmutableFile.h
@@ -14,7 +14,6 @@ using DiskPtr = std::shared_ptr<IDisk>;
 class BackupEntryFromImmutableFile : public IBackupEntry
 {
 public:
-
     /// The constructor is allowed to not set `file_size_` or `checksum_`, in that case it will be calculated from the data.
     BackupEntryFromImmutableFile(
         const DiskPtr & disk_,
@@ -28,15 +27,20 @@ public:
     UInt64 getSize() const override;
     std::optional<UInt128> getChecksum() const override { return checksum; }
     std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override;
+    
+    bool isEncryptedByDisk() const override { return data_source_description.is_encrypted; }
 
-    String getFilePath() const override;
-    DataSourceDescription getDataSourceDescription() const override;
+    DataSourceDescription getDataSourceDescription() const override { return data_source_description; }
 
-    DiskPtr tryGetDiskIfExists() const override { return disk; }
+    bool isFromFile() const override { return true; }
+    bool isFromImmutableFile() const override { return true; }
+    DiskPtr getDisk() const override { return disk; }
+    String getFilePath() const override { return file_path; }
 
 private:
     const DiskPtr disk;
     const String file_path;
+    const DataSourceDescription data_source_description;
     ReadSettings settings;
     mutable std::optional<UInt64> file_size TSA_GUARDED_BY(get_file_size_mutex);
     mutable std::mutex get_file_size_mutex;
diff --git a/src/Backups/BackupEntryFromMemory.h b/src/Backups/BackupEntryFromMemory.h
index df3b9de40e3..64f46d68580 100644
--- a/src/Backups/BackupEntryFromMemory.h
+++ b/src/Backups/BackupEntryFromMemory.h
@@ -19,18 +19,11 @@ public:
     std::optional<UInt128> getChecksum() const override { return checksum; }
     std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override;
 
-    String getFilePath() const override
-    {
-        return "";
-    }
-
     DataSourceDescription getDataSourceDescription() const override
     {
         return DataSourceDescription{DataSourceType::RAM, "", false, false};
     }
 
-    DiskPtr tryGetDiskIfExists() const override { return nullptr; }
-
 private:
     const String data;
     const std::optional<UInt128> checksum;
diff --git a/src/Backups/BackupEntryFromSmallFile.cpp b/src/Backups/BackupEntryFromSmallFile.cpp
index d24b3a6498d..6f7d2364031 100644
--- a/src/Backups/BackupEntryFromSmallFile.cpp
+++ b/src/Backups/BackupEntryFromSmallFile.cpp
@@ -1,6 +1,9 @@
 #include <Backups/BackupEntryFromSmallFile.h>
+#include <Common/filesystemHelpers.h>
+#include <Disks/DiskLocal.h>
 #include <Disks/IDisk.h>
 #include <Disks/IO/createReadBufferFromFileBase.h>
+#include <IO/ReadBufferFromString.h>
 #include <IO/ReadHelpers.h>
 
 
@@ -16,9 +19,9 @@ namespace
         return s;
     }
 
-    String readFile(const DiskPtr & disk, const String & file_path)
+    String readFile(const DiskPtr & disk, const String & file_path, bool read_encrypted)
     {
-        auto buf = disk->readFile(file_path);
+        auto buf = read_encrypted ? disk->readEncryptedFile(file_path) : disk->readFile(file_path);
         String s;
         readStringUntilEOF(s, *buf);
         return s;
@@ -27,14 +30,26 @@ namespace
 
 
 BackupEntryFromSmallFile::BackupEntryFromSmallFile(const String & file_path_, const std::optional<UInt128> & checksum_)
-    : BackupEntryFromMemory(readFile(file_path_), checksum_), file_path(file_path_)
+    : file_path(file_path_)
+    , data_source_description(DiskLocal::getLocalDataSourceDescription(file_path_))
+    , data(readFile(file_path_))
+    , checksum(checksum_)
 {
 }
-
+    
 BackupEntryFromSmallFile::BackupEntryFromSmallFile(
     const DiskPtr & disk_, const String & file_path_, const std::optional<UInt128> & checksum_)
-    : BackupEntryFromMemory(readFile(disk_, file_path_), checksum_), disk(disk_), file_path(file_path_)
+    : disk(disk_)
+    , file_path(file_path_)
+    , data_source_description(disk_->getDataSourceDescription())
+    , data(readFile(disk_, file_path, data_source_description.is_encrypted))
+    , checksum(data_source_description.is_encrypted ? std::optional<UInt128>{} : checksum_)
 {
 }
 
+std::unique_ptr<SeekableReadBuffer> BackupEntryFromSmallFile::getReadBuffer() const
+{
+    return std::make_unique<ReadBufferFromString>(data);
+}
+
 }
diff --git a/src/Backups/BackupEntryFromSmallFile.h b/src/Backups/BackupEntryFromSmallFile.h
index 99e319f07a0..2f7f3764571 100644
--- a/src/Backups/BackupEntryFromSmallFile.h
+++ b/src/Backups/BackupEntryFromSmallFile.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <Backups/BackupEntryFromMemory.h>
+#include <Backups/IBackupEntry.h>
 
 
 namespace DB
@@ -10,7 +10,7 @@ using DiskPtr = std::shared_ptr<IDisk>;
 
 /// Represents a file prepared to be included in a backup,
 /// assuming that the file is small and can be easily loaded into memory.
-class BackupEntryFromSmallFile : public BackupEntryFromMemory
+class BackupEntryFromSmallFile : public IBackupEntry
 {
 public:
     /// The constructor is allowed to not set `checksum_`, in that case it will be calculated from the data.
@@ -23,12 +23,24 @@ public:
         const String & file_path_,
         const std::optional<UInt128> & checksum_ = {});
 
+    UInt64 getSize() const override { return data.size(); }
+    std::optional<UInt128> getChecksum() const override { return checksum; }
+    std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override;
+
+    bool isEncryptedByDisk() const override { return data_source_description.is_encrypted; }
+
+    bool isFromFile() const override { return true; }
+    DiskPtr getDisk() const override { return disk; }
     String getFilePath() const override { return file_path; }
 
-    DiskPtr tryGetDiskIfExists() const override { return disk; }
+    DataSourceDescription getDataSourceDescription() const override { return data_source_description; }
+
 private:
     const DiskPtr disk;
     const String file_path;
+    const DataSourceDescription data_source_description;
+    const String data;
+    const std::optional<UInt128> checksum;
 };
 
 }
diff --git a/src/Backups/BackupEntryWrappedWith.h b/src/Backups/BackupEntryWrappedWith.h
index da3b70e9ba9..933fec39634 100644
--- a/src/Backups/BackupEntryWrappedWith.h
+++ b/src/Backups/BackupEntryWrappedWith.h
@@ -18,8 +18,11 @@ public:
     UInt64 getSize() const override { return entry->getSize(); }
     std::optional<UInt128> getChecksum() const override { return entry->getChecksum(); }
     std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override { return entry->getReadBuffer(); }
+    bool isEncryptedByDisk() const override { return entry->isEncryptedByDisk(); }
+    bool isFromFile() const override { return entry->isFromFile(); }
+    bool isFromImmutableFile() const override { return entry->isFromImmutableFile(); }
     String getFilePath() const override { return entry->getFilePath(); }
-    DiskPtr tryGetDiskIfExists() const override { return entry->tryGetDiskIfExists(); }
+    DiskPtr getDisk() const override { return entry->getDisk(); }
     DataSourceDescription getDataSourceDescription() const override { return entry->getDataSourceDescription(); }
 
 private:
diff --git a/src/Backups/BackupFileInfo.cpp b/src/Backups/BackupFileInfo.cpp
index 5a3076d1647..91ddc52ae44 100644
--- a/src/Backups/BackupFileInfo.cpp
+++ b/src/Backups/BackupFileInfo.cpp
@@ -111,6 +111,7 @@ String BackupFileInfo::describe() const
     result += fmt::format("base_checksum: {};\n", getHexUIntLowercase(checksum));
     result += fmt::format("data_file_name: {};\n", data_file_name);
     result += fmt::format("data_file_index: {};\n", data_file_index);
+    result += fmt::format("encrypted_by_disk: {};\n", encrypted_by_disk);
     return result;
 }
 
@@ -122,6 +123,7 @@ BackupFileInfo buildFileInfoForBackupEntry(const String & file_name, const Backu
     BackupFileInfo info;
     info.file_name = adjusted_path;
     info.size = backup_entry->getSize();
+    info.encrypted_by_disk = backup_entry->isEncryptedByDisk();
 
     /// We don't set `info.data_file_name` and `info.data_file_index` in this function because they're set during backup coordination
     /// (see the class BackupCoordinationFileInfos).
diff --git a/src/Backups/BackupFileInfo.h b/src/Backups/BackupFileInfo.h
index ae6ec83a37b..a925a1e81ac 100644
--- a/src/Backups/BackupFileInfo.h
+++ b/src/Backups/BackupFileInfo.h
@@ -35,6 +35,9 @@ struct BackupFileInfo
     /// This field is set during backup coordination (see the class BackupCoordinationFileInfos).
     size_t data_file_index = static_cast<size_t>(-1);
 
+    /// Whether this file is encrypted by an encrypted disk.
+    bool encrypted_by_disk = false;
+
     struct LessByFileName
     {
         bool operator()(const BackupFileInfo & lhs, const BackupFileInfo & rhs) const { return (lhs.file_name < rhs.file_name); }
diff --git a/src/Backups/BackupIO.cpp b/src/Backups/BackupIO.cpp
index 7b269bd965f..4d890ed7419 100644
--- a/src/Backups/BackupIO.cpp
+++ b/src/Backups/BackupIO.cpp
@@ -14,13 +14,18 @@ IBackupReader::IBackupReader(Poco::Logger * log_) : log(log_)
 {
 }
 
-void IBackupReader::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path,
-                                   WriteMode write_mode, const WriteSettings & write_settings)
+void IBackupReader::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
+                                   DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings)
 {
-    LOG_TRACE(log, "Copying file {} through buffers", file_name);
-    auto read_buffer = readFile(file_name);
-    auto write_buffer = destination_disk->writeFile(destination_path, std::min<size_t>(size, DBMS_DEFAULT_BUFFER_SIZE), write_mode, write_settings);
-    copyData(*read_buffer, *write_buffer, size);
+    LOG_TRACE(log, "Copying file {} to disk {} through buffers", path_in_backup, destination_disk->getName());
+    auto read_buffer = readFile(path_in_backup);
+    auto buf_size = std::min<size_t>(file_size, DBMS_DEFAULT_BUFFER_SIZE);
+    std::unique_ptr<WriteBuffer> write_buffer;
+    if (encrypted_in_backup)
+        write_buffer = destination_disk->writeEncryptedFile(destination_path, buf_size, write_mode, write_settings);
+    else
+        write_buffer = destination_disk->writeFile(destination_path, buf_size, write_mode, write_settings);
+    copyData(*read_buffer, *write_buffer, file_size);
     write_buffer->finalize();
 }
 
@@ -29,22 +34,28 @@ IBackupWriter::IBackupWriter(const ContextPtr & context_, Poco::Logger * log_)
 {
 }
 
-void IBackupWriter::copyDataToFile(const CreateReadBufferFunction & create_read_buffer, UInt64 offset, UInt64 size, const String & dest_file_name)
+void IBackupWriter::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length)
 {
     auto read_buffer = create_read_buffer();
-    if (offset)
-        read_buffer->seek(offset, SEEK_SET);
-    auto write_buffer = writeFile(dest_file_name);
-    copyData(*read_buffer, *write_buffer, size);
+    if (start_pos)
+        read_buffer->seek(start_pos, SEEK_SET);
+    auto write_buffer = writeFile(path_in_backup);
+    copyData(*read_buffer, *write_buffer, length);
     write_buffer->finalize();
 }
 
-void IBackupWriter::copyFileFromDisk(
-    DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name)
+void IBackupWriter::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
+                                     bool copy_encrypted, UInt64 start_pos, UInt64 length)
 {
-    LOG_TRACE(log, "Copying file {} through buffers", src_file_name);
-    auto create_read_buffer = [this, src_disk, src_file_name] { return src_disk->readFile(src_file_name, read_settings); };
-    copyDataToFile(create_read_buffer, src_offset, src_size, dest_file_name);
+    LOG_TRACE(log, "Copying file {} from disk {} through buffers", src_path, src_disk->getName());
+    auto create_read_buffer = [this, src_disk, src_path, copy_encrypted]
+    {
+        if (copy_encrypted)
+            return src_disk->readEncryptedFile(src_path, read_settings);
+        else
+            return src_disk->readFile(src_path, read_settings);
+    };
+    copyDataToFile(path_in_backup, create_read_buffer, start_pos, length);
 }
 
 }
diff --git a/src/Backups/BackupIO.h b/src/Backups/BackupIO.h
index dae13422bf2..f65f0871698 100644
--- a/src/Backups/BackupIO.h
+++ b/src/Backups/BackupIO.h
@@ -16,14 +16,19 @@ class IBackupReader /// BackupReaderFile, BackupReaderDisk
 {
 public:
     explicit IBackupReader(Poco::Logger * log_);
-
     virtual ~IBackupReader() = default;
+
     virtual bool fileExists(const String & file_name) = 0;
     virtual UInt64 getFileSize(const String & file_name) = 0;
+
     virtual std::unique_ptr<SeekableReadBuffer> readFile(const String & file_name) = 0;
-    virtual void copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path,
-                                WriteMode write_mode, const WriteSettings & write_settings);
-    virtual DataSourceDescription getDataSourceDescription() const = 0;
+
+    /// The function copyFileToDisk() can be much faster than reading the file with readFile() and then writing it to some disk.
+    /// (especially for S3 where it can use CopyObject to copy objects inside S3 instead of downloading and uploading them).
+    /// Parameters:
+    /// `encrypted_in_backup` specify if this file is encrypted in the backup,  so it shouldn't be encrypted again while restoring to an encrypted disk.
+    virtual void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
+                                DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings);
 
 protected:
     Poco::Logger * const log;
@@ -33,28 +38,29 @@ protected:
 class IBackupWriter /// BackupWriterFile, BackupWriterDisk
 {
 public:
-    using CreateReadBufferFunction = std::function<std::unique_ptr<SeekableReadBuffer>()>;
-
     IBackupWriter(const ContextPtr & context_, Poco::Logger * log_);
-
     virtual ~IBackupWriter() = default;
+
     virtual bool fileExists(const String & file_name) = 0;
     virtual UInt64 getFileSize(const String & file_name) = 0;
     virtual bool fileContentsEqual(const String & file_name, const String & expected_file_contents) = 0;
 
     virtual std::unique_ptr<WriteBuffer> writeFile(const String & file_name) = 0;
 
-    virtual void copyDataToFile(const CreateReadBufferFunction & create_read_buffer, UInt64 offset, UInt64 size, const String & dest_file_name);
+    using CreateReadBufferFunction = std::function<std::unique_ptr<SeekableReadBuffer>()>;
+    virtual void copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length);
 
-    /// copyFileFromDisk() can be much faster than copyDataToFile()
+    /// The function copyFileFromDisk() can be much faster than copyDataToFile()
     /// (especially for S3 where it can use CopyObject to copy objects inside S3 instead of downloading and uploading them).
-    virtual void copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name);
+    /// Parameters:
+    /// `start_pos` and `length` specify a part of the file on `src_disk` to copy to the backup.
+    /// `copy_encrypted` specify whether this function should copy encrypted data of the file `src_path` to the backup.
+    virtual void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
+                                  bool copy_encrypted, UInt64 start_pos, UInt64 length);
 
     virtual void removeFile(const String & file_name) = 0;
     virtual void removeFiles(const Strings & file_names) = 0;
 
-    virtual DataSourceDescription getDataSourceDescription() const = 0;
-
 protected:
     Poco::Logger * const log;
 
diff --git a/src/Backups/BackupIO_Disk.cpp b/src/Backups/BackupIO_Disk.cpp
index 643922cf3d0..d24d90ae7ae 100644
--- a/src/Backups/BackupIO_Disk.cpp
+++ b/src/Backups/BackupIO_Disk.cpp
@@ -36,24 +36,19 @@ std::unique_ptr<SeekableReadBuffer> BackupReaderDisk::readFile(const String & fi
     return disk->readFile(path / file_name);
 }
 
-void BackupReaderDisk::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path,
-                                      WriteMode write_mode, const WriteSettings & write_settings)
+void BackupReaderDisk::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
+                                      DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings)
 {
-    if ((write_mode == WriteMode::Rewrite) && (destination_disk->getDataSourceDescription() == getDataSourceDescription()))
+    if ((write_mode == WriteMode::Rewrite) && !encrypted_in_backup)
     {
         /// Use more optimal way.
-        LOG_TRACE(log, "Copying file {} using {} disk", file_name, toString(destination_disk->getDataSourceDescription().type));
-        disk->copyFile(path / file_name, *destination_disk, destination_path, write_settings);
+        LOG_TRACE(log, "Copying file {} from disk {} to disk {}", path_in_backup, disk->getName(), destination_disk->getName());
+        disk->copyFile(path / path_in_backup, *destination_disk, destination_path, write_settings);
         return;
     }
 
     /// Fallback to copy through buffers.
-    IBackupReader::copyFileToDisk(file_name, size, destination_disk, destination_path, write_mode, write_settings);
-}
-
-DataSourceDescription BackupReaderDisk::getDataSourceDescription() const
-{
-    return disk->getDataSourceDescription();
+    IBackupReader::copyFileToDisk(path_in_backup, file_size, encrypted_in_backup, destination_disk, destination_path, write_mode, write_settings);
 }
 
 
@@ -118,30 +113,21 @@ void BackupWriterDisk::removeFiles(const Strings & file_names)
         disk->removeDirectory(path);
 }
 
-DataSourceDescription BackupWriterDisk::getDataSourceDescription() const
+void BackupWriterDisk::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
+                                        bool copy_encrypted, UInt64 start_pos, UInt64 length)
 {
-    return disk->getDataSourceDescription();
-}
-
-void BackupWriterDisk::copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name)
-{
-    /// IDisk::copyFile() can copy to the same disk only, and it cannot do the throttling.
-    if (!has_throttling && (getDataSourceDescription() == src_disk->getDataSourceDescription()))
+    if (!copy_encrypted && !start_pos && (length == src_disk->getFileSize(src_path)))
     {
-        /// IDisk::copyFile() can copy a file as a whole only.
-        if ((src_offset == 0) && (src_size == src_disk->getFileSize(src_file_name)))
-        {
-            /// Use more optimal way.
-            LOG_TRACE(log, "Copying file {} using {} disk", src_file_name, toString(src_disk->getDataSourceDescription().type));
-            auto dest_file_path = path / dest_file_name;
-            disk->createDirectories(dest_file_path.parent_path());
-            src_disk->copyFile(src_file_name, *disk, dest_file_path);
-            return;
-        }
+        /// Use more optimal way.
+        LOG_TRACE(log, "Copying file {} from disk {} to disk {}", src_path, src_disk->getName(), disk->getName());
+        auto dest_file_path = path / path_in_backup;
+        disk->createDirectories(dest_file_path.parent_path());
+        src_disk->copyFile(src_path, *disk, dest_file_path);
+        return;
     }
 
     /// Fallback to copy through buffers.
-    IBackupWriter::copyFileFromDisk(src_disk, src_file_name, src_offset, src_size, dest_file_name);
+    IBackupWriter::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length);
 }
 
 }
diff --git a/src/Backups/BackupIO_Disk.h b/src/Backups/BackupIO_Disk.h
index 884282ea4e0..d2af06668eb 100644
--- a/src/Backups/BackupIO_Disk.h
+++ b/src/Backups/BackupIO_Disk.h
@@ -18,13 +18,12 @@ public:
     bool fileExists(const String & file_name) override;
     UInt64 getFileSize(const String & file_name) override;
     std::unique_ptr<SeekableReadBuffer> readFile(const String & file_name) override;
-    void copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path,
-                        WriteMode write_mode, const WriteSettings & write_settings) override;
-    DataSourceDescription getDataSourceDescription() const override;
+    void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
+                        DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings) override;
 
 private:
-    DiskPtr disk;
-    std::filesystem::path path;
+    const DiskPtr disk;
+    const std::filesystem::path path;
 };
 
 class BackupWriterDisk : public IBackupWriter
@@ -37,14 +36,16 @@ public:
     UInt64 getFileSize(const String & file_name) override;
     bool fileContentsEqual(const String & file_name, const String & expected_file_contents) override;
     std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
-    void copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name) override;
+
+    void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
+                          bool copy_encrypted, UInt64 start_pos, UInt64 length) override;
+
     void removeFile(const String & file_name) override;
     void removeFiles(const Strings & file_names) override;
-    DataSourceDescription getDataSourceDescription() const override;
 
 private:
-    DiskPtr disk;
-    std::filesystem::path path;
+    const DiskPtr disk;
+    const std::filesystem::path path;
     const bool has_throttling;
 };
 
diff --git a/src/Backups/BackupIO_File.cpp b/src/Backups/BackupIO_File.cpp
index b569e65284b..69d899528cd 100644
--- a/src/Backups/BackupIO_File.cpp
+++ b/src/Backups/BackupIO_File.cpp
@@ -1,4 +1,5 @@
 #include <Backups/BackupIO_File.h>
+#include <Disks/DiskLocal.h>
 #include <Disks/IDisk.h>
 #include <Disks/IO/createReadBufferFromFileBase.h>
 #include <IO/WriteBufferFromFile.h>
@@ -15,7 +16,9 @@ namespace DB
 {
 
 BackupReaderFile::BackupReaderFile(const String & path_)
-    : IBackupReader(&Poco::Logger::get("BackupReaderFile")), path(path_)
+    : IBackupReader(&Poco::Logger::get("BackupReaderFile"))
+    , path(path_)
+    , data_source_description(DiskLocal::getLocalDataSourceDescription(path_))
 {
 }
 
@@ -36,25 +39,44 @@ std::unique_ptr<SeekableReadBuffer> BackupReaderFile::readFile(const String & fi
     return createReadBufferFromFileBase(path / file_name, {});
 }
 
-void BackupReaderFile::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path,
-                                      WriteMode write_mode, const WriteSettings & write_settings)
+void BackupReaderFile::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
+                                      DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings)
 {
-    if (destination_disk->getDataSourceDescription() == getDataSourceDescription())
+    if (write_mode == WriteMode::Rewrite)
     {
-        /// Use more optimal way.
-        LOG_TRACE(log, "Copying file {} locally", file_name);
-        fs::copy(path / file_name, fullPath(destination_disk, destination_path), fs::copy_options::overwrite_existing);
-        return;
+        auto destination_data_source_description = destination_disk->getDataSourceDescription();
+        if (destination_data_source_description.sameKind(data_source_description)
+            && (destination_data_source_description.is_encrypted == encrypted_in_backup))
+        {
+            /// Use more optimal way.
+            LOG_TRACE(log, "Copying file {} to disk {} locally", path_in_backup, destination_disk->getName());
+
+            auto write_blob_function
+                = [abs_source_path = path / path_in_backup, file_size](
+                      const Strings & blob_path, WriteMode mode, const std::optional<ObjectAttributes> &) -> size_t
+            {
+                if (blob_path.size() != 1 || mode != WriteMode::Rewrite)
+                    throw Exception(ErrorCodes::LOGICAL_ERROR,
+                                    "Blob writing function called with unexpected blob_path.size={} or mode={}",
+                                    blob_path.size(), mode);
+                fs::copy(abs_source_path, blob_path.at(0), fs::copy_options::overwrite_existing);
+                return file_size;
+            };
+
+            destination_disk->writeFileUsingBlobWritingFunction(destination_path, write_mode, write_blob_function);
+            return;
+        }
     }
 
     /// Fallback to copy through buffers.
-    IBackupReader::copyFileToDisk(path / file_name, size, destination_disk, destination_path, write_mode, write_settings);
+    IBackupReader::copyFileToDisk(path_in_backup, file_size, encrypted_in_backup, destination_disk, destination_path, write_mode, write_settings);
 }
 
 
 BackupWriterFile::BackupWriterFile(const String & path_, const ContextPtr & context_)
     : IBackupWriter(context_, &Poco::Logger::get("BackupWriterFile"))
     , path(path_)
+    , data_source_description(DiskLocal::getLocalDataSourceDescription(path_))
     , has_throttling(static_cast<bool>(context_->getBackupsThrottler()))
 {
 }
@@ -112,59 +134,36 @@ void BackupWriterFile::removeFiles(const Strings & file_names)
         fs::remove(path);
 }
 
-DataSourceDescription BackupWriterFile::getDataSourceDescription() const
-{
-    DataSourceDescription data_source_description;
-
-    data_source_description.type = DataSourceType::Local;
-
-    if (auto block_device_id = tryGetBlockDeviceId(path); block_device_id.has_value())
-        data_source_description.description = *block_device_id;
-    else
-        data_source_description.description = path;
-    data_source_description.is_encrypted = false;
-    data_source_description.is_cached = false;
-
-    return data_source_description;
-}
-
-DataSourceDescription BackupReaderFile::getDataSourceDescription() const
-{
-    DataSourceDescription data_source_description;
-
-    data_source_description.type = DataSourceType::Local;
-
-    if (auto block_device_id = tryGetBlockDeviceId(path); block_device_id.has_value())
-        data_source_description.description = *block_device_id;
-    else
-        data_source_description.description = path;
-    data_source_description.is_encrypted = false;
-    data_source_description.is_cached = false;
-
-    return data_source_description;
-}
-
-
-void BackupWriterFile::copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name)
+void BackupWriterFile::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
+                                        bool copy_encrypted, UInt64 start_pos, UInt64 length)
 {
     /// std::filesystem::copy() can copy from the filesystem only, and it cannot do the throttling.
-    if (!has_throttling && (getDataSourceDescription() == src_disk->getDataSourceDescription()))
+    if (!has_throttling)
     {
-        std::string abs_source_path = fullPath(src_disk, src_file_name);
-        /// std::filesystem::copy() can copy a file as a whole only.
-        if ((src_offset == 0) && (src_size == fs::file_size(abs_source_path)))
+        auto source_data_source_description = src_disk->getDataSourceDescription();
+        if (source_data_source_description.sameKind(data_source_description)
+            && (source_data_source_description.is_encrypted == copy_encrypted))
         {
-            /// Use more optimal way.
-            LOG_TRACE(log, "Copying file {} locally", src_file_name);
-            auto abs_dest_path = path / dest_file_name;
-            fs::create_directories(abs_dest_path.parent_path());
-            fs::copy(abs_source_path, abs_dest_path, fs::copy_options::overwrite_existing);
-            return;
+            if (auto blob_path = src_disk->getBlobPath(src_path); blob_path.size() == 1)
+            {
+                auto abs_source_path = blob_path[0];
+
+                /// std::filesystem::copy() can copy a file as a whole only.
+                if ((start_pos == 0) && (length == fs::file_size(abs_source_path)))
+                {
+                    /// Use more optimal way.
+                    LOG_TRACE(log, "Copying file {} from disk {} locally", src_path, src_disk->getName());
+                    auto abs_dest_path = path / path_in_backup;
+                    fs::create_directories(abs_dest_path.parent_path());
+                    fs::copy(abs_source_path, abs_dest_path, fs::copy_options::overwrite_existing);
+                    return;
+                }
+            }
         }
     }
 
     /// Fallback to copy through buffers.
-    IBackupWriter::copyFileFromDisk(src_disk, src_file_name, src_offset, src_size, dest_file_name);
+    IBackupWriter::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length);
 }
 
 }
diff --git a/src/Backups/BackupIO_File.h b/src/Backups/BackupIO_File.h
index 1f0a247c321..a8ada969ca4 100644
--- a/src/Backups/BackupIO_File.h
+++ b/src/Backups/BackupIO_File.h
@@ -16,12 +16,13 @@ public:
     bool fileExists(const String & file_name) override;
     UInt64 getFileSize(const String & file_name) override;
     std::unique_ptr<SeekableReadBuffer> readFile(const String & file_name) override;
-    void copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path,
-                        WriteMode write_mode, const WriteSettings & write_settings) override;
-    DataSourceDescription getDataSourceDescription() const override;
+
+    void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
+                        DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings) override;
 
 private:
-    std::filesystem::path path;
+    const std::filesystem::path path;
+    const DataSourceDescription data_source_description;
 };
 
 class BackupWriterFile : public IBackupWriter
@@ -34,13 +35,16 @@ public:
     UInt64 getFileSize(const String & file_name) override;
     bool fileContentsEqual(const String & file_name, const String & expected_file_contents) override;
     std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
-    void copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name) override;
+
+    void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
+                          bool copy_encrypted, UInt64 start_pos, UInt64 length) override;
+
     void removeFile(const String & file_name) override;
     void removeFiles(const Strings & file_names) override;
-    DataSourceDescription getDataSourceDescription() const override;
 
 private:
-    std::filesystem::path path;
+    const std::filesystem::path path;
+    const DataSourceDescription data_source_description;
     const bool has_throttling;
 };
 
diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp
index d2861500159..00694a5e01d 100644
--- a/src/Backups/BackupIO_S3.cpp
+++ b/src/Backups/BackupIO_S3.cpp
@@ -2,7 +2,6 @@
 
 #if USE_AWS_S3
 #include <Common/quoteString.h>
-#include <Disks/ObjectStorages/S3/copyS3FileToDisk.h>
 #include <Interpreters/threadPoolCallbackRunner.h>
 #include <Interpreters/Context.h>
 #include <IO/SharedThreadPools.h>
@@ -107,16 +106,11 @@ BackupReaderS3::BackupReaderS3(
     , client(makeS3Client(s3_uri_, access_key_id_, secret_access_key_, context_))
     , read_settings(context_->getReadSettings())
     , request_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString()).request_settings)
+    , data_source_description{DataSourceType::S3, s3_uri.endpoint, false, false}
 {
     request_settings.max_single_read_retries = context_->getSettingsRef().s3_max_single_read_retries; // FIXME: Avoid taking value for endpoint
 }
 
-DataSourceDescription BackupReaderS3::getDataSourceDescription() const
-{
-    return DataSourceDescription{DataSourceType::S3, s3_uri.endpoint, false, false};
-}
-
-
 BackupReaderS3::~BackupReaderS3() = default;
 
 bool BackupReaderS3::fileExists(const String & file_name)
@@ -138,23 +132,45 @@ std::unique_ptr<SeekableReadBuffer> BackupReaderS3::readFile(const String & file
         client, s3_uri.bucket, fs::path(s3_uri.key) / file_name, s3_uri.version_id, request_settings, read_settings);
 }
 
-void BackupReaderS3::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path,
-                                    WriteMode write_mode, const WriteSettings & write_settings)
+void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
+                                    DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings)
 {
-    copyS3FileToDisk(
-        client,
-        s3_uri.bucket,
-        fs::path(s3_uri.key) / file_name,
-        s3_uri.version_id,
-        0,
-        size,
-        destination_disk,
-        destination_path,
-        write_mode,
-        read_settings,
-        write_settings,
-        request_settings,
-        threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupReaderS3"));
+    auto destination_data_source_description = destination_disk->getDataSourceDescription();
+    if (destination_data_source_description.sameKind(data_source_description)
+        && (destination_data_source_description.is_encrypted == encrypted_in_backup))
+    {
+        /// Use native copy, the more optimal way.
+        LOG_TRACE(log, "Copying {} from S3 to disk {} using native copy", path_in_backup, destination_disk->getName());
+        auto write_blob_function = [&](const Strings & blob_path, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes) -> size_t
+        {
+            /// Object storage always uses mode `Rewrite` because it simulates append using metadata and different files.
+            if (blob_path.size() != 2 || mode != WriteMode::Rewrite)
+                throw Exception(ErrorCodes::LOGICAL_ERROR,
+                                "Blob writing function called with unexpected blob_path.size={} or mode={}",
+                                blob_path.size(), mode);
+
+            copyS3File(
+                client,
+                s3_uri.bucket,
+                fs::path(s3_uri.key) / path_in_backup,
+                0,
+                file_size,
+                /* dest_bucket= */ blob_path[0],
+                /* dest_key= */ blob_path[1],
+                request_settings,
+                object_attributes,
+                threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupReaderS3"),
+                /* for_disk_s3= */ true);
+
+            return file_size;
+        };
+
+        destination_disk->writeFileUsingBlobWritingFunction(destination_path, write_mode, write_blob_function);
+        return;
+    }
+
+    /// Fallback to copy through buffers.
+    IBackupReader::copyFileToDisk(path_in_backup, file_size, encrypted_in_backup, destination_disk, destination_path, write_mode, write_settings);
 }
 
 
@@ -164,27 +180,47 @@ BackupWriterS3::BackupWriterS3(
     , s3_uri(s3_uri_)
     , client(makeS3Client(s3_uri_, access_key_id_, secret_access_key_, context_))
     , request_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString()).request_settings)
+    , data_source_description{DataSourceType::S3, s3_uri.endpoint, false, false}
 {
     request_settings.updateFromSettings(context_->getSettingsRef());
     request_settings.max_single_read_retries = context_->getSettingsRef().s3_max_single_read_retries; // FIXME: Avoid taking value for endpoint
 }
 
-DataSourceDescription BackupWriterS3::getDataSourceDescription() const
+void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
+                                      bool copy_encrypted, UInt64 start_pos, UInt64 length)
 {
-    return DataSourceDescription{DataSourceType::S3, s3_uri.endpoint, false, false};
+    auto source_data_source_description = src_disk->getDataSourceDescription();
+    if (source_data_source_description.sameKind(data_source_description)
+        && (source_data_source_description.is_encrypted == copy_encrypted))
+    {
+        /// getBlobPath() can return std::nullopt if the file is stored as multiple objects in S3 bucket.
+        /// In this case we can't use native copy.
+        if (auto blob_path = src_disk->getBlobPath(src_path); blob_path.size() == 2)
+        {
+            /// Use native copy, the more optimal way.
+            LOG_TRACE(log, "Copying file {} from disk {} to S3 using native copy", src_path, src_disk->getName());
+            copyS3File(
+                client,
+                /* src_bucket */ blob_path[0],
+                /* src_key= */ blob_path[1],
+                start_pos,
+                length,
+                s3_uri.endpoint,
+                fs::path(s3_uri.key) / path_in_backup,
+                request_settings,
+                {},
+                threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupWriterS3"));
+            return;
+        }
+    }
+
+    /// Fallback to copy through buffers.
+    IBackupWriter::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length);
 }
 
-void BackupWriterS3::copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name)
+void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length)
 {
-    copyS3FileFromDisk(src_disk, src_file_name, src_offset, src_size,
-        client, s3_uri.bucket, fs::path(s3_uri.key) / dest_file_name, read_settings, request_settings,
-        threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupWriterS3"));
-}
-
-void BackupWriterS3::copyDataToFile(
-    const CreateReadBufferFunction & create_read_buffer, UInt64 offset, UInt64 size, const String & dest_file_name)
-{
-    copyDataToS3File(create_read_buffer, offset, size, client, s3_uri.bucket, fs::path(s3_uri.key) / dest_file_name, request_settings, {},
+    copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, request_settings, {},
                      threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupWriterS3"));
 }
 
diff --git a/src/Backups/BackupIO_S3.h b/src/Backups/BackupIO_S3.h
index 7d53d30e8d6..1db9d5f8c4c 100644
--- a/src/Backups/BackupIO_S3.h
+++ b/src/Backups/BackupIO_S3.h
@@ -23,15 +23,16 @@ public:
     bool fileExists(const String & file_name) override;
     UInt64 getFileSize(const String & file_name) override;
     std::unique_ptr<SeekableReadBuffer> readFile(const String & file_name) override;
-    void copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path,
-                        WriteMode write_mode, const WriteSettings & write_settings) override;
-    DataSourceDescription getDataSourceDescription() const override;
+
+    void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
+                        DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings) override;
 
 private:
-    S3::URI s3_uri;
-    std::shared_ptr<S3::Client> client;
-    ReadSettings read_settings;
+    const S3::URI s3_uri;
+    const std::shared_ptr<S3::Client> client;
+    const ReadSettings read_settings;
     S3Settings::RequestSettings request_settings;
+    const DataSourceDescription data_source_description;
 };
 
 
@@ -46,9 +47,9 @@ public:
     bool fileContentsEqual(const String & file_name, const String & expected_file_contents) override;
     std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
 
-    void copyDataToFile(const CreateReadBufferFunction & create_read_buffer, UInt64 offset, UInt64 size, const String & dest_file_name) override;
-    void copyFileFromDisk(DiskPtr src_disk, const String & src_file_name, UInt64 src_offset, UInt64 src_size, const String & dest_file_name) override;
-    DataSourceDescription getDataSourceDescription() const override;
+    void copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) override;
+    void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
+                          bool copy_encrypted, UInt64 start_pos, UInt64 length) override;
 
     void removeFile(const String & file_name) override;
     void removeFiles(const Strings & file_names) override;
@@ -56,10 +57,11 @@ public:
 private:
     void removeFilesBatch(const Strings & file_names);
 
-    S3::URI s3_uri;
-    std::shared_ptr<S3::Client> client;
+    const S3::URI s3_uri;
+    const std::shared_ptr<S3::Client> client;
     S3Settings::RequestSettings request_settings;
     std::optional<bool> supports_batch_delete;
+    const DataSourceDescription data_source_description;
 };
 
 }
diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp
index 55fb6dbfe03..9bfa2d77353 100644
--- a/src/Backups/BackupImpl.cpp
+++ b/src/Backups/BackupImpl.cpp
@@ -36,6 +36,7 @@ namespace ErrorCodes
     extern const int WRONG_BASE_BACKUP;
     extern const int BACKUP_ENTRY_NOT_FOUND;
     extern const int BACKUP_IS_EMPTY;
+    extern const int CANNOT_RESTORE_TO_NONENCRYPTED_DISK;
     extern const int FAILED_TO_SYNC_BACKUP_OR_RESTORE;
     extern const int LOGICAL_ERROR;
 }
@@ -339,6 +340,8 @@ void BackupImpl::writeBackupMetadata()
             }
             if (!info.data_file_name.empty() && (info.data_file_name != info.file_name))
                 *out << "<data_file>" << xml << info.data_file_name << "</data_file>";
+            if (info.encrypted_by_disk)
+                *out << "<encrypted_by_disk>true</encrypted_by_disk>";
         }
 
         total_size += info.size;
@@ -444,6 +447,7 @@ void BackupImpl::readBackupMetadata()
                 {
                     info.data_file_name = getString(file_config, "data_file", info.file_name);
                 }
+                info.encrypted_by_disk = getBool(file_config, "encrypted_by_disk", false);
             }
 
             file_names.emplace(info.file_name, std::pair{info.size, info.checksum});
@@ -633,6 +637,11 @@ std::unique_ptr<SeekableReadBuffer> BackupImpl::readFile(const String & file_nam
 }
 
 std::unique_ptr<SeekableReadBuffer> BackupImpl::readFile(const SizeAndChecksum & size_and_checksum) const
+{
+    return readFileImpl(size_and_checksum, /* read_encrypted= */ false);
+}
+
+std::unique_ptr<SeekableReadBuffer> BackupImpl::readFileImpl(const SizeAndChecksum & size_and_checksum, bool read_encrypted) const
 {
     if (open_mode != OpenMode::READ)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Backup is not opened for reading");
@@ -660,6 +669,14 @@ std::unique_ptr<SeekableReadBuffer> BackupImpl::readFile(const SizeAndChecksum &
         info = it->second;
     }
 
+    if (info.encrypted_by_disk != read_encrypted)
+    {
+        throw Exception(
+            ErrorCodes::CANNOT_RESTORE_TO_NONENCRYPTED_DISK,
+            "File {} is encrypted in the backup, it can be restored only to an encrypted disk",
+            info.data_file_name);
+    }
+
     std::unique_ptr<SeekableReadBuffer> read_buffer;
     std::unique_ptr<SeekableReadBuffer> base_read_buffer;
 
@@ -760,14 +777,21 @@ size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum, Dis
         info = it->second;
     }
 
+    if (info.encrypted_by_disk && !destination_disk->getDataSourceDescription().is_encrypted)
+    {
+        throw Exception(
+            ErrorCodes::CANNOT_RESTORE_TO_NONENCRYPTED_DISK,
+            "File {} is encrypted in the backup, it can be restored only to an encrypted disk",
+            info.data_file_name);
+    }
+
     bool file_copied = false;
 
     if (info.size && !info.base_size && !use_archive)
     {
         /// Data comes completely from this backup.
-        reader->copyFileToDisk(info.data_file_name, info.size, destination_disk, destination_path, write_mode, write_settings);
+        reader->copyFileToDisk(info.data_file_name, info.size, info.encrypted_by_disk, destination_disk, destination_path, write_mode, write_settings);
         file_copied = true;
-
     }
     else if (info.size && (info.size == info.base_size))
     {
@@ -786,9 +810,13 @@ size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum, Dis
     else
     {
         /// Use the generic way to copy data. `readFile()` will update `num_read_files`.
-        auto read_buffer = readFile(size_and_checksum);
-        auto write_buffer = destination_disk->writeFile(destination_path, std::min<size_t>(info.size, DBMS_DEFAULT_BUFFER_SIZE),
-                                                        write_mode, write_settings);
+        auto read_buffer = readFileImpl(size_and_checksum, /* read_encrypted= */ info.encrypted_by_disk);
+        std::unique_ptr<WriteBuffer> write_buffer;
+        size_t buf_size = std::min<size_t>(info.size, DBMS_DEFAULT_BUFFER_SIZE);
+        if (info.encrypted_by_disk)
+            write_buffer = destination_disk->writeEncryptedFile(destination_path, buf_size, write_mode, write_settings);
+        else
+            write_buffer = destination_disk->writeFile(destination_path, buf_size, write_mode, write_settings);
         copyData(*read_buffer, *write_buffer, info.size);
         write_buffer->finalize();
     }
@@ -814,8 +842,9 @@ void BackupImpl::writeFile(const BackupFileInfo & info, BackupEntryPtr entry)
             should_check_lock_file = true;
     }
 
-    auto src_disk = entry->tryGetDiskIfExists();
+    auto src_disk = entry->getDisk();
     auto src_file_path = entry->getFilePath();
+    bool from_immutable_file = entry->isFromImmutableFile();
     String src_file_desc = src_file_path.empty() ? "memory buffer" : ("file " + src_file_path);
 
     if (info.data_file_name.empty())
@@ -845,16 +874,16 @@ void BackupImpl::writeFile(const BackupFileInfo & info, BackupEntryPtr entry)
         copyData(*read_buffer, *out);
         out->finalize();
     }
-    else if (src_disk)
+    else if (src_disk && from_immutable_file)
     {
         LOG_TRACE(log, "Writing backup for file {} from {} (disk {}): data file #{}", info.data_file_name, src_file_desc, src_disk->getName(), info.data_file_index);
-        writer->copyFileFromDisk(src_disk, src_file_path, info.base_size, info.size - info.base_size, info.data_file_name);
+        writer->copyFileFromDisk(info.data_file_name, src_disk, src_file_path, info.encrypted_by_disk, info.base_size, info.size - info.base_size);
     }
     else
     {
         LOG_TRACE(log, "Writing backup for file {} from {}: data file #{}", info.data_file_name, src_file_desc, info.data_file_index);
         auto create_read_buffer = [entry] { return entry->getReadBuffer(); };
-        writer->copyDataToFile(create_read_buffer, info.base_size, info.size - info.base_size, info.data_file_name);
+        writer->copyDataToFile(info.data_file_name, create_read_buffer, info.base_size, info.size - info.base_size);
     }
 
     {
diff --git a/src/Backups/BackupImpl.h b/src/Backups/BackupImpl.h
index bf94926c46c..511b100c557 100644
--- a/src/Backups/BackupImpl.h
+++ b/src/Backups/BackupImpl.h
@@ -109,6 +109,8 @@ private:
     /// Calculates and sets `compressed_size`.
     void setCompressedSize();
 
+    std::unique_ptr<SeekableReadBuffer> readFileImpl(const SizeAndChecksum & size_and_checksum, bool read_encrypted) const;
+
     const String backup_name_for_logging;
     const bool use_archive;
     const ArchiveParams archive_params;
diff --git a/src/Backups/IBackupEntriesLazyBatch.cpp b/src/Backups/IBackupEntriesLazyBatch.cpp
index 78086015e7b..cd4d470967b 100644
--- a/src/Backups/IBackupEntriesLazyBatch.cpp
+++ b/src/Backups/IBackupEntriesLazyBatch.cpp
@@ -20,20 +20,12 @@ public:
     UInt64 getSize() const override { return getInternalBackupEntry()->getSize(); }
     std::optional<UInt128> getChecksum() const override { return getInternalBackupEntry()->getChecksum(); }
     std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override { return getInternalBackupEntry()->getReadBuffer(); }
-    String getFilePath() const override
-    {
-        return getInternalBackupEntry()->getFilePath();
-    }
-
-    DiskPtr tryGetDiskIfExists() const override
-    {
-        return getInternalBackupEntry()->tryGetDiskIfExists();
-    }
-
-    DataSourceDescription getDataSourceDescription() const override
-    {
-        return getInternalBackupEntry()->getDataSourceDescription();
-    }
+    bool isEncryptedByDisk() const override { return getInternalBackupEntry()->isEncryptedByDisk(); }
+    DataSourceDescription getDataSourceDescription() const override { return getInternalBackupEntry()->getDataSourceDescription(); }
+    bool isFromFile() const override { return getInternalBackupEntry()->isFromFile(); }
+    bool isFromImmutableFile() const override { return getInternalBackupEntry()->isFromImmutableFile(); }
+    String getFilePath() const override { return getInternalBackupEntry()->getFilePath(); }
+    DiskPtr getDisk() const override { return getInternalBackupEntry()->getDisk(); }
 
 private:
     BackupEntryPtr getInternalBackupEntry() const
diff --git a/src/Backups/IBackupEntry.h b/src/Backups/IBackupEntry.h
index 2a71a1e9756..7a93d4035df 100644
--- a/src/Backups/IBackupEntry.h
+++ b/src/Backups/IBackupEntry.h
@@ -27,9 +27,14 @@ public:
     /// Returns a read buffer for reading the data.
     virtual std::unique_ptr<SeekableReadBuffer> getReadBuffer() const = 0;
 
-    virtual String getFilePath() const = 0;
+    /// Returns true if the data returned by getReadBuffer() is encrypted by an encrypted disk.
+    virtual bool isEncryptedByDisk() const { return false; }
 
-    virtual DiskPtr tryGetDiskIfExists() const = 0;
+    /// Returns information about disk and file if this backup entry is generated from a file.  
+    virtual bool isFromFile() const { return false; }
+    virtual bool isFromImmutableFile() const { return false; }
+    virtual String getFilePath() const { return ""; }
+    virtual DiskPtr getDisk() const { return nullptr; }
 
     virtual DataSourceDescription getDataSourceDescription() const = 0;
 };
diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp
index 24e6114b26c..83a7314ac7a 100644
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@@ -579,6 +579,7 @@
     M(694, ASYNC_LOAD_CYCLE) \
     M(695, ASYNC_LOAD_FAILED) \
     M(696, ASYNC_LOAD_CANCELED) \
+    M(697, CANNOT_RESTORE_TO_NONENCRYPTED_DISK) \
     \
     M(999, KEEPER_EXCEPTION) \
     M(1000, POCO_EXCEPTION) \
diff --git a/src/Disks/DiskEncrypted.h b/src/Disks/DiskEncrypted.h
index 9c9e61275ce..be726ef46b4 100644
--- a/src/Disks/DiskEncrypted.h
+++ b/src/Disks/DiskEncrypted.h
@@ -131,18 +131,6 @@ public:
         WriteMode mode,
         const WriteSettings & settings) override;
 
-    std::optional<std::pair<String, String>> getBlobPath(const String & path) const override
-    {
-        auto wrapped_path = wrappedPath(path);
-        return delegate->getBlobPath(wrapped_path);
-    }
-
-    void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) override
-    {
-        auto wrapped_path = wrappedPath(path);
-        delegate->writeFileUsingBlobWritingFunction(wrapped_path, mode, std::move(write_blob_function));
-    }
-
     void removeFile(const String & path) override
     {
         auto wrapped_path = wrappedPath(path);
@@ -198,6 +186,18 @@ public:
         delegate->removeSharedFileIfExists(wrapped_path, flag);
     }
 
+    Strings getBlobPath(const String & path) const override
+    {
+        auto wrapped_path = wrappedPath(path);
+        return delegate->getBlobPath(wrapped_path);
+    }
+
+    void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) override
+    {
+        auto wrapped_path = wrappedPath(path);
+        delegate->writeFileUsingBlobWritingFunction(wrapped_path, mode, std::move(write_blob_function));
+    }
+
     std::unique_ptr<ReadBufferFromFileBase> readEncryptedFile(
         const String & path, const ReadSettings & settings) const override
     {
diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp
index a40368fae88..c76ea289101 100644
--- a/src/Disks/DiskLocal.cpp
+++ b/src/Disks/DiskLocal.cpp
@@ -328,15 +328,16 @@ DiskLocal::writeFile(const String & path, size_t buf_size, WriteMode mode, const
         fs::path(disk_path) / path, buf_size, flags, settings.local_throttler);
 }
 
-std::optional<std::pair<String, String>> DiskLocal::getBlobPath(const String & path) const
+std::vector<String> DiskLocal::getBlobPath(const String & path) const
 {
-    return std::make_pair(fs::path(disk_path) / path, "");
+    auto fs_path = fs::path(disk_path) / path;
+    return {fs_path};
 }
 
 void DiskLocal::writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function)
 {
-    auto blob_path = std::make_pair(fs::path(disk_path) / path, "");
-    std::move(write_blob_function)(blob_path, mode, {});
+    auto fs_path = fs::path(disk_path) / path;
+    std::move(write_blob_function)({fs_path}, mode, {});
 }
 
 void DiskLocal::removeFile(const String & path)
@@ -466,15 +467,8 @@ DiskLocal::DiskLocal(const String & name_, const String & path_, UInt64 keep_fre
     , disk_path(path_)
     , keep_free_space_bytes(keep_free_space_bytes_)
     , logger(&Poco::Logger::get("DiskLocal"))
+    , data_source_description(getLocalDataSourceDescription(disk_path))
 {
-    data_source_description.type = DataSourceType::Local;
-
-    if (auto block_device_id = tryGetBlockDeviceId(disk_path); block_device_id.has_value())
-        data_source_description.description = *block_device_id;
-    else
-        data_source_description.description = disk_path;
-    data_source_description.is_encrypted = false;
-    data_source_description.is_cached = false;
 }
 
 DiskLocal::DiskLocal(
@@ -490,6 +484,20 @@ DataSourceDescription DiskLocal::getDataSourceDescription() const
     return data_source_description;
 }
 
+DataSourceDescription DiskLocal::getLocalDataSourceDescription(const String & path)
+{
+    DataSourceDescription res;
+    res.type = DataSourceType::Local;
+
+    if (auto block_device_id = tryGetBlockDeviceId(path); block_device_id.has_value())
+        res.description = *block_device_id;
+    else
+        res.description = path;
+    res.is_encrypted = false;
+    res.is_cached = false;
+    return res;
+}
+
 void DiskLocal::shutdown()
 {
     if (disk_checker)
diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h
index b838654925d..3d340ae40b7 100644
--- a/src/Disks/DiskLocal.h
+++ b/src/Disks/DiskLocal.h
@@ -81,7 +81,7 @@ public:
         WriteMode mode,
         const WriteSettings & settings) override;
 
-    std::optional<std::pair<String, String>> getBlobPath(const String & path) const override;
+    Strings getBlobPath(const String & path) const override;
     void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) override;
 
     void removeFile(const String & path) override;
@@ -102,6 +102,7 @@ public:
     void truncateFile(const String & path, size_t size) override;
 
     DataSourceDescription getDataSourceDescription() const override;
+    static DataSourceDescription getLocalDataSourceDescription(const String & path);
 
     bool isRemote() const override { return false; }
 
diff --git a/src/Disks/DiskType.cpp b/src/Disks/DiskType.cpp
index 92979ab505c..aa18cc6e0cb 100644
--- a/src/Disks/DiskType.cpp
+++ b/src/Disks/DiskType.cpp
@@ -8,4 +8,9 @@ bool DataSourceDescription::operator==(const DataSourceDescription & other) cons
     return std::tie(type, description, is_encrypted) == std::tie(other.type, other.description, other.is_encrypted);
 }
 
+bool DataSourceDescription::sameKind(const DataSourceDescription & other) const
+{
+    return std::tie(type, description) == std::tie(other.type, other.description);
+}
+
 }
diff --git a/src/Disks/DiskType.h b/src/Disks/DiskType.h
index 840ed5549e6..7d47fa8da78 100644
--- a/src/Disks/DiskType.h
+++ b/src/Disks/DiskType.h
@@ -51,6 +51,7 @@ struct DataSourceDescription
     bool is_cached = false;
 
     bool operator==(const DataSourceDescription & other) const;
+    bool sameKind(const DataSourceDescription & other) const;
 };
 
 }
diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h
index b98d8a74308..006f2d882a0 100644
--- a/src/Disks/IDisk.h
+++ b/src/Disks/IDisk.h
@@ -209,19 +209,6 @@ public:
         WriteMode mode = WriteMode::Rewrite,
         const WriteSettings & settings = {}) = 0;
 
-    /// Returns the path to a blob representing a specified file.
-    /// The meaning of the returned path depends on disk's type.
-    /// For DiskLocal it the absolute path to the file and for DiskObjectStorage it's the name of a namespace
-    /// combined with StoredObject::absolute_path.
-    virtual std::optional<std::pair<String, String>> getBlobPath(const String & path) const = 0;
-
-    using WriteBlobFunction = std::function<size_t(const std::pair<String, String> & blob_path, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>;
-
-    /// Write a file using a custom function to write a blob representing the file.
-    /// This method is alternative to writeFile(), the difference is that writeFile() calls IObjectStorage::writeObject()
-    /// to write an object to the object storage while this method allows to specify a callback for that.
-    virtual void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) = 0;
-
     /// Remove file. Throws exception if file doesn't exists or it's a directory.
     /// Return whether file was finally removed. (For remote disks it is not always removed).
     virtual void removeFile(const String & path) = 0;
@@ -251,6 +238,20 @@ public:
     /// Second bool param is a flag to remove (true) or keep (false) shared data on S3
     virtual void removeSharedFileIfExists(const String & path, bool /* keep_shared_data */) { removeFileIfExists(path); }
 
+    /// Returns the path to a blob representing a specified file.
+    /// The meaning of the returned path depends on disk's type.
+    /// E.g. for DiskLocal it the absolute path to the file and for DiskObjectStorage it's the name of the objects' namespace
+    /// combined with StoredObject::absolute_path for each stored object representing a specified file.
+    virtual Strings getBlobPath(const String & path) const = 0;
+
+    using WriteBlobFunction = std::function<size_t(const Strings & blob_path, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>;
+
+    /// Write a file using a custom function to write a blob representing the file.
+    /// This method is alternative to writeFile(), the difference is that for example for DiskObjectStorage
+    /// writeFile() calls IObjectStorage::writeObject() to write an object to the object storage while
+    /// this method allows to specify a callback for that.
+    virtual void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) = 0;
+
     /// Reads a file from an encrypted disk without decrypting it.
     virtual std::unique_ptr<ReadBufferFromFileBase> readEncryptedFile(
         const String & path, const ReadSettings & settings = ReadSettings{}) const;
diff --git a/src/Disks/IDiskTransaction.h b/src/Disks/IDiskTransaction.h
index 376d7bd78e6..f0c32e04f48 100644
--- a/src/Disks/IDiskTransaction.h
+++ b/src/Disks/IDiskTransaction.h
@@ -68,7 +68,7 @@ public:
         const WriteSettings & settings = {},
         bool autocommit = true) = 0;
 
-    using WriteBlobFunction = std::function<size_t(const std::pair<String, String> & blob_path, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>;
+    using WriteBlobFunction = std::function<size_t(const Strings & blob_path, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>;
 
     /// Write a file using a custom function to write an object to the disk's object storage.
     virtual void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) = 0;
diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp
index b01f1b327e7..c080c2cd92d 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp
@@ -579,12 +579,15 @@ std::unique_ptr<WriteBufferFromFileBase> DiskObjectStorage::writeFile(
     return result;
 }
 
-std::optional<std::pair<String, String>> DiskObjectStorage::getBlobPath(const String & path) const
+Strings DiskObjectStorage::getBlobPath(const String & path) const
 {
     auto objects = getStorageObjects(path);
-    if (objects.size() != 1)
-        return {};
-    return std::make_pair(object_storage->getObjectsNamespace(), objects[0].absolute_path);
+    Strings res;
+    res.reserve(objects.size() + 1);
+    res.emplace_back(object_storage->getObjectsNamespace());
+    for (const auto & object : objects)
+        res.emplace_back(object.absolute_path);
+    return res;
 }
 
 void DiskObjectStorage::writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function)
diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h
index 97751edc3f5..b7dfaf67cf2 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorage.h
+++ b/src/Disks/ObjectStorages/DiskObjectStorage.h
@@ -149,7 +149,7 @@ public:
         WriteMode mode,
         const WriteSettings & settings) override;
 
-    std::optional<std::pair<String, String>> getBlobPath(const String & path) const override;
+    Strings getBlobPath(const String & path) const override;
     void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) override;
 
     void copy(const String & from_path, const std::shared_ptr<IDisk> & to_disk, const String & to_path) override;
diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
index f578bcb9772..f98ac55889b 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
+++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
@@ -691,7 +691,11 @@ void DiskObjectStorageTransaction::writeFileUsingBlobWritingFunction(
 
     operations_to_execute.emplace_back(std::move(write_operation));
 
-    auto blob_path = std::make_pair(object_storage.getObjectsNamespace(), object.absolute_path);
+    /// See DiskObjectStorage::getBlobPath().
+    Strings blob_path;
+    blob_path.reserve(2);
+    blob_path.emplace_back(object_storage.getObjectsNamespace());
+    blob_path.emplace_back(object.absolute_path);
 
     /// We always use mode Rewrite because we simulate append using metadata and different files
     size_t object_size = std::move(write_blob_function)(blob_path, WriteMode::Rewrite, object_attributes);
diff --git a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp
deleted file mode 100644
index e43d88b2519..00000000000
--- a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp
+++ /dev/null
@@ -1,117 +0,0 @@
-#include <Disks/ObjectStorages/S3/copyS3FileToDisk.h>
-
-#if USE_AWS_S3
-
-#include <IO/S3/getObjectInfo.h>
-#include <IO/ReadBufferFromS3.h>
-#include <IO/WriteBufferFromFileBase.h>
-#include <IO/copyData.h>
-#include <IO/S3/copyS3File.h>
-
-
-namespace DB
-{
-
-void copyS3FileToDisk(
-    const std::shared_ptr<const S3::Client> & src_s3_client,
-    const String & src_bucket,
-    const String & src_key,
-    const std::optional<String> & version_id,
-    std::optional<size_t> src_offset,
-    std::optional<size_t> src_size,
-    DiskPtr destination_disk,
-    const String & destination_path,
-    WriteMode write_mode,
-    const ReadSettings & read_settings,
-    const WriteSettings & write_settings,
-    const S3Settings::RequestSettings & request_settings,
-    ThreadPoolCallbackRunner<void> scheduler)
-{
-    if (!src_offset)
-        src_offset = 0;
-
-    if (!src_size)
-        src_size = S3::getObjectSize(*src_s3_client, src_bucket, src_key, version_id.value_or(""), request_settings) - *src_offset;
-
-    auto destination_data_source_description = destination_disk->getDataSourceDescription();
-    if (destination_data_source_description == DataSourceDescription{DataSourceType::S3, src_s3_client->getInitialEndpoint(), false, false})
-    {
-        /// Use native copy, the more optimal way.
-        LOG_TRACE(&Poco::Logger::get("copyS3FileToDisk"), "Copying {} to disk {} using native copy", src_key, destination_disk->getName());
-        auto write_blob_function = [&](const std::pair<String, String> & blob_path_, WriteMode write_mode_, const std::optional<ObjectAttributes> & object_attributes_) -> size_t
-        {
-            /// Object storage always uses mode `Rewrite` because it simulates append using metadata and different files.
-            chassert(write_mode_ == WriteMode::Rewrite);
-
-            copyS3File(
-                src_s3_client,
-                src_bucket,
-                src_key,
-                *src_offset,
-                *src_size,
-                /* dest_bucket= */ blob_path_.first,
-                /* dest_key= */ blob_path_.second,
-                request_settings,
-                object_attributes_,
-                scheduler,
-                /* for_disk_s3= */ true);
-
-            return *src_size;
-        };
-
-        destination_disk->writeFileUsingBlobWritingFunction(destination_path, write_mode, write_blob_function);
-        return;
-    }
-
-    /// Fallback to copy through buffers.
-    LOG_TRACE(&Poco::Logger::get("copyS3FileToDisk"), "Copying {} to disk {} through buffers", src_key, destination_disk->getName());
-    ReadBufferFromS3 read_buffer{src_s3_client, src_bucket, src_key, {}, request_settings, read_settings};
-    if (*src_offset)
-        read_buffer.seek(*src_offset, SEEK_SET);
-    auto write_buffer = destination_disk->writeFile(destination_path, std::min<size_t>(*src_size, DBMS_DEFAULT_BUFFER_SIZE), write_mode, write_settings);
-    copyData(read_buffer, *write_buffer, *src_size);
-    write_buffer->finalize();
-}
-
-void copyS3FileFromDisk(
-    DiskPtr src_disk,
-    const String & src_path,
-    std::optional<size_t> src_offset,
-    std::optional<size_t> src_size,
-    const std::shared_ptr<const S3::Client> & dest_s3_client,
-    const String & dest_bucket,
-    const String & dest_key,
-    const ReadSettings & read_settings,
-    const S3Settings::RequestSettings & request_settings,
-    ThreadPoolCallbackRunner<void> scheduler)
-{
-    if (!src_offset)
-        src_offset = 0;
-
-    if (!src_size)
-        src_size = src_disk->getFileSize(src_path) - *src_offset;
-
-    auto source_data_source_description = src_disk->getDataSourceDescription();
-    if (source_data_source_description == DataSourceDescription{DataSourceType::S3, dest_s3_client->getInitialEndpoint(), false, false})
-    {
-        /// getBlobPath() can return std::nullopt if the file is stored as multiple objects in S3 bucket.
-        /// In this case we can't use native copy.
-        if (auto blob_path = src_disk->getBlobPath(src_path))
-        {
-            /// Use native copy, the more optimal way.
-            LOG_TRACE(&Poco::Logger::get("copyS3FileFromDisk"), "Copying file {} to S3 using native copy", src_path);
-            const auto & [src_bucket, src_key] = *blob_path;
-            copyS3File(dest_s3_client, src_bucket, src_key, *src_offset, *src_size, dest_bucket, dest_key, request_settings, {}, scheduler);
-            return;
-        }
-    }
-
-    /// Fallback to copy through buffers.
-    LOG_TRACE(&Poco::Logger::get("copyS3FileFromDisk"), "Copying {} to S3 through buffers", src_path);
-    auto create_read_buffer = [src_disk, &src_path, &read_settings] { return src_disk->readFile(src_path, read_settings); };
-    copyDataToS3File(create_read_buffer, *src_offset, *src_size, dest_s3_client, dest_bucket, dest_key, request_settings, {}, scheduler);
-}
-
-}
-
-#endif
diff --git a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.h b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.h
deleted file mode 100644
index 78caf2f50c8..00000000000
--- a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.h
+++ /dev/null
@@ -1,52 +0,0 @@
-#pragma once
-
-#include "config.h"
-
-#if USE_AWS_S3
-
-#include <Disks/IDisk.h>
-#include <Storages/StorageS3Settings.h>
-#include <Interpreters/threadPoolCallbackRunner.h>
-
-
-namespace DB
-{
-
-/// Copies an object from S3 bucket to a disk of any type.
-/// Depending on the disk the function can either do copying through buffers
-/// (i.e. download the object by portions and then write those portions to the specified disk),
-/// or perform a server-side copy.
-void copyS3FileToDisk(
-    const std::shared_ptr<const S3::Client> & src_s3_client,
-    const String & src_bucket,
-    const String & src_key,
-    const std::optional<String> & version_id,
-    std::optional<size_t> src_offset,
-    std::optional<size_t> src_size,
-    DiskPtr destination_disk,
-    const String & destination_path,
-    WriteMode write_mode = WriteMode::Rewrite,
-    const ReadSettings & read_settings = {},
-    const WriteSettings & write_settings = {},
-    const S3Settings::RequestSettings & request_settings = {},
-    ThreadPoolCallbackRunner<void> scheduler = {});
-
-/// Copies an object from a disk of any type to S3 bucket.
-/// Depending on the disk the function can either do copying through buffers
-/// (i.e. read the object by portions and then upload those portions to the specified disk),
-/// or perform a server-side copy.
-void copyS3FileFromDisk(
-    DiskPtr src_disk,
-    const String & src_path,
-    std::optional<size_t> src_offset,
-    std::optional<size_t> src_size,
-    const std::shared_ptr<const S3::Client> & dest_s3_client,
-    const String & dest_bucket,
-    const String & dest_key,
-    const ReadSettings & read_settings = {},
-    const S3Settings::RequestSettings & request_settings = {},
-    ThreadPoolCallbackRunner<void> scheduler = {});
-
-}
-
-#endif
diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp
index 19887d6695e..31f499a7d96 100644
--- a/src/Storages/StorageLog.cpp
+++ b/src/Storages/StorageLog.cpp
@@ -27,6 +27,7 @@
 
 #include <Backups/BackupEntriesCollector.h>
 #include <Backups/BackupEntryFromAppendOnlyFile.h>
+#include <Backups/BackupEntryFromMemory.h>
 #include <Backups/BackupEntryFromSmallFile.h>
 #include <Backups/BackupEntryWrappedWith.h>
 #include <Backups/IBackup.h>
diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp
index 37d9e3bc32c..68c888a2d23 100644
--- a/src/Storages/StorageMemory.cpp
+++ b/src/Storages/StorageMemory.cpp
@@ -24,6 +24,7 @@
 #include <Compression/CompressedWriteBuffer.h>
 #include <Backups/BackupEntriesCollector.h>
 #include <Backups/BackupEntryFromImmutableFile.h>
+#include <Backups/BackupEntryFromMemory.h>
 #include <Backups/BackupEntryFromSmallFile.h>
 #include <Backups/IBackup.h>
 #include <Backups/IBackupEntriesLazyBatch.h>
diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp
index ddb55c119c4..5b22db91631 100644
--- a/src/Storages/StorageStripeLog.cpp
+++ b/src/Storages/StorageStripeLog.cpp
@@ -31,6 +31,7 @@
 
 #include <Backups/BackupEntriesCollector.h>
 #include <Backups/BackupEntryFromAppendOnlyFile.h>
+#include <Backups/BackupEntryFromMemory.h>
 #include <Backups/BackupEntryFromSmallFile.h>
 #include <Backups/BackupEntryWrappedWith.h>
 #include <Backups/IBackup.h>

From 002fd19cb7833f1babc6345329d293ba6445155a Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Fri, 28 Apr 2023 13:15:29 +0200
Subject: [PATCH 561/689] Move the common part of BackupIO_* to
 BackupIO_Default.

---
 src/Backups/BackupIO.cpp                      | 61 ------------
 src/Backups/BackupIO.h                        | 39 ++++----
 src/Backups/BackupIO_Default.cpp              | 95 +++++++++++++++++++
 src/Backups/BackupIO_Default.h                | 73 ++++++++++++++
 src/Backups/BackupIO_Disk.cpp                 | 70 ++++++--------
 src/Backups/BackupIO_Disk.h                   | 25 ++---
 src/Backups/BackupIO_File.cpp                 | 84 +++++++---------
 src/Backups/BackupIO_File.h                   | 25 ++---
 src/Backups/BackupIO_S3.cpp                   | 40 +++-----
 src/Backups/BackupIO_S3.h                     | 13 ++-
 src/Backups/BackupImpl.cpp                    | 20 ++--
 src/Backups/BackupImpl.h                      |  6 +-
 src/Backups/IBackup.h                         |  4 +-
 .../registerBackupEnginesFileAndDisk.cpp      |  4 +-
 src/Disks/DiskEncrypted.h                     |  6 +-
 src/Disks/IDisk.cpp                           |  2 +-
 src/Disks/IDisk.h                             |  5 +-
 .../ObjectStorages/DiskObjectStorage.cpp      |  4 +-
 18 files changed, 324 insertions(+), 252 deletions(-)
 delete mode 100644 src/Backups/BackupIO.cpp
 create mode 100644 src/Backups/BackupIO_Default.cpp
 create mode 100644 src/Backups/BackupIO_Default.h

diff --git a/src/Backups/BackupIO.cpp b/src/Backups/BackupIO.cpp
deleted file mode 100644
index 4d890ed7419..00000000000
--- a/src/Backups/BackupIO.cpp
+++ /dev/null
@@ -1,61 +0,0 @@
-#include <Backups/BackupIO.h>
-
-#include <IO/copyData.h>
-#include <IO/WriteBufferFromFileBase.h>
-#include <IO/SeekableReadBuffer.h>
-#include <Interpreters/Context.h>
-#include <Common/logger_useful.h>
-
-
-namespace DB
-{
-
-IBackupReader::IBackupReader(Poco::Logger * log_) : log(log_)
-{
-}
-
-void IBackupReader::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
-                                   DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings)
-{
-    LOG_TRACE(log, "Copying file {} to disk {} through buffers", path_in_backup, destination_disk->getName());
-    auto read_buffer = readFile(path_in_backup);
-    auto buf_size = std::min<size_t>(file_size, DBMS_DEFAULT_BUFFER_SIZE);
-    std::unique_ptr<WriteBuffer> write_buffer;
-    if (encrypted_in_backup)
-        write_buffer = destination_disk->writeEncryptedFile(destination_path, buf_size, write_mode, write_settings);
-    else
-        write_buffer = destination_disk->writeFile(destination_path, buf_size, write_mode, write_settings);
-    copyData(*read_buffer, *write_buffer, file_size);
-    write_buffer->finalize();
-}
-
-IBackupWriter::IBackupWriter(const ContextPtr & context_, Poco::Logger * log_)
-    : log(log_), read_settings(context_->getBackupReadSettings())
-{
-}
-
-void IBackupWriter::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length)
-{
-    auto read_buffer = create_read_buffer();
-    if (start_pos)
-        read_buffer->seek(start_pos, SEEK_SET);
-    auto write_buffer = writeFile(path_in_backup);
-    copyData(*read_buffer, *write_buffer, length);
-    write_buffer->finalize();
-}
-
-void IBackupWriter::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
-                                     bool copy_encrypted, UInt64 start_pos, UInt64 length)
-{
-    LOG_TRACE(log, "Copying file {} from disk {} through buffers", src_path, src_disk->getName());
-    auto create_read_buffer = [this, src_disk, src_path, copy_encrypted]
-    {
-        if (copy_encrypted)
-            return src_disk->readEncryptedFile(src_path, read_settings);
-        else
-            return src_disk->readFile(src_path, read_settings);
-    };
-    copyDataToFile(path_in_backup, create_read_buffer, start_pos, length);
-}
-
-}
diff --git a/src/Backups/BackupIO.h b/src/Backups/BackupIO.h
index f65f0871698..d522387deba 100644
--- a/src/Backups/BackupIO.h
+++ b/src/Backups/BackupIO.h
@@ -1,21 +1,23 @@
 #pragma once
 
 #include <Core/Types.h>
-#include <Disks/DiskType.h>
-#include <Disks/IDisk.h>
-#include <IO/ReadSettings.h>
-#include <Interpreters/Context_fwd.h>
+
 
 namespace DB
 {
+class IDisk;
+using DiskPtr = std::shared_ptr<IDisk>;
 class SeekableReadBuffer;
 class WriteBuffer;
+enum class WriteMode;
+struct WriteSettings;
+struct ReadSettings;
 
 /// Represents operations of loading from disk or downloading for reading a backup.
-class IBackupReader /// BackupReaderFile, BackupReaderDisk
+/// See also implementations: BackupReaderFile, BackupReaderDisk.
+class IBackupReader
 {
 public:
-    explicit IBackupReader(Poco::Logger * log_);
     virtual ~IBackupReader() = default;
 
     virtual bool fileExists(const String & file_name) = 0;
@@ -28,17 +30,18 @@ public:
     /// Parameters:
     /// `encrypted_in_backup` specify if this file is encrypted in the backup,  so it shouldn't be encrypted again while restoring to an encrypted disk.
     virtual void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
-                                DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings);
-
-protected:
-    Poco::Logger * const log;
+                                DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) = 0;
+    
+    virtual const ReadSettings & getReadSettings() const = 0;
+    virtual const WriteSettings & getWriteSettings() const = 0;
+    virtual size_t getWriteBufferSize() const = 0;
 };
 
 /// Represents operations of storing to disk or uploading for writing a backup.
-class IBackupWriter /// BackupWriterFile, BackupWriterDisk
+/// See also implementations: BackupWriterFile, BackupWriterDisk
+class IBackupWriter
 {
 public:
-    IBackupWriter(const ContextPtr & context_, Poco::Logger * log_);
     virtual ~IBackupWriter() = default;
 
     virtual bool fileExists(const String & file_name) = 0;
@@ -48,7 +51,7 @@ public:
     virtual std::unique_ptr<WriteBuffer> writeFile(const String & file_name) = 0;
 
     using CreateReadBufferFunction = std::function<std::unique_ptr<SeekableReadBuffer>()>;
-    virtual void copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length);
+    virtual void copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) = 0;
 
     /// The function copyFileFromDisk() can be much faster than copyDataToFile()
     /// (especially for S3 where it can use CopyObject to copy objects inside S3 instead of downloading and uploading them).
@@ -56,16 +59,14 @@ public:
     /// `start_pos` and `length` specify a part of the file on `src_disk` to copy to the backup.
     /// `copy_encrypted` specify whether this function should copy encrypted data of the file `src_path` to the backup.
     virtual void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
-                                  bool copy_encrypted, UInt64 start_pos, UInt64 length);
+                                  bool copy_encrypted, UInt64 start_pos, UInt64 length) = 0;
 
     virtual void removeFile(const String & file_name) = 0;
     virtual void removeFiles(const Strings & file_names) = 0;
 
-protected:
-    Poco::Logger * const log;
-
-    /// These read settings are used to read from the source disk in copyFileFromDisk().
-    const ReadSettings read_settings;
+    virtual const ReadSettings & getReadSettings() const = 0;
+    virtual const WriteSettings & getWriteSettings() const = 0;
+    virtual size_t getWriteBufferSize() const = 0;
 };
 
 }
diff --git a/src/Backups/BackupIO_Default.cpp b/src/Backups/BackupIO_Default.cpp
new file mode 100644
index 00000000000..3b4851e9441
--- /dev/null
+++ b/src/Backups/BackupIO_Default.cpp
@@ -0,0 +1,95 @@
+#include <Backups/BackupIO_Default.h>
+
+#include <Disks/IDisk.h>
+#include <IO/copyData.h>
+#include <IO/WriteBufferFromFileBase.h>
+#include <IO/SeekableReadBuffer.h>
+#include <Interpreters/Context.h>
+#include <Common/logger_useful.h>
+
+
+namespace DB
+{
+
+BackupReaderDefault::BackupReaderDefault(Poco::Logger * log_, const ContextPtr & context_)
+    : log(log_)
+    , read_settings(context_->getBackupReadSettings())
+    , write_settings(context_->getWriteSettings())
+    , write_buffer_size(DBMS_DEFAULT_BUFFER_SIZE)
+{
+}
+
+void BackupReaderDefault::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
+                                         DiskPtr destination_disk, const String & destination_path, WriteMode write_mode)
+{
+    LOG_TRACE(log, "Copying file {} to disk {} through buffers", path_in_backup, destination_disk->getName());
+
+    auto read_buffer = readFile(path_in_backup);
+
+    std::unique_ptr<WriteBuffer> write_buffer;
+    auto buf_size = std::min(file_size, write_buffer_size);
+    if (encrypted_in_backup)
+        write_buffer = destination_disk->writeEncryptedFile(destination_path, buf_size, write_mode, write_settings);
+    else
+        write_buffer = destination_disk->writeFile(destination_path, buf_size, write_mode, write_settings);
+
+    copyData(*read_buffer, *write_buffer, file_size);
+    write_buffer->finalize();
+}
+
+BackupWriterDefault::BackupWriterDefault(Poco::Logger * log_, const ContextPtr & context_)
+    : log(log_)
+    , read_settings(context_->getBackupReadSettings())
+    , write_settings(context_->getWriteSettings())
+    , write_buffer_size(DBMS_DEFAULT_BUFFER_SIZE)
+{
+}
+
+bool BackupWriterDefault::fileContentsEqual(const String & file_name, const String & expected_file_contents)
+{
+    if (!fileExists(file_name))
+        return false;
+
+    try
+    {
+        auto in = readFile(file_name, expected_file_contents.size());
+        String actual_file_contents(expected_file_contents.size(), ' ');
+        return (in->read(actual_file_contents.data(), actual_file_contents.size()) == actual_file_contents.size())
+            && (actual_file_contents == expected_file_contents) && in->eof();
+    }
+    catch (...)
+    {
+        tryLogCurrentException(__PRETTY_FUNCTION__);
+        return false;
+    }
+}
+
+void BackupWriterDefault::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length)
+{
+    auto read_buffer = create_read_buffer();
+
+    if (start_pos)
+        read_buffer->seek(start_pos, SEEK_SET);
+
+    auto write_buffer = writeFile(path_in_backup);
+
+    copyData(*read_buffer, *write_buffer, length);
+    write_buffer->finalize();
+}
+
+void BackupWriterDefault::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
+                                           bool copy_encrypted, UInt64 start_pos, UInt64 length)
+{
+    LOG_TRACE(log, "Copying file {} from disk {} through buffers", src_path, src_disk->getName());
+
+    auto create_read_buffer = [this, src_disk, src_path, file_size = start_pos + length, copy_encrypted]
+    {
+        if (copy_encrypted)
+            return src_disk->readEncryptedFile(src_path, read_settings, {}, file_size);
+        else
+            return src_disk->readFile(src_path, read_settings, {}, file_size);
+    };
+
+    copyDataToFile(path_in_backup, create_read_buffer, start_pos, length);
+}
+}
diff --git a/src/Backups/BackupIO_Default.h b/src/Backups/BackupIO_Default.h
new file mode 100644
index 00000000000..0fc510f9361
--- /dev/null
+++ b/src/Backups/BackupIO_Default.h
@@ -0,0 +1,73 @@
+#pragma once
+
+#include <Backups/BackupIO.h>
+#include <IO/ReadSettings.h>
+#include <IO/WriteSettings.h>
+#include <Interpreters/Context_fwd.h>
+
+
+namespace DB
+{
+class IDisk;
+using DiskPtr = std::shared_ptr<IDisk>;
+class ReadBuffer;
+class SeekableReadBuffer;
+class WriteBuffer;
+enum class WriteMode;
+
+/// Represents operations of loading from disk or downloading for reading a backup.
+class BackupReaderDefault : public IBackupReader
+{
+public:
+    BackupReaderDefault(Poco::Logger * log_, const ContextPtr & context_);
+    ~BackupReaderDefault() override = default;
+
+    /// The function copyFileToDisk() can be much faster than reading the file with readFile() and then writing it to some disk.
+    /// (especially for S3 where it can use CopyObject to copy objects inside S3 instead of downloading and uploading them).
+    /// Parameters:
+    /// `encrypted_in_backup` specify if this file is encrypted in the backup,  so it shouldn't be encrypted again while restoring to an encrypted disk.
+    void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
+                        DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) override;
+    
+    const ReadSettings & getReadSettings() const override { return read_settings; }
+    const WriteSettings & getWriteSettings() const override { return write_settings; }
+    size_t getWriteBufferSize() const override { return write_buffer_size; }
+
+protected:
+    Poco::Logger * const log;
+    const ReadSettings read_settings;
+
+    /// The write settings are used to write to the source disk in copyFileToDisk().
+    const WriteSettings write_settings;
+    const size_t write_buffer_size;
+};
+
+/// Represents operations of storing to disk or uploading for writing a backup.
+class BackupWriterDefault : public IBackupWriter
+{
+public:
+    BackupWriterDefault(Poco::Logger * log_, const ContextPtr & context_);
+    ~BackupWriterDefault() override = default;
+
+    bool fileContentsEqual(const String & file_name, const String & expected_file_contents) override;
+    void copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) override;
+    void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, bool copy_encrypted, UInt64 start_pos, UInt64 length) override;
+
+    const ReadSettings & getReadSettings() const override { return read_settings; }
+    const WriteSettings & getWriteSettings() const override { return write_settings; }
+    size_t getWriteBufferSize() const override { return write_buffer_size; }
+
+protected:
+    /// Here readFile() is used only to implement fileContentsEqual().
+    virtual std::unique_ptr<ReadBuffer> readFile(const String & file_name, size_t expected_file_size) = 0;
+
+    Poco::Logger * const log;
+
+    /// The read settings are used to read from the source disk in copyFileFromDisk().
+    const ReadSettings read_settings;
+
+    const WriteSettings write_settings;
+    const size_t write_buffer_size;
+};
+
+}
diff --git a/src/Backups/BackupIO_Disk.cpp b/src/Backups/BackupIO_Disk.cpp
index d24d90ae7ae..f64e929131c 100644
--- a/src/Backups/BackupIO_Disk.cpp
+++ b/src/Backups/BackupIO_Disk.cpp
@@ -3,7 +3,6 @@
 #include <Disks/IDisk.h>
 #include <IO/ReadBufferFromFileBase.h>
 #include <IO/WriteBufferFromFileBase.h>
-#include <Interpreters/Context.h>
 
 
 namespace DB
@@ -14,8 +13,10 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-BackupReaderDisk::BackupReaderDisk(const DiskPtr & disk_, const String & path_)
-    : IBackupReader(&Poco::Logger::get("BackupReaderDisk")), disk(disk_), path(path_)
+BackupReaderDisk::BackupReaderDisk(const DiskPtr & disk_, const String & root_path_, const ContextPtr & context_)
+    : BackupReaderDefault(&Poco::Logger::get("BackupReaderDisk"), context_)
+    , disk(disk_)
+    , root_path(root_path_)
 {
 }
 
@@ -23,40 +24,39 @@ BackupReaderDisk::~BackupReaderDisk() = default;
 
 bool BackupReaderDisk::fileExists(const String & file_name)
 {
-    return disk->exists(path / file_name);
+    return disk->exists(root_path / file_name);
 }
 
 UInt64 BackupReaderDisk::getFileSize(const String & file_name)
 {
-    return disk->getFileSize(path / file_name);
+    return disk->getFileSize(root_path / file_name);
 }
 
 std::unique_ptr<SeekableReadBuffer> BackupReaderDisk::readFile(const String & file_name)
 {
-    return disk->readFile(path / file_name);
+    return disk->readFile(root_path / file_name, read_settings);
 }
 
 void BackupReaderDisk::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
-                                      DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings)
+                                      DiskPtr destination_disk, const String & destination_path, WriteMode write_mode)
 {
     if ((write_mode == WriteMode::Rewrite) && !encrypted_in_backup)
     {
         /// Use more optimal way.
         LOG_TRACE(log, "Copying file {} from disk {} to disk {}", path_in_backup, disk->getName(), destination_disk->getName());
-        disk->copyFile(path / path_in_backup, *destination_disk, destination_path, write_settings);
+        disk->copyFile(root_path / path_in_backup, *destination_disk, destination_path, write_settings);
         return;
     }
 
     /// Fallback to copy through buffers.
-    IBackupReader::copyFileToDisk(path_in_backup, file_size, encrypted_in_backup, destination_disk, destination_path, write_mode, write_settings);
+    BackupReaderDefault::copyFileToDisk(path_in_backup, file_size, encrypted_in_backup, destination_disk, destination_path, write_mode);
 }
 
 
-BackupWriterDisk::BackupWriterDisk(const DiskPtr & disk_, const String & path_, const ContextPtr & context_)
-    : IBackupWriter(context_, &Poco::Logger::get("BackupWriterDisk"))
+BackupWriterDisk::BackupWriterDisk(const DiskPtr & disk_, const String & root_path_, const ContextPtr & context_)
+    : BackupWriterDefault(&Poco::Logger::get("BackupWriterDisk"), context_)
     , disk(disk_)
-    , path(path_)
-    , has_throttling(static_cast<bool>(context_->getBackupsThrottler()))
+    , root_path(root_path_)
 {
 }
 
@@ -64,53 +64,39 @@ BackupWriterDisk::~BackupWriterDisk() = default;
 
 bool BackupWriterDisk::fileExists(const String & file_name)
 {
-    return disk->exists(path / file_name);
+    return disk->exists(root_path / file_name);
 }
 
 UInt64 BackupWriterDisk::getFileSize(const String & file_name)
 {
-    return disk->getFileSize(path / file_name);
+    return disk->getFileSize(root_path / file_name);
 }
 
-bool BackupWriterDisk::fileContentsEqual(const String & file_name, const String & expected_file_contents)
+std::unique_ptr<ReadBuffer> BackupWriterDisk::readFile(const String & file_name, size_t expected_file_size)
 {
-    if (!disk->exists(path / file_name))
-        return false;
-
-    try
-    {
-        auto in = disk->readFile(path / file_name);
-        String actual_file_contents(expected_file_contents.size(), ' ');
-        return (in->read(actual_file_contents.data(), actual_file_contents.size()) == actual_file_contents.size())
-            && (actual_file_contents == expected_file_contents) && in->eof();
-    }
-    catch (...)
-    {
-        tryLogCurrentException(__PRETTY_FUNCTION__);
-        return false;
-    }
+    return disk->readFile(file_name, read_settings, {}, expected_file_size);
 }
 
 std::unique_ptr<WriteBuffer> BackupWriterDisk::writeFile(const String & file_name)
 {
-    auto file_path = path / file_name;
+    auto file_path = root_path / file_name;
     disk->createDirectories(file_path.parent_path());
-    return disk->writeFile(file_path);
+    return disk->writeFile(file_path, write_buffer_size, WriteMode::Rewrite, write_settings);
 }
 
 void BackupWriterDisk::removeFile(const String & file_name)
 {
-    disk->removeFileIfExists(path / file_name);
-    if (disk->isDirectory(path) && disk->isDirectoryEmpty(path))
-        disk->removeDirectory(path);
+    disk->removeFileIfExists(root_path / file_name);
+    if (disk->isDirectory(root_path) && disk->isDirectoryEmpty(root_path))
+        disk->removeDirectory(root_path);
 }
 
 void BackupWriterDisk::removeFiles(const Strings & file_names)
 {
     for (const auto & file_name : file_names)
-        disk->removeFileIfExists(path / file_name);
-    if (disk->isDirectory(path) && disk->isDirectoryEmpty(path))
-        disk->removeDirectory(path);
+        disk->removeFileIfExists(root_path / file_name);
+    if (disk->isDirectory(root_path) && disk->isDirectoryEmpty(root_path))
+        disk->removeDirectory(root_path);
 }
 
 void BackupWriterDisk::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
@@ -120,14 +106,14 @@ void BackupWriterDisk::copyFileFromDisk(const String & path_in_backup, DiskPtr s
     {
         /// Use more optimal way.
         LOG_TRACE(log, "Copying file {} from disk {} to disk {}", src_path, src_disk->getName(), disk->getName());
-        auto dest_file_path = path / path_in_backup;
+        auto dest_file_path = root_path / path_in_backup;
         disk->createDirectories(dest_file_path.parent_path());
-        src_disk->copyFile(src_path, *disk, dest_file_path);
+        src_disk->copyFile(src_path, *disk, dest_file_path, write_settings);
         return;
     }
 
     /// Fallback to copy through buffers.
-    IBackupWriter::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length);
+    BackupWriterDefault::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length);
 }
 
 }
diff --git a/src/Backups/BackupIO_Disk.h b/src/Backups/BackupIO_Disk.h
index d2af06668eb..399ebeaa227 100644
--- a/src/Backups/BackupIO_Disk.h
+++ b/src/Backups/BackupIO_Disk.h
@@ -1,40 +1,42 @@
 #pragma once
 
+#include <Backups/BackupIO_Default.h>
 #include <filesystem>
-#include <Backups/BackupIO.h>
-#include <Interpreters/Context_fwd.h>
+
 
 namespace DB
 {
 class IDisk;
 using DiskPtr = std::shared_ptr<IDisk>;
 
-class BackupReaderDisk : public IBackupReader
+class BackupReaderDisk : public BackupReaderDefault
 {
 public:
-    BackupReaderDisk(const DiskPtr & disk_, const String & path_);
+    BackupReaderDisk(const DiskPtr & disk_, const String & root_path_, const ContextPtr & context_);
     ~BackupReaderDisk() override;
 
     bool fileExists(const String & file_name) override;
     UInt64 getFileSize(const String & file_name) override;
+
     std::unique_ptr<SeekableReadBuffer> readFile(const String & file_name) override;
+
     void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
-                        DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings) override;
+                        DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) override;
 
 private:
     const DiskPtr disk;
-    const std::filesystem::path path;
+    const std::filesystem::path root_path;
 };
 
-class BackupWriterDisk : public IBackupWriter
+class BackupWriterDisk : public BackupWriterDefault
 {
 public:
-    BackupWriterDisk(const DiskPtr & disk_, const String & path_, const ContextPtr & context_);
+    BackupWriterDisk(const DiskPtr & disk_, const String & root_path_, const ContextPtr & context_);
     ~BackupWriterDisk() override;
 
     bool fileExists(const String & file_name) override;
     UInt64 getFileSize(const String & file_name) override;
-    bool fileContentsEqual(const String & file_name, const String & expected_file_contents) override;
+
     std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
 
     void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
@@ -44,9 +46,10 @@ public:
     void removeFiles(const Strings & file_names) override;
 
 private:
+    std::unique_ptr<ReadBuffer> readFile(const String & file_name, size_t expected_file_size) override;
+
     const DiskPtr disk;
-    const std::filesystem::path path;
-    const bool has_throttling;
+    const std::filesystem::path root_path;
 };
 
 }
diff --git a/src/Backups/BackupIO_File.cpp b/src/Backups/BackupIO_File.cpp
index 69d899528cd..69265799793 100644
--- a/src/Backups/BackupIO_File.cpp
+++ b/src/Backups/BackupIO_File.cpp
@@ -1,12 +1,11 @@
 #include <Backups/BackupIO_File.h>
 #include <Disks/DiskLocal.h>
-#include <Disks/IDisk.h>
+//#include <Disks/IDisk.h>
 #include <Disks/IO/createReadBufferFromFileBase.h>
 #include <IO/WriteBufferFromFile.h>
-#include <IO/copyData.h>
-#include <Common/filesystemHelpers.h>
+//#include <IO/copyData.h>
+//#include <Common/filesystemHelpers.h>
 #include <Common/logger_useful.h>
-#include <Interpreters/Context.h>
 
 
 namespace fs = std::filesystem;
@@ -15,32 +14,30 @@ namespace fs = std::filesystem;
 namespace DB
 {
 
-BackupReaderFile::BackupReaderFile(const String & path_)
-    : IBackupReader(&Poco::Logger::get("BackupReaderFile"))
-    , path(path_)
-    , data_source_description(DiskLocal::getLocalDataSourceDescription(path_))
+BackupReaderFile::BackupReaderFile(const String & root_path_, const ContextPtr & context_)
+    : BackupReaderDefault(&Poco::Logger::get("BackupReaderFile"), context_)
+    , root_path(root_path_)
+    , data_source_description(DiskLocal::getLocalDataSourceDescription(root_path))
 {
 }
 
-BackupReaderFile::~BackupReaderFile() = default;
-
 bool BackupReaderFile::fileExists(const String & file_name)
 {
-    return fs::exists(path / file_name);
+    return fs::exists(root_path / file_name);
 }
 
 UInt64 BackupReaderFile::getFileSize(const String & file_name)
 {
-    return fs::file_size(path / file_name);
+    return fs::file_size(root_path / file_name);
 }
 
 std::unique_ptr<SeekableReadBuffer> BackupReaderFile::readFile(const String & file_name)
 {
-    return createReadBufferFromFileBase(path / file_name, {});
+    return createReadBufferFromFileBase(root_path / file_name, read_settings);
 }
 
 void BackupReaderFile::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
-                                      DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings)
+                                      DiskPtr destination_disk, const String & destination_path, WriteMode write_mode)
 {
     if (write_mode == WriteMode::Rewrite)
     {
@@ -52,7 +49,7 @@ void BackupReaderFile::copyFileToDisk(const String & path_in_backup, size_t file
             LOG_TRACE(log, "Copying file {} to disk {} locally", path_in_backup, destination_disk->getName());
 
             auto write_blob_function
-                = [abs_source_path = path / path_in_backup, file_size](
+                = [abs_source_path = root_path / path_in_backup, file_size](
                       const Strings & blob_path, WriteMode mode, const std::optional<ObjectAttributes> &) -> size_t
             {
                 if (blob_path.size() != 1 || mode != WriteMode::Rewrite)
@@ -69,69 +66,53 @@ void BackupReaderFile::copyFileToDisk(const String & path_in_backup, size_t file
     }
 
     /// Fallback to copy through buffers.
-    IBackupReader::copyFileToDisk(path_in_backup, file_size, encrypted_in_backup, destination_disk, destination_path, write_mode, write_settings);
+    BackupReaderDefault::copyFileToDisk(path_in_backup, file_size, encrypted_in_backup, destination_disk, destination_path, write_mode);
 }
 
 
-BackupWriterFile::BackupWriterFile(const String & path_, const ContextPtr & context_)
-    : IBackupWriter(context_, &Poco::Logger::get("BackupWriterFile"))
-    , path(path_)
-    , data_source_description(DiskLocal::getLocalDataSourceDescription(path_))
-    , has_throttling(static_cast<bool>(context_->getBackupsThrottler()))
+BackupWriterFile::BackupWriterFile(const String & root_path_, const ContextPtr & context_)
+    : BackupWriterDefault(&Poco::Logger::get("BackupWriterFile"), context_)
+    , root_path(root_path_)
+    , data_source_description(DiskLocal::getLocalDataSourceDescription(root_path))
+    , has_throttling(static_cast<bool>(read_settings.local_throttler))
 {
 }
 
-BackupWriterFile::~BackupWriterFile() = default;
-
 bool BackupWriterFile::fileExists(const String & file_name)
 {
-    return fs::exists(path / file_name);
+    return fs::exists(root_path / file_name);
 }
 
 UInt64 BackupWriterFile::getFileSize(const String & file_name)
 {
-    return fs::file_size(path / file_name);
+    return fs::file_size(root_path / file_name);
 }
 
-bool BackupWriterFile::fileContentsEqual(const String & file_name, const String & expected_file_contents)
+std::unique_ptr<ReadBuffer> BackupWriterFile::readFile(const String & file_name, size_t expected_file_size)
 {
-    if (!fs::exists(path / file_name))
-        return false;
-
-    try
-    {
-        auto in = createReadBufferFromFileBase(path / file_name, {});
-        String actual_file_contents(expected_file_contents.size(), ' ');
-        return (in->read(actual_file_contents.data(), actual_file_contents.size()) == actual_file_contents.size())
-            && (actual_file_contents == expected_file_contents) && in->eof();
-    }
-    catch (...)
-    {
-        tryLogCurrentException(__PRETTY_FUNCTION__);
-        return false;
-    }
+    return createReadBufferFromFileBase(root_path / file_name, read_settings, {}, expected_file_size);
 }
 
 std::unique_ptr<WriteBuffer> BackupWriterFile::writeFile(const String & file_name)
 {
-    auto file_path = path / file_name;
+    auto file_path = root_path / file_name;
     fs::create_directories(file_path.parent_path());
-    return std::make_unique<WriteBufferFromFile>(file_path);
+    return std::make_unique<WriteBufferFromFile>(file_path, write_buffer_size, -1, write_settings.local_throttler);
 }
 
 void BackupWriterFile::removeFile(const String & file_name)
 {
-    fs::remove(path / file_name);
-    if (fs::is_directory(path) && fs::is_empty(path))
-        fs::remove(path);
+    fs::remove(root_path / file_name);
+    if (fs::is_directory(root_path) && fs::is_empty(root_path))
+        fs::remove(root_path);
 }
 
 void BackupWriterFile::removeFiles(const Strings & file_names)
 {
     for (const auto & file_name : file_names)
-        fs::remove(path / file_name);
-    if (fs::is_directory(path) && fs::is_empty(path))
-        fs::remove(path);
+        fs::remove(root_path / file_name);
+    if (fs::is_directory(root_path) && fs::is_empty(root_path))
+        fs::remove(root_path);
 }
 
 void BackupWriterFile::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
@@ -144,6 +125,7 @@ void BackupWriterFile::copyFileFromDisk(const String & path_in_backup, DiskPtr s
         if (source_data_source_description.sameKind(data_source_description)
             && (source_data_source_description.is_encrypted == copy_encrypted))
         {
+            /// std::filesystem::copy() can copy from a single file only.
             if (auto blob_path = src_disk->getBlobPath(src_path); blob_path.size() == 1)
             {
                 auto abs_source_path = blob_path[0];
@@ -153,7 +135,7 @@ void BackupWriterFile::copyFileFromDisk(const String & path_in_backup, DiskPtr s
                 {
                     /// Use more optimal way.
                     LOG_TRACE(log, "Copying file {} from disk {} locally", src_path, src_disk->getName());
-                    auto abs_dest_path = path / path_in_backup;
+                    auto abs_dest_path = root_path / path_in_backup;
                     fs::create_directories(abs_dest_path.parent_path());
                     fs::copy(abs_source_path, abs_dest_path, fs::copy_options::overwrite_existing);
                     return;
@@ -163,7 +145,7 @@ void BackupWriterFile::copyFileFromDisk(const String & path_in_backup, DiskPtr s
     }
 
     /// Fallback to copy through buffers.
-    IBackupWriter::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length);
+    BackupWriterDefault::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length);
 }
 
 }
diff --git a/src/Backups/BackupIO_File.h b/src/Backups/BackupIO_File.h
index a8ada969ca4..45fc0d47115 100644
--- a/src/Backups/BackupIO_File.h
+++ b/src/Backups/BackupIO_File.h
@@ -1,39 +1,38 @@
 #pragma once
 
+#include <Backups/BackupIO_Default.h>
+#include <Disks/DiskType.h>
 #include <filesystem>
-#include <Backups/BackupIO.h>
-#include <Interpreters/Context_fwd.h>
+
 
 namespace DB
 {
 
-class BackupReaderFile : public IBackupReader
+class BackupReaderFile : public BackupReaderDefault
 {
 public:
-    explicit BackupReaderFile(const String & path_);
-    ~BackupReaderFile() override;
+    explicit BackupReaderFile(const String & root_path_, const ContextPtr & context_);
 
     bool fileExists(const String & file_name) override;
     UInt64 getFileSize(const String & file_name) override;
+
     std::unique_ptr<SeekableReadBuffer> readFile(const String & file_name) override;
 
     void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
-                        DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings) override;
+                        DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) override;
 
 private:
-    const std::filesystem::path path;
+    const std::filesystem::path root_path;
     const DataSourceDescription data_source_description;
 };
 
-class BackupWriterFile : public IBackupWriter
+class BackupWriterFile : public BackupWriterDefault
 {
 public:
-    BackupWriterFile(const String & path_, const ContextPtr & context_);
-    ~BackupWriterFile() override;
+    BackupWriterFile(const String & root_path_, const ContextPtr & context_);
 
     bool fileExists(const String & file_name) override;
     UInt64 getFileSize(const String & file_name) override;
-    bool fileContentsEqual(const String & file_name, const String & expected_file_contents) override;
     std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
 
     void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
@@ -43,7 +42,9 @@ public:
     void removeFiles(const Strings & file_names) override;
 
 private:
-    const std::filesystem::path path;
+    std::unique_ptr<ReadBuffer> readFile(const String & file_name, size_t expected_file_size) override;
+
+    const std::filesystem::path root_path;
     const DataSourceDescription data_source_description;
     const bool has_throttling;
 };
diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp
index 00694a5e01d..8d9d34bf9b5 100644
--- a/src/Backups/BackupIO_S3.cpp
+++ b/src/Backups/BackupIO_S3.cpp
@@ -11,6 +11,7 @@
 #include <IO/S3/copyS3File.h>
 #include <IO/S3/Client.h>
 #include <IO/S3/Credentials.h>
+#include <Disks/IDisk.h>
 
 #include <Poco/Util/AbstractConfiguration.h>
 
@@ -101,10 +102,9 @@ namespace
 
 BackupReaderS3::BackupReaderS3(
     const S3::URI & s3_uri_, const String & access_key_id_, const String & secret_access_key_, const ContextPtr & context_)
-    : IBackupReader(&Poco::Logger::get("BackupReaderS3"))
+    : BackupReaderDefault(&Poco::Logger::get("BackupReaderS3"), context_)
     , s3_uri(s3_uri_)
     , client(makeS3Client(s3_uri_, access_key_id_, secret_access_key_, context_))
-    , read_settings(context_->getReadSettings())
     , request_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString()).request_settings)
     , data_source_description{DataSourceType::S3, s3_uri.endpoint, false, false}
 {
@@ -133,7 +133,7 @@ std::unique_ptr<SeekableReadBuffer> BackupReaderS3::readFile(const String & file
 }
 
 void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
-                                    DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings)
+                                    DiskPtr destination_disk, const String & destination_path, WriteMode write_mode)
 {
     auto destination_data_source_description = destination_disk->getDataSourceDescription();
     if (destination_data_source_description.sameKind(data_source_description)
@@ -170,13 +170,13 @@ void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_s
     }
 
     /// Fallback to copy through buffers.
-    IBackupReader::copyFileToDisk(path_in_backup, file_size, encrypted_in_backup, destination_disk, destination_path, write_mode, write_settings);
+    BackupReaderDefault::copyFileToDisk(path_in_backup, file_size, encrypted_in_backup, destination_disk, destination_path, write_mode);
 }
 
 
 BackupWriterS3::BackupWriterS3(
     const S3::URI & s3_uri_, const String & access_key_id_, const String & secret_access_key_, const ContextPtr & context_)
-    : IBackupWriter(context_, &Poco::Logger::get("BackupWriterS3"))
+    : BackupWriterDefault(&Poco::Logger::get("BackupWriterS3"), context_)
     , s3_uri(s3_uri_)
     , client(makeS3Client(s3_uri_, access_key_id_, secret_access_key_, context_))
     , request_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString()).request_settings)
@@ -193,8 +193,8 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src
     if (source_data_source_description.sameKind(data_source_description)
         && (source_data_source_description.is_encrypted == copy_encrypted))
     {
-        /// getBlobPath() can return std::nullopt if the file is stored as multiple objects in S3 bucket.
-        /// In this case we can't use native copy.
+        /// getBlobPath() can return more than 2 elements if the file is stored as multiple objects in S3 bucket.
+        /// In this case we can't use the native copy.
         if (auto blob_path = src_disk->getBlobPath(src_path); blob_path.size() == 2)
         {
             /// Use native copy, the more optimal way.
@@ -215,7 +215,7 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src
     }
 
     /// Fallback to copy through buffers.
-    IBackupWriter::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length);
+    BackupWriterDefault::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length);
 }
 
 void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length)
@@ -239,24 +239,11 @@ UInt64 BackupWriterS3::getFileSize(const String & file_name)
     return objects[0].GetSize();
 }
 
-bool BackupWriterS3::fileContentsEqual(const String & file_name, const String & expected_file_contents)
+std::unique_ptr<ReadBuffer> BackupWriterS3::readFile(const String & file_name, size_t expected_file_size)
 {
-    if (listObjects(*client, s3_uri, file_name).empty())
-        return false;
-
-    try
-    {
-        auto in = std::make_unique<ReadBufferFromS3>(
-            client, s3_uri.bucket, fs::path(s3_uri.key) / file_name, s3_uri.version_id, request_settings, read_settings);
-        String actual_file_contents(expected_file_contents.size(), ' ');
-        return (in->read(actual_file_contents.data(), actual_file_contents.size()) == actual_file_contents.size())
-            && (actual_file_contents == expected_file_contents) && in->eof();
-    }
-    catch (...)
-    {
-        tryLogCurrentException(__PRETTY_FUNCTION__);
-        return false;
-    }
+    return std::make_unique<ReadBufferFromS3>(
+            client, s3_uri.bucket, fs::path(s3_uri.key) / file_name, s3_uri.version_id, request_settings, read_settings,
+            false, 0, 0, false, expected_file_size);
 }
 
 std::unique_ptr<WriteBuffer> BackupWriterS3::writeFile(const String & file_name)
@@ -267,7 +254,8 @@ std::unique_ptr<WriteBuffer> BackupWriterS3::writeFile(const String & file_name)
         fs::path(s3_uri.key) / file_name,
         request_settings,
         std::nullopt,
-        threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupWriterS3"));
+        threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupWriterS3"),
+        write_settings);
 }
 
 void BackupWriterS3::removeFile(const String & file_name)
diff --git a/src/Backups/BackupIO_S3.h b/src/Backups/BackupIO_S3.h
index 1db9d5f8c4c..cca56bae6bc 100644
--- a/src/Backups/BackupIO_S3.h
+++ b/src/Backups/BackupIO_S3.h
@@ -3,8 +3,8 @@
 #include "config.h"
 
 #if USE_AWS_S3
-#include <Backups/BackupIO.h>
-#include <IO/ReadSettings.h>
+#include <Backups/BackupIO_Default.h>
+#include <Disks/DiskType.h>
 #include <IO/S3Common.h>
 #include <Storages/StorageS3Settings.h>
 #include <Interpreters/Context_fwd.h>
@@ -14,7 +14,7 @@ namespace DB
 {
 
 /// Represents a backup stored to AWS S3.
-class BackupReaderS3 : public IBackupReader
+class BackupReaderS3 : public BackupReaderDefault
 {
 public:
     BackupReaderS3(const S3::URI & s3_uri_, const String & access_key_id_, const String & secret_access_key_, const ContextPtr & context_);
@@ -25,18 +25,17 @@ public:
     std::unique_ptr<SeekableReadBuffer> readFile(const String & file_name) override;
 
     void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
-                        DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, const WriteSettings & write_settings) override;
+                        DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) override;
 
 private:
     const S3::URI s3_uri;
     const std::shared_ptr<S3::Client> client;
-    const ReadSettings read_settings;
     S3Settings::RequestSettings request_settings;
     const DataSourceDescription data_source_description;
 };
 
 
-class BackupWriterS3 : public IBackupWriter
+class BackupWriterS3 : public BackupWriterDefault
 {
 public:
     BackupWriterS3(const S3::URI & s3_uri_, const String & access_key_id_, const String & secret_access_key_, const ContextPtr & context_);
@@ -44,7 +43,6 @@ public:
 
     bool fileExists(const String & file_name) override;
     UInt64 getFileSize(const String & file_name) override;
-    bool fileContentsEqual(const String & file_name, const String & expected_file_contents) override;
     std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
 
     void copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) override;
@@ -55,6 +53,7 @@ public:
     void removeFiles(const Strings & file_names) override;
 
 private:
+    std::unique_ptr<ReadBuffer> readFile(const String & file_name, size_t expected_file_size) override;
     void removeFilesBatch(const Strings & file_names);
 
     const S3::URI s3_uri;
diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp
index 9bfa2d77353..715fe9e0d13 100644
--- a/src/Backups/BackupImpl.cpp
+++ b/src/Backups/BackupImpl.cpp
@@ -737,14 +737,14 @@ std::unique_ptr<SeekableReadBuffer> BackupImpl::readFileImpl(const SizeAndChecks
     }
 }
 
-size_t BackupImpl::copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path,
-                                  WriteMode write_mode, const WriteSettings & write_settings) const
+size_t BackupImpl::copyFileToDisk(const String & file_name,
+                                  DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const
 {
-    return copyFileToDisk(getFileSizeAndChecksum(file_name), destination_disk, destination_path, write_mode, write_settings);
+    return copyFileToDisk(getFileSizeAndChecksum(file_name), destination_disk, destination_path, write_mode);
 }
 
-size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path,
-                                  WriteMode write_mode, const WriteSettings & write_settings) const
+size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum,
+                                  DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const
 {
     if (open_mode != OpenMode::READ)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Backup is not opened for reading");
@@ -790,13 +790,13 @@ size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum, Dis
     if (info.size && !info.base_size && !use_archive)
     {
         /// Data comes completely from this backup.
-        reader->copyFileToDisk(info.data_file_name, info.size, info.encrypted_by_disk, destination_disk, destination_path, write_mode, write_settings);
+        reader->copyFileToDisk(info.data_file_name, info.size, info.encrypted_by_disk, destination_disk, destination_path, write_mode);
         file_copied = true;
     }
     else if (info.size && (info.size == info.base_size))
     {
         /// Data comes completely from the base backup (nothing comes from this backup).
-        base_backup->copyFileToDisk(std::pair{info.base_size, info.base_checksum}, destination_disk, destination_path, write_mode, write_settings);
+        base_backup->copyFileToDisk(std::pair{info.base_size, info.base_checksum}, destination_disk, destination_path, write_mode);
         file_copied = true;
     }
 
@@ -812,11 +812,11 @@ size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum, Dis
         /// Use the generic way to copy data. `readFile()` will update `num_read_files`.
         auto read_buffer = readFileImpl(size_and_checksum, /* read_encrypted= */ info.encrypted_by_disk);
         std::unique_ptr<WriteBuffer> write_buffer;
-        size_t buf_size = std::min<size_t>(info.size, DBMS_DEFAULT_BUFFER_SIZE);
+        size_t buf_size = std::min<size_t>(info.size, reader->getWriteBufferSize());
         if (info.encrypted_by_disk)
-            write_buffer = destination_disk->writeEncryptedFile(destination_path, buf_size, write_mode, write_settings);
+            write_buffer = destination_disk->writeEncryptedFile(destination_path, buf_size, write_mode, reader->getWriteSettings());
         else
-            write_buffer = destination_disk->writeFile(destination_path, buf_size, write_mode, write_settings);
+            write_buffer = destination_disk->writeFile(destination_path, buf_size, write_mode, reader->getWriteSettings());
         copyData(*read_buffer, *write_buffer, info.size);
         write_buffer->finalize();
     }
diff --git a/src/Backups/BackupImpl.h b/src/Backups/BackupImpl.h
index 511b100c557..7e95d156162 100644
--- a/src/Backups/BackupImpl.h
+++ b/src/Backups/BackupImpl.h
@@ -76,10 +76,8 @@ public:
     SizeAndChecksum getFileSizeAndChecksum(const String & file_name) const override;
     std::unique_ptr<SeekableReadBuffer> readFile(const String & file_name) const override;
     std::unique_ptr<SeekableReadBuffer> readFile(const SizeAndChecksum & size_and_checksum) const override;
-    size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path,
-                          WriteMode write_mode, const WriteSettings & write_settings) const override;
-    size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path,
-                          WriteMode write_mode, const WriteSettings & write_settings) const override;
+    size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const override;
+    size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const override;
     void writeFile(const BackupFileInfo & info, BackupEntryPtr entry) override;
     void finalizeWriting() override;
     bool supportsWritingInMultipleThreads() const override { return !use_archive; }
diff --git a/src/Backups/IBackup.h b/src/Backups/IBackup.h
index 031130fa3b4..660f7d5da22 100644
--- a/src/Backups/IBackup.h
+++ b/src/Backups/IBackup.h
@@ -109,10 +109,10 @@ public:
 
     /// Copies a file from the backup to a specified destination disk. Returns the number of bytes written.
     virtual size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path,
-                                  WriteMode write_mode = WriteMode::Rewrite, const WriteSettings & write_settings = {}) const = 0;
+                                  WriteMode write_mode = WriteMode::Rewrite) const = 0;
 
     virtual size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path,
-                                  WriteMode write_mode = WriteMode::Rewrite, const WriteSettings & write_settings = {}) const = 0;
+                                  WriteMode write_mode = WriteMode::Rewrite) const = 0;
 
     /// Puts a new entry to the backup.
     virtual void writeFile(const BackupFileInfo & file_info, BackupEntryPtr entry) = 0;
diff --git a/src/Backups/registerBackupEnginesFileAndDisk.cpp b/src/Backups/registerBackupEnginesFileAndDisk.cpp
index 46f44471e6f..daae9627759 100644
--- a/src/Backups/registerBackupEnginesFileAndDisk.cpp
+++ b/src/Backups/registerBackupEnginesFileAndDisk.cpp
@@ -169,9 +169,9 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
         {
             std::shared_ptr<IBackupReader> reader;
             if (engine_name == "File")
-                reader = std::make_shared<BackupReaderFile>(path);
+                reader = std::make_shared<BackupReaderFile>(path, params.context);
             else
-                reader = std::make_shared<BackupReaderDisk>(disk, path);
+                reader = std::make_shared<BackupReaderDisk>(disk, path, params.context);
             return std::make_unique<BackupImpl>(backup_name_for_logging, archive_params, params.base_backup_info, reader, params.context);
         }
         else
diff --git a/src/Disks/DiskEncrypted.h b/src/Disks/DiskEncrypted.h
index be726ef46b4..db53accc062 100644
--- a/src/Disks/DiskEncrypted.h
+++ b/src/Disks/DiskEncrypted.h
@@ -199,10 +199,12 @@ public:
     }
 
     std::unique_ptr<ReadBufferFromFileBase> readEncryptedFile(
-        const String & path, const ReadSettings & settings) const override
+        const String & path, const ReadSettings & settings,
+        std::optional<size_t> read_hint,
+        std::optional<size_t> file_size) const override
     {
         auto wrapped_path = wrappedPath(path);
-        return delegate->readFile(wrapped_path, settings);
+        return delegate->readFile(wrapped_path, settings, read_hint, file_size);
     }
 
     std::unique_ptr<WriteBufferFromFileBase> writeEncryptedFile(
diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp
index 54e09b09d2f..3c8ca454463 100644
--- a/src/Disks/IDisk.cpp
+++ b/src/Disks/IDisk.cpp
@@ -52,7 +52,7 @@ void IDisk::removeSharedFiles(const RemoveBatchRequest & files, bool keep_all_ba
     }
 }
 
-std::unique_ptr<ReadBufferFromFileBase> IDisk::readEncryptedFile(const String &, const ReadSettings &) const
+std::unique_ptr<ReadBufferFromFileBase> IDisk::readEncryptedFile(const String &, const ReadSettings &, std::optional<size_t>, std::optional<size_t>) const
 {
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "File encryption is not implemented for disk of type {}", getDataSourceDescription().type);
 }
diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h
index 006f2d882a0..770eec46ce6 100644
--- a/src/Disks/IDisk.h
+++ b/src/Disks/IDisk.h
@@ -254,7 +254,10 @@ public:
 
     /// Reads a file from an encrypted disk without decrypting it.
     virtual std::unique_ptr<ReadBufferFromFileBase> readEncryptedFile(
-        const String & path, const ReadSettings & settings = ReadSettings{}) const;
+        const String & path,
+        const ReadSettings & settings = ReadSettings{},
+        std::optional<size_t> read_hint = {},
+        std::optional<size_t> file_size = {}) const;
 
     /// Writes an already encrypted file to an encrypted disk.
     virtual std::unique_ptr<WriteBufferFromFileBase> writeEncryptedFile(
diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp
index c080c2cd92d..bfa1ed1ab26 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp
@@ -584,7 +584,9 @@ Strings DiskObjectStorage::getBlobPath(const String & path) const
     auto objects = getStorageObjects(path);
     Strings res;
     res.reserve(objects.size() + 1);
-    res.emplace_back(object_storage->getObjectsNamespace());
+    String objects_namespace = object_storage->getObjectsNamespace();
+    if (!objects_namespace.empty())
+        res.emplace_back(objects_namespace);
     for (const auto & object : objects)
         res.emplace_back(object.absolute_path);
     return res;

From 517e119e0385079b22531032ece33f4a8ed45977 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Mon, 1 May 2023 14:23:59 +0200
Subject: [PATCH 562/689] Move checksum calculation to IBackupEntry.

---
 src/Backups/BackupEntryFromAppendOnlyFile.cpp | 45 +++++++++++---
 src/Backups/BackupEntryFromAppendOnlyFile.h   | 26 +++++---
 src/Backups/BackupEntryFromImmutableFile.cpp  | 60 +++++++++++++------
 src/Backups/BackupEntryFromImmutableFile.h    | 23 +++----
 src/Backups/BackupEntryFromMemory.cpp         |  7 +--
 src/Backups/BackupEntryFromMemory.h           | 18 ++----
 src/Backups/BackupEntryFromSmallFile.cpp      |  9 +--
 src/Backups/BackupEntryFromSmallFile.h        | 22 ++-----
 .../BackupEntryWithChecksumCalculation.cpp    | 48 +++++++++++++++
 .../BackupEntryWithChecksumCalculation.h      | 22 +++++++
 src/Backups/BackupEntryWrappedWith.h          |  7 ++-
 src/Backups/BackupFileInfo.cpp                | 39 +++---------
 src/Backups/BackupIO_File.cpp                 |  3 -
 src/Backups/IBackupEntriesLazyBatch.cpp       |  7 ++-
 src/Backups/IBackupEntry.h                    |  9 ++-
 src/Disks/DiskEncrypted.cpp                   |  7 +++
 src/Disks/DiskEncrypted.h                     |  2 +
 src/Storages/StorageReplicatedMergeTree.cpp   |  3 +-
 18 files changed, 228 insertions(+), 129 deletions(-)
 create mode 100644 src/Backups/BackupEntryWithChecksumCalculation.cpp
 create mode 100644 src/Backups/BackupEntryWithChecksumCalculation.h

diff --git a/src/Backups/BackupEntryFromAppendOnlyFile.cpp b/src/Backups/BackupEntryFromAppendOnlyFile.cpp
index 83117f686bf..e3b79695a0c 100644
--- a/src/Backups/BackupEntryFromAppendOnlyFile.cpp
+++ b/src/Backups/BackupEntryFromAppendOnlyFile.cpp
@@ -1,25 +1,52 @@
 #include <Backups/BackupEntryFromAppendOnlyFile.h>
+#include <Disks/DiskEncrypted.h>
 #include <IO/LimitSeekableReadBuffer.h>
 
 
 namespace DB
 {
 
+namespace
+{
+    UInt64 calculateSize(const DiskPtr & disk, const String & file_path, const std::optional<UInt64> & file_size, bool disk_is_encrypted)
+    {
+        if (file_size)
+        {
+            if (disk_is_encrypted)
+                return DiskEncrypted::convertFileSizeToEncryptedFileSize(*file_size);
+            else
+                return *file_size;
+        }
+        else
+        {
+            if (disk_is_encrypted)
+                return disk->getEncryptedFileSize(file_path);
+            else
+                return disk->getFileSize(file_path);
+        }
+    }
+}
+
 BackupEntryFromAppendOnlyFile::BackupEntryFromAppendOnlyFile(
-    const DiskPtr & disk_,
-    const String & file_path_,
-    const ReadSettings & settings_,
-    const std::optional<UInt64> & file_size_,
-    const std::optional<UInt128> & checksum_)
-    : BackupEntryFromImmutableFile(disk_, file_path_, settings_, file_size_, checksum_)
-    , limit(BackupEntryFromImmutableFile::getSize())
+    const DiskPtr & disk_, const String & file_path_, const ReadSettings & settings_, const std::optional<UInt64> & file_size_)
+    : disk(disk_)
+    , file_path(file_path_)
+    , data_source_description(disk->getDataSourceDescription())
+    , settings(settings_)
+    , size(calculateSize(disk_, file_path_, file_size_, data_source_description.is_encrypted))
 {
 }
 
+BackupEntryFromAppendOnlyFile::~BackupEntryFromAppendOnlyFile() = default;
+
 std::unique_ptr<SeekableReadBuffer> BackupEntryFromAppendOnlyFile::getReadBuffer() const
 {
-    auto buf = BackupEntryFromImmutableFile::getReadBuffer();
-    return std::make_unique<LimitSeekableReadBuffer>(std::move(buf), 0, limit);
+    std::unique_ptr<SeekableReadBuffer> buf;
+    if (data_source_description.is_encrypted)
+        buf = disk->readEncryptedFile(file_path, settings);
+    else
+        buf = disk->readFile(file_path, settings);
+    return std::make_unique<LimitSeekableReadBuffer>(std::move(buf), 0, size);
 }
 
 }
diff --git a/src/Backups/BackupEntryFromAppendOnlyFile.h b/src/Backups/BackupEntryFromAppendOnlyFile.h
index 7c57e55923e..e6814c4c345 100644
--- a/src/Backups/BackupEntryFromAppendOnlyFile.h
+++ b/src/Backups/BackupEntryFromAppendOnlyFile.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <Backups/BackupEntryFromImmutableFile.h>
+#include <Backups/BackupEntryWithChecksumCalculation.h>
 
 
 namespace DB
@@ -8,24 +8,34 @@ namespace DB
 
 /// Represents a file prepared to be included in a backup, assuming that until this backup entry is destroyed
 /// the file can be appended with new data, but the bytes which are already in the file won't be changed.
-class BackupEntryFromAppendOnlyFile : public BackupEntryFromImmutableFile
+class BackupEntryFromAppendOnlyFile : public BackupEntryWithChecksumCalculation<IBackupEntry>
 {
 public:
-    /// The constructor is allowed to not set `file_size_` or `checksum_`, in that case it will be calculated from the data.
+    /// The constructor is allowed to not set `file_size_`, in that case it will be calculated from the data.
     BackupEntryFromAppendOnlyFile(
         const DiskPtr & disk_,
         const String & file_path_,
         const ReadSettings & settings_,
-        const std::optional<UInt64> & file_size_ = {},
-        const std::optional<UInt128> & checksum_ = {});
+        const std::optional<UInt64> & file_size_ = {});
+
+    ~BackupEntryFromAppendOnlyFile() override;
 
-    UInt64 getSize() const override { return limit; }
     std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override;
+    UInt64 getSize() const override { return size; }
 
-    bool isFromImmutableFile() const override { return false; }
+    DataSourceDescription getDataSourceDescription() const override { return data_source_description; }
+    bool isEncryptedByDisk() const override { return data_source_description.is_encrypted; }
+
+    bool isFromFile() const override { return true; }
+    DiskPtr getDisk() const override { return disk; }
+    String getFilePath() const override { return file_path; }
 
 private:
-    const UInt64 limit;
+    const DiskPtr disk;
+    const String file_path;
+    const DataSourceDescription data_source_description;
+    const ReadSettings settings;
+    const UInt64 size;
 };
 
 }
diff --git a/src/Backups/BackupEntryFromImmutableFile.cpp b/src/Backups/BackupEntryFromImmutableFile.cpp
index 7545134f638..ad0ce477600 100644
--- a/src/Backups/BackupEntryFromImmutableFile.cpp
+++ b/src/Backups/BackupEntryFromImmutableFile.cpp
@@ -1,8 +1,6 @@
 #include <Backups/BackupEntryFromImmutableFile.h>
 #include <Disks/IDisk.h>
-#include <Disks/IO/createReadBufferFromFileBase.h>
-#include <Poco/File.h>
-#include <Common/filesystemHelpers.h>
+#include <Disks/DiskEncrypted.h>
 
 
 namespace DB
@@ -18,26 +16,13 @@ BackupEntryFromImmutableFile::BackupEntryFromImmutableFile(
     , file_path(file_path_)
     , data_source_description(disk->getDataSourceDescription())
     , settings(settings_)
-    , file_size(data_source_description.is_encrypted ? std::optional<UInt64>{} : file_size_)
-    , checksum(data_source_description.is_encrypted ? std::optional<UInt128>{} : checksum_)
+    , file_size(file_size_)
+    , checksum(checksum_)
 {
 }
 
 BackupEntryFromImmutableFile::~BackupEntryFromImmutableFile() = default;
 
-UInt64 BackupEntryFromImmutableFile::getSize() const
-{
-    std::lock_guard lock{get_file_size_mutex};
-    if (!file_size)
-    {
-        if (data_source_description.is_encrypted)
-            file_size = disk->getEncryptedFileSize(file_path);
-        else
-            file_size = disk->getFileSize(file_path);
-    }
-    return *file_size;
-}
-
 std::unique_ptr<SeekableReadBuffer> BackupEntryFromImmutableFile::getReadBuffer() const
 {
     if (data_source_description.is_encrypted)
@@ -46,4 +31,43 @@ std::unique_ptr<SeekableReadBuffer> BackupEntryFromImmutableFile::getReadBuffer(
         return disk->readFile(file_path, settings);
 }
 
+UInt64 BackupEntryFromImmutableFile::getSize() const
+{
+    std::lock_guard lock{size_and_checksum_mutex};
+    if (!file_size_adjusted)
+    {
+        if (!file_size)
+            file_size = disk->getFileSize(file_path);
+        if (data_source_description.is_encrypted)
+            *file_size = DiskEncrypted::convertFileSizeToEncryptedFileSize(*file_size);
+        file_size_adjusted = true;
+    }
+    return *file_size;
+}
+
+UInt128 BackupEntryFromImmutableFile::getChecksum() const
+{
+    std::lock_guard lock{size_and_checksum_mutex};
+    if (!checksum_adjusted)
+    {
+        /// TODO: We should not just ignore `checksum` if `data_source_description.is_encrypted == true`, we should use it somehow.
+        if (!checksum || data_source_description.is_encrypted)
+            checksum = BackupEntryWithChecksumCalculation<IBackupEntry>::getChecksum();
+        checksum_adjusted = true;
+    }
+    return *checksum;
+}
+
+std::optional<UInt128> BackupEntryFromImmutableFile::getPartialChecksum(size_t prefix_length) const
+{
+    if (prefix_length == 0)
+        return 0;
+
+    if (prefix_length >= getSize())
+        return getChecksum();
+
+    /// For immutable files we don't use partial checksums.
+    return std::nullopt;
+}
+
 }
diff --git a/src/Backups/BackupEntryFromImmutableFile.h b/src/Backups/BackupEntryFromImmutableFile.h
index f2801b67df6..23f48e8335a 100644
--- a/src/Backups/BackupEntryFromImmutableFile.h
+++ b/src/Backups/BackupEntryFromImmutableFile.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <Backups/IBackupEntry.h>
+#include <Backups/BackupEntryWithChecksumCalculation.h>
 #include <IO/ReadSettings.h>
 #include <base/defines.h>
 #include <mutex>
@@ -11,7 +11,7 @@ class IDisk;
 using DiskPtr = std::shared_ptr<IDisk>;
 
 /// Represents a file prepared to be included in a backup, assuming that until this backup entry is destroyed the file won't be changed.
-class BackupEntryFromImmutableFile : public IBackupEntry
+class BackupEntryFromImmutableFile : public BackupEntryWithChecksumCalculation<IBackupEntry>
 {
 public:
     /// The constructor is allowed to not set `file_size_` or `checksum_`, in that case it will be calculated from the data.
@@ -24,13 +24,14 @@ public:
 
     ~BackupEntryFromImmutableFile() override;
 
-    UInt64 getSize() const override;
-    std::optional<UInt128> getChecksum() const override { return checksum; }
     std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override;
-    
-    bool isEncryptedByDisk() const override { return data_source_description.is_encrypted; }
+
+    UInt64 getSize() const override;
+    UInt128 getChecksum() const override;
+    std::optional<UInt128> getPartialChecksum(size_t prefix_length) const override;
 
     DataSourceDescription getDataSourceDescription() const override { return data_source_description; }
+    bool isEncryptedByDisk() const override { return data_source_description.is_encrypted; }
 
     bool isFromFile() const override { return true; }
     bool isFromImmutableFile() const override { return true; }
@@ -41,10 +42,12 @@ private:
     const DiskPtr disk;
     const String file_path;
     const DataSourceDescription data_source_description;
-    ReadSettings settings;
-    mutable std::optional<UInt64> file_size TSA_GUARDED_BY(get_file_size_mutex);
-    mutable std::mutex get_file_size_mutex;
-    const std::optional<UInt128> checksum;
+    const ReadSettings settings;
+    mutable std::optional<UInt64> file_size;
+    mutable std::optional<UInt64> checksum;
+    mutable bool file_size_adjusted = false;
+    mutable bool checksum_adjusted = false;
+    mutable std::mutex size_and_checksum_mutex;
 };
 
 }
diff --git a/src/Backups/BackupEntryFromMemory.cpp b/src/Backups/BackupEntryFromMemory.cpp
index f59eadc2d7f..82e033caca0 100644
--- a/src/Backups/BackupEntryFromMemory.cpp
+++ b/src/Backups/BackupEntryFromMemory.cpp
@@ -5,13 +5,12 @@
 namespace DB
 {
 
-BackupEntryFromMemory::BackupEntryFromMemory(const void * data_, size_t size_, const std::optional<UInt128> & checksum_)
-    : BackupEntryFromMemory(String{reinterpret_cast<const char *>(data_), size_}, checksum_)
+BackupEntryFromMemory::BackupEntryFromMemory(const void * data_, size_t size_)
+    : BackupEntryFromMemory(String{reinterpret_cast<const char *>(data_), size_})
 {
 }
 
-BackupEntryFromMemory::BackupEntryFromMemory(String data_, const std::optional<UInt128> & checksum_)
-    : data(std::move(data_)), checksum(checksum_)
+BackupEntryFromMemory::BackupEntryFromMemory(String data_) : data(std::move(data_))
 {
 }
 
diff --git a/src/Backups/BackupEntryFromMemory.h b/src/Backups/BackupEntryFromMemory.h
index 64f46d68580..19b72c16e53 100644
--- a/src/Backups/BackupEntryFromMemory.h
+++ b/src/Backups/BackupEntryFromMemory.h
@@ -1,32 +1,26 @@
 #pragma once
 
-#include <Backups/IBackupEntry.h>
-#include <IO/ReadBufferFromString.h>
+#include <Backups/BackupEntryWithChecksumCalculation.h>
 
 
 namespace DB
 {
 
 /// Represents small preloaded data to be included in a backup.
-class BackupEntryFromMemory : public IBackupEntry
+class BackupEntryFromMemory : public BackupEntryWithChecksumCalculation<IBackupEntry>
 {
 public:
     /// The constructor is allowed to not set `checksum_`, in that case it will be calculated from the data.
-    BackupEntryFromMemory(const void * data_, size_t size_, const std::optional<UInt128> & checksum_ = {});
-    explicit BackupEntryFromMemory(String data_, const std::optional<UInt128> & checksum_ = {});
+    BackupEntryFromMemory(const void * data_, size_t size_);
+    explicit BackupEntryFromMemory(String data_);
 
-    UInt64 getSize() const override { return data.size(); }
-    std::optional<UInt128> getChecksum() const override { return checksum; }
     std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override;
+    UInt64 getSize() const override { return data.size(); }
 
-    DataSourceDescription getDataSourceDescription() const override
-    {
-        return DataSourceDescription{DataSourceType::RAM, "", false, false};
-    }
+    DataSourceDescription getDataSourceDescription() const override { return DataSourceDescription{DataSourceType::RAM, "", false, false}; }
 
 private:
     const String data;
-    const std::optional<UInt128> checksum;
 };
 
 }
diff --git a/src/Backups/BackupEntryFromSmallFile.cpp b/src/Backups/BackupEntryFromSmallFile.cpp
index 6f7d2364031..3662522c243 100644
--- a/src/Backups/BackupEntryFromSmallFile.cpp
+++ b/src/Backups/BackupEntryFromSmallFile.cpp
@@ -29,21 +29,18 @@ namespace
 }
 
 
-BackupEntryFromSmallFile::BackupEntryFromSmallFile(const String & file_path_, const std::optional<UInt128> & checksum_)
+BackupEntryFromSmallFile::BackupEntryFromSmallFile(const String & file_path_)
     : file_path(file_path_)
     , data_source_description(DiskLocal::getLocalDataSourceDescription(file_path_))
     , data(readFile(file_path_))
-    , checksum(checksum_)
 {
 }
-    
-BackupEntryFromSmallFile::BackupEntryFromSmallFile(
-    const DiskPtr & disk_, const String & file_path_, const std::optional<UInt128> & checksum_)
+
+BackupEntryFromSmallFile::BackupEntryFromSmallFile(const DiskPtr & disk_, const String & file_path_)
     : disk(disk_)
     , file_path(file_path_)
     , data_source_description(disk_->getDataSourceDescription())
     , data(readFile(disk_, file_path, data_source_description.is_encrypted))
-    , checksum(data_source_description.is_encrypted ? std::optional<UInt128>{} : checksum_)
 {
 }
 
diff --git a/src/Backups/BackupEntryFromSmallFile.h b/src/Backups/BackupEntryFromSmallFile.h
index 2f7f3764571..6846ca71a55 100644
--- a/src/Backups/BackupEntryFromSmallFile.h
+++ b/src/Backups/BackupEntryFromSmallFile.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <Backups/IBackupEntry.h>
+#include <Backups/BackupEntryWithChecksumCalculation.h>
 
 
 namespace DB
@@ -10,37 +10,27 @@ using DiskPtr = std::shared_ptr<IDisk>;
 
 /// Represents a file prepared to be included in a backup,
 /// assuming that the file is small and can be easily loaded into memory.
-class BackupEntryFromSmallFile : public IBackupEntry
+class BackupEntryFromSmallFile : public BackupEntryWithChecksumCalculation<IBackupEntry>
 {
 public:
-    /// The constructor is allowed to not set `checksum_`, in that case it will be calculated from the data.
-    explicit BackupEntryFromSmallFile(
-        const String & file_path_,
-        const std::optional<UInt128> & checksum_ = {});
+    explicit BackupEntryFromSmallFile(const String & file_path_);
+    BackupEntryFromSmallFile(const DiskPtr & disk_, const String & file_path_);
 
-    BackupEntryFromSmallFile(
-        const DiskPtr & disk_,
-        const String & file_path_,
-        const std::optional<UInt128> & checksum_ = {});
-
-    UInt64 getSize() const override { return data.size(); }
-    std::optional<UInt128> getChecksum() const override { return checksum; }
     std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override;
+    UInt64 getSize() const override { return data.size(); }
 
+    DataSourceDescription getDataSourceDescription() const override { return data_source_description; }
     bool isEncryptedByDisk() const override { return data_source_description.is_encrypted; }
 
     bool isFromFile() const override { return true; }
     DiskPtr getDisk() const override { return disk; }
     String getFilePath() const override { return file_path; }
 
-    DataSourceDescription getDataSourceDescription() const override { return data_source_description; }
-
 private:
     const DiskPtr disk;
     const String file_path;
     const DataSourceDescription data_source_description;
     const String data;
-    const std::optional<UInt128> checksum;
 };
 
 }
diff --git a/src/Backups/BackupEntryWithChecksumCalculation.cpp b/src/Backups/BackupEntryWithChecksumCalculation.cpp
new file mode 100644
index 00000000000..bc97a907cff
--- /dev/null
+++ b/src/Backups/BackupEntryWithChecksumCalculation.cpp
@@ -0,0 +1,48 @@
+#include <Backups/BackupEntryWithChecksumCalculation.h>
+#include <IO/HashingReadBuffer.h>
+
+
+namespace DB
+{
+
+template <typename Base>
+UInt128 BackupEntryWithChecksumCalculation<Base>::getChecksum() const
+{
+    std::lock_guard lock{checksum_calculation_mutex};
+    if (!calculated_checksum)
+    {
+        auto read_buffer = this->getReadBuffer();
+        HashingReadBuffer hashing_read_buffer(*read_buffer);
+        hashing_read_buffer.ignoreAll();
+        calculated_checksum = hashing_read_buffer.getHash();
+    }
+    return *calculated_checksum;
+}
+
+template <typename Base>
+std::optional<UInt128> BackupEntryWithChecksumCalculation<Base>::getPartialChecksum(size_t prefix_length) const
+{
+    if (prefix_length == 0)
+        return 0;
+
+    if (prefix_length >= this->getSize())
+        return this->getChecksum();
+
+    auto read_buffer = this->getReadBuffer();
+    HashingReadBuffer hashing_read_buffer(*read_buffer);
+    hashing_read_buffer.ignore(prefix_length);
+    auto partial_checksum = hashing_read_buffer.getHash();
+
+    std::lock_guard lock{checksum_calculation_mutex};
+    if (!calculated_checksum)
+    {
+        hashing_read_buffer.ignoreAll();
+        calculated_checksum = hashing_read_buffer.getHash();
+    }
+
+    return partial_checksum;
+}
+
+template class BackupEntryWithChecksumCalculation<IBackupEntry>;
+
+}
diff --git a/src/Backups/BackupEntryWithChecksumCalculation.h b/src/Backups/BackupEntryWithChecksumCalculation.h
new file mode 100644
index 00000000000..32701ab9952
--- /dev/null
+++ b/src/Backups/BackupEntryWithChecksumCalculation.h
@@ -0,0 +1,22 @@
+#pragma once
+
+#include <Backups/IBackupEntry.h>
+
+
+namespace DB
+{
+
+/// Calculates the checksum and the partial checksum for a backup entry based on ReadBuffer returned by getReadBuffer().
+template <typename Base>
+class BackupEntryWithChecksumCalculation : public Base
+{
+public:
+    UInt128 getChecksum() const override;
+    std::optional<UInt128> getPartialChecksum(size_t prefix_length) const override;
+
+private:
+    mutable std::optional<UInt128> calculated_checksum;
+    mutable std::mutex checksum_calculation_mutex;
+};
+
+}
diff --git a/src/Backups/BackupEntryWrappedWith.h b/src/Backups/BackupEntryWrappedWith.h
index 933fec39634..01b6163039b 100644
--- a/src/Backups/BackupEntryWrappedWith.h
+++ b/src/Backups/BackupEntryWrappedWith.h
@@ -15,15 +15,16 @@ public:
     BackupEntryWrappedWith(BackupEntryPtr entry_, T && custom_value_) : entry(entry_), custom_value(std::move(custom_value_)) { }
     ~BackupEntryWrappedWith() override = default;
 
-    UInt64 getSize() const override { return entry->getSize(); }
-    std::optional<UInt128> getChecksum() const override { return entry->getChecksum(); }
     std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override { return entry->getReadBuffer(); }
+    UInt64 getSize() const override { return entry->getSize(); }
+    UInt128 getChecksum() const override { return entry->getChecksum(); }
+    std::optional<UInt128> getPartialChecksum(size_t prefix_length) const override { return entry->getPartialChecksum(prefix_length); }
+    DataSourceDescription getDataSourceDescription() const override { return entry->getDataSourceDescription(); }
     bool isEncryptedByDisk() const override { return entry->isEncryptedByDisk(); }
     bool isFromFile() const override { return entry->isFromFile(); }
     bool isFromImmutableFile() const override { return entry->isFromImmutableFile(); }
     String getFilePath() const override { return entry->getFilePath(); }
     DiskPtr getDisk() const override { return entry->getDisk(); }
-    DataSourceDescription getDataSourceDescription() const override { return entry->getDataSourceDescription(); }
 
 private:
     BackupEntryPtr entry;
diff --git a/src/Backups/BackupFileInfo.cpp b/src/Backups/BackupFileInfo.cpp
index 91ddc52ae44..42546d1b1b8 100644
--- a/src/Backups/BackupFileInfo.cpp
+++ b/src/Backups/BackupFileInfo.cpp
@@ -7,7 +7,7 @@
 #include <Common/scope_guard_safe.h>
 #include <Common/setThreadName.h>
 #include <Common/ThreadPool.h>
-#include <IO/HashingReadBuffer.h>
+#include <base/hex.h>
 
 
 namespace DB
@@ -49,44 +49,19 @@ namespace
     struct ChecksumsForNewEntry
     {
         UInt128 full_checksum;
-        UInt128 prefix_checksum;
+        std::optional<UInt128> prefix_checksum;
     };
 
     /// Calculate checksum for backup entry if it's empty.
     /// Also able to calculate additional checksum of some prefix.
     ChecksumsForNewEntry calculateNewEntryChecksumsIfNeeded(const BackupEntryPtr & entry, size_t prefix_size)
     {
+        ChecksumsForNewEntry res;
+        /// The partial checksum should be calculated before the full checksum to enable optimization in BackupEntryWithChecksumCalculation.
         if (prefix_size > 0)
-        {
-            auto read_buffer = entry->getReadBuffer();
-            HashingReadBuffer hashing_read_buffer(*read_buffer);
-            hashing_read_buffer.ignore(prefix_size);
-            auto prefix_checksum = hashing_read_buffer.getHash();
-            if (entry->getChecksum() == std::nullopt)
-            {
-                hashing_read_buffer.ignoreAll();
-                auto full_checksum = hashing_read_buffer.getHash();
-                return ChecksumsForNewEntry{full_checksum, prefix_checksum};
-            }
-            else
-            {
-                return ChecksumsForNewEntry{*(entry->getChecksum()), prefix_checksum};
-            }
-        }
-        else
-        {
-            if (entry->getChecksum() == std::nullopt)
-            {
-                auto read_buffer = entry->getReadBuffer();
-                HashingReadBuffer hashing_read_buffer(*read_buffer);
-                hashing_read_buffer.ignoreAll();
-                return ChecksumsForNewEntry{hashing_read_buffer.getHash(), 0};
-            }
-            else
-            {
-                return ChecksumsForNewEntry{*(entry->getChecksum()), 0};
-            }
-        }
+            res.prefix_checksum = entry->getPartialChecksum(prefix_size);
+        res.full_checksum = entry->getChecksum();
+        return res;
     }
 
     /// We store entries' file names in the backup without leading slashes.
diff --git a/src/Backups/BackupIO_File.cpp b/src/Backups/BackupIO_File.cpp
index 69265799793..4268f653e30 100644
--- a/src/Backups/BackupIO_File.cpp
+++ b/src/Backups/BackupIO_File.cpp
@@ -1,10 +1,7 @@
 #include <Backups/BackupIO_File.h>
 #include <Disks/DiskLocal.h>
-//#include <Disks/IDisk.h>
 #include <Disks/IO/createReadBufferFromFileBase.h>
 #include <IO/WriteBufferFromFile.h>
-//#include <IO/copyData.h>
-//#include <Common/filesystemHelpers.h>
 #include <Common/logger_useful.h>
 
 
diff --git a/src/Backups/IBackupEntriesLazyBatch.cpp b/src/Backups/IBackupEntriesLazyBatch.cpp
index cd4d470967b..8913d60f2eb 100644
--- a/src/Backups/IBackupEntriesLazyBatch.cpp
+++ b/src/Backups/IBackupEntriesLazyBatch.cpp
@@ -17,11 +17,12 @@ class IBackupEntriesLazyBatch::BackupEntryFromBatch : public IBackupEntry
 public:
     BackupEntryFromBatch(const std::shared_ptr<IBackupEntriesLazyBatch> & batch_, size_t index_) : batch(batch_), index(index_) { }
 
-    UInt64 getSize() const override { return getInternalBackupEntry()->getSize(); }
-    std::optional<UInt128> getChecksum() const override { return getInternalBackupEntry()->getChecksum(); }
     std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override { return getInternalBackupEntry()->getReadBuffer(); }
-    bool isEncryptedByDisk() const override { return getInternalBackupEntry()->isEncryptedByDisk(); }
+    UInt64 getSize() const override { return getInternalBackupEntry()->getSize(); }
+    UInt128 getChecksum() const override { return getInternalBackupEntry()->getChecksum(); }
+    std::optional<UInt128> getPartialChecksum(size_t prefix_length) const override { return getInternalBackupEntry()->getPartialChecksum(prefix_length); }
     DataSourceDescription getDataSourceDescription() const override { return getInternalBackupEntry()->getDataSourceDescription(); }
+    bool isEncryptedByDisk() const override { return getInternalBackupEntry()->isEncryptedByDisk(); }
     bool isFromFile() const override { return getInternalBackupEntry()->isFromFile(); }
     bool isFromImmutableFile() const override { return getInternalBackupEntry()->isFromImmutableFile(); }
     String getFilePath() const override { return getInternalBackupEntry()->getFilePath(); }
diff --git a/src/Backups/IBackupEntry.h b/src/Backups/IBackupEntry.h
index 7a93d4035df..249c290cbe7 100644
--- a/src/Backups/IBackupEntry.h
+++ b/src/Backups/IBackupEntry.h
@@ -20,9 +20,12 @@ public:
     /// Returns the size of the data.
     virtual UInt64 getSize() const = 0;
 
-    /// Returns the checksum of the data if it's precalculated.
-    /// Can return nullopt which means the checksum should be calculated from the read buffer.
-    virtual std::optional<UInt128> getChecksum() const { return {}; }
+    /// Returns the checksum of the data.
+    virtual UInt128 getChecksum() const = 0;
+
+    /// Returns a partial checksum, i.e. the checksum calculated for a prefix part of the data.
+    /// Can return nullopt if the partial checksum is too difficult to calculate.
+    virtual std::optional<UInt128> getPartialChecksum(size_t /* prefix_length */) const { return {}; }
 
     /// Returns a read buffer for reading the data.
     virtual std::unique_ptr<SeekableReadBuffer> getReadBuffer() const = 0;
diff --git a/src/Disks/DiskEncrypted.cpp b/src/Disks/DiskEncrypted.cpp
index db18e9652e7..15f314dcfc9 100644
--- a/src/Disks/DiskEncrypted.cpp
+++ b/src/Disks/DiskEncrypted.cpp
@@ -348,6 +348,13 @@ size_t DiskEncrypted::getFileSize(const String & path) const
     return size > FileEncryption::Header::kSize ? (size - FileEncryption::Header::kSize) : 0;
 }
 
+size_t DiskEncrypted::convertFileSizeToEncryptedFileSize(size_t file_size)
+{
+    if (file_size)
+        return file_size + FileEncryption::Header::kSize;
+    return 0;
+}
+
 void DiskEncrypted::truncateFile(const String & path, size_t size)
 {
     auto wrapped_path = wrappedPath(path);
diff --git a/src/Disks/DiskEncrypted.h b/src/Disks/DiskEncrypted.h
index db53accc062..295e70a2c66 100644
--- a/src/Disks/DiskEncrypted.h
+++ b/src/Disks/DiskEncrypted.h
@@ -223,6 +223,8 @@ public:
         return delegate->getFileSize(wrapped_path);
     }
 
+    static size_t convertFileSizeToEncryptedFileSize(size_t file_size);
+
     void setLastModified(const String & path, const Poco::Timestamp & timestamp) override
     {
         auto wrapped_path = wrappedPath(path);
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 1dde057569e..fba6e6cbafc 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -9278,10 +9278,9 @@ void StorageReplicatedMergeTree::backupData(
                 auto & hash = part_names_with_hashes_calculating[part_name];
                 if (relative_path.ends_with(".bin"))
                 {
-                    auto checksum = backup_entry->getChecksum();
                     hash.update(relative_path);
                     hash.update(backup_entry->getSize());
-                    hash.update(*checksum);
+                    hash.update(backup_entry->getChecksum());
                 }
                 continue;
             }

From c48c20fac83a0f7014a97e74fa81ce8ab49bdd1d Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Mon, 1 May 2023 18:43:20 +0200
Subject: [PATCH 563/689] Use combined checksums for encrypted immutable files.

---
 src/Backups/BackupEntryFromAppendOnlyFile.cpp | 25 +++++++------------
 src/Backups/BackupEntryFromAppendOnlyFile.h   |  2 +-
 src/Backups/BackupEntryFromImmutableFile.cpp  | 22 ++++++++++++----
 src/Backups/BackupIO_Disk.cpp                 |  2 +-
 src/Backups/BackupIO_File.cpp                 |  2 +-
 src/Backups/BackupIO_S3.cpp                   |  2 +-
 src/Disks/DiskEncrypted.cpp                   | 16 +++++++++---
 src/Disks/DiskEncrypted.h                     |  4 +++
 src/Disks/IDisk.cpp                           | 10 ++++++++
 src/Disks/IDisk.h                             | 10 +++++---
 .../test_backup_restore_s3/test.py            | 12 ++++-----
 11 files changed, 70 insertions(+), 37 deletions(-)

diff --git a/src/Backups/BackupEntryFromAppendOnlyFile.cpp b/src/Backups/BackupEntryFromAppendOnlyFile.cpp
index e3b79695a0c..2c28abc2e2f 100644
--- a/src/Backups/BackupEntryFromAppendOnlyFile.cpp
+++ b/src/Backups/BackupEntryFromAppendOnlyFile.cpp
@@ -8,22 +8,15 @@ namespace DB
 
 namespace
 {
-    UInt64 calculateSize(const DiskPtr & disk, const String & file_path, const std::optional<UInt64> & file_size, bool disk_is_encrypted)
+    /// For append-only files we must calculate its size on the construction of a backup entry.
+    UInt64 calculateSize(const DiskPtr & disk, const String & file_path, bool is_encrypted, std::optional<UInt64> unencrypted_file_size)
     {
-        if (file_size)
-        {
-            if (disk_is_encrypted)
-                return DiskEncrypted::convertFileSizeToEncryptedFileSize(*file_size);
-            else
-                return *file_size;
-        }
+        if (!unencrypted_file_size)
+            return is_encrypted ? disk->getEncryptedFileSize(file_path) : disk->getFileSize(file_path);
+        else if (is_encrypted)
+            return disk->getEncryptedFileSize(*unencrypted_file_size);
         else
-        {
-            if (disk_is_encrypted)
-                return disk->getEncryptedFileSize(file_path);
-            else
-                return disk->getFileSize(file_path);
-        }
+            return *unencrypted_file_size;
     }
 }
 
@@ -32,8 +25,8 @@ BackupEntryFromAppendOnlyFile::BackupEntryFromAppendOnlyFile(
     : disk(disk_)
     , file_path(file_path_)
     , data_source_description(disk->getDataSourceDescription())
-    , settings(settings_)
-    , size(calculateSize(disk_, file_path_, file_size_, data_source_description.is_encrypted))
+    , size(calculateSize(disk_, file_path_, data_source_description.is_encrypted, file_size_))
+    , settings(settings_.adjustBufferSize(size))
 {
 }
 
diff --git a/src/Backups/BackupEntryFromAppendOnlyFile.h b/src/Backups/BackupEntryFromAppendOnlyFile.h
index e6814c4c345..fac8c319d8c 100644
--- a/src/Backups/BackupEntryFromAppendOnlyFile.h
+++ b/src/Backups/BackupEntryFromAppendOnlyFile.h
@@ -34,8 +34,8 @@ private:
     const DiskPtr disk;
     const String file_path;
     const DataSourceDescription data_source_description;
-    const ReadSettings settings;
     const UInt64 size;
+    const ReadSettings settings;
 };
 
 }
diff --git a/src/Backups/BackupEntryFromImmutableFile.cpp b/src/Backups/BackupEntryFromImmutableFile.cpp
index ad0ce477600..f377e54d9b0 100644
--- a/src/Backups/BackupEntryFromImmutableFile.cpp
+++ b/src/Backups/BackupEntryFromImmutableFile.cpp
@@ -6,6 +6,17 @@
 namespace DB
 {
 
+namespace
+{
+    /// We mix the checksum calculated for non-encrypted data with IV generated to encrypt the file
+    /// to generate kind of a checksum for encrypted data. Of course it differs from the CityHash properly calculated for encrypted data.
+    UInt128 combineChecksums(UInt128 checksum1, UInt128 checksum2)
+    {
+        chassert(std::size(checksum2.items) == 2);
+        return CityHash_v1_0_2::CityHash128WithSeed(reinterpret_cast<const char *>(&checksum1), sizeof(checksum1), {checksum2.items[0], checksum2.items[1]});
+    }
+}
+
 BackupEntryFromImmutableFile::BackupEntryFromImmutableFile(
     const DiskPtr & disk_,
     const String & file_path_,
@@ -37,9 +48,9 @@ UInt64 BackupEntryFromImmutableFile::getSize() const
     if (!file_size_adjusted)
     {
         if (!file_size)
-            file_size = disk->getFileSize(file_path);
-        if (data_source_description.is_encrypted)
-            *file_size = DiskEncrypted::convertFileSizeToEncryptedFileSize(*file_size);
+            file_size = data_source_description.is_encrypted ? disk->getEncryptedFileSize(file_path) : disk->getFileSize(file_path);
+        else if (data_source_description.is_encrypted)
+            file_size = disk->getEncryptedFileSize(*file_size);
         file_size_adjusted = true;
     }
     return *file_size;
@@ -50,9 +61,10 @@ UInt128 BackupEntryFromImmutableFile::getChecksum() const
     std::lock_guard lock{size_and_checksum_mutex};
     if (!checksum_adjusted)
     {
-        /// TODO: We should not just ignore `checksum` if `data_source_description.is_encrypted == true`, we should use it somehow.
-        if (!checksum || data_source_description.is_encrypted)
+        if (!checksum)
             checksum = BackupEntryWithChecksumCalculation<IBackupEntry>::getChecksum();
+        else if (data_source_description.is_encrypted)
+            checksum = combineChecksums(*checksum, disk->getEncryptedFileIV(file_path));
         checksum_adjusted = true;
     }
     return *checksum;
diff --git a/src/Backups/BackupIO_Disk.cpp b/src/Backups/BackupIO_Disk.cpp
index f64e929131c..1e1af63cdc2 100644
--- a/src/Backups/BackupIO_Disk.cpp
+++ b/src/Backups/BackupIO_Disk.cpp
@@ -74,7 +74,7 @@ UInt64 BackupWriterDisk::getFileSize(const String & file_name)
 
 std::unique_ptr<ReadBuffer> BackupWriterDisk::readFile(const String & file_name, size_t expected_file_size)
 {
-    return disk->readFile(file_name, read_settings, {}, expected_file_size);
+    return disk->readFile(root_path / file_name, read_settings.adjustBufferSize(expected_file_size));
 }
 
 std::unique_ptr<WriteBuffer> BackupWriterDisk::writeFile(const String & file_name)
diff --git a/src/Backups/BackupIO_File.cpp b/src/Backups/BackupIO_File.cpp
index 4268f653e30..649637e567d 100644
--- a/src/Backups/BackupIO_File.cpp
+++ b/src/Backups/BackupIO_File.cpp
@@ -87,7 +87,7 @@ UInt64 BackupWriterFile::getFileSize(const String & file_name)
 
 std::unique_ptr<ReadBuffer> BackupWriterFile::readFile(const String & file_name, size_t expected_file_size)
 {
-    return createReadBufferFromFileBase(root_path / file_name, read_settings, {}, expected_file_size);
+    return createReadBufferFromFileBase(root_path / file_name, read_settings.adjustBufferSize(expected_file_size));
 }
 
 std::unique_ptr<WriteBuffer> BackupWriterFile::writeFile(const String & file_name)
diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp
index 8d9d34bf9b5..09d2bdbcfaf 100644
--- a/src/Backups/BackupIO_S3.cpp
+++ b/src/Backups/BackupIO_S3.cpp
@@ -205,7 +205,7 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src
                 /* src_key= */ blob_path[1],
                 start_pos,
                 length,
-                s3_uri.endpoint,
+                s3_uri.bucket,
                 fs::path(s3_uri.key) / path_in_backup,
                 request_settings,
                 {},
diff --git a/src/Disks/DiskEncrypted.cpp b/src/Disks/DiskEncrypted.cpp
index 15f314dcfc9..72f668db00b 100644
--- a/src/Disks/DiskEncrypted.cpp
+++ b/src/Disks/DiskEncrypted.cpp
@@ -348,10 +348,20 @@ size_t DiskEncrypted::getFileSize(const String & path) const
     return size > FileEncryption::Header::kSize ? (size - FileEncryption::Header::kSize) : 0;
 }
 
-size_t DiskEncrypted::convertFileSizeToEncryptedFileSize(size_t file_size)
+UInt128 DiskEncrypted::getEncryptedFileIV(const String & path) const
 {
-    if (file_size)
-        return file_size + FileEncryption::Header::kSize;
+    auto wrapped_path = wrappedPath(path);
+    auto read_buffer = delegate->readFile(wrapped_path, ReadSettings().adjustBufferSize(FileEncryption::Header::kSize));
+    if (read_buffer->eof())
+        return 0;
+    auto header = readHeader(*read_buffer);
+    return header.init_vector.get();
+}
+
+size_t DiskEncrypted::getEncryptedFileSize(size_t unencrypted_size) const
+{
+    if (unencrypted_size)
+        return unencrypted_size + FileEncryption::Header::kSize;
     return 0;
 }
 
diff --git a/src/Disks/DiskEncrypted.h b/src/Disks/DiskEncrypted.h
index 295e70a2c66..c494dd6a216 100644
--- a/src/Disks/DiskEncrypted.h
+++ b/src/Disks/DiskEncrypted.h
@@ -223,6 +223,10 @@ public:
         return delegate->getFileSize(wrapped_path);
     }
 
+    size_t getEncryptedFileSize(size_t unencrypted_size) const override;
+
+    UInt128 getEncryptedFileIV(const String & path) const override;
+
     static size_t convertFileSizeToEncryptedFileSize(size_t file_size);
 
     void setLastModified(const String & path, const Poco::Timestamp & timestamp) override
diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp
index 3c8ca454463..9a5ae997b46 100644
--- a/src/Disks/IDisk.cpp
+++ b/src/Disks/IDisk.cpp
@@ -67,6 +67,16 @@ size_t IDisk::getEncryptedFileSize(const String &) const
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "File encryption is not implemented for disk of type {}", getDataSourceDescription().type);
 }
 
+size_t IDisk::getEncryptedFileSize(size_t) const
+{
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "File encryption is not implemented for disk of type {}", getDataSourceDescription().type);
+}
+
+UInt128 IDisk::getEncryptedFileIV(const String &) const
+{
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "File encryption is not implemented for disk of type {}", getDataSourceDescription().type);
+}
+
 
 using ResultsCollector = std::vector<std::future<void>>;
 
diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h
index 770eec46ce6..47b4ed80ebf 100644
--- a/src/Disks/IDisk.h
+++ b/src/Disks/IDisk.h
@@ -252,22 +252,26 @@ public:
     /// this method allows to specify a callback for that.
     virtual void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) = 0;
 
-    /// Reads a file from an encrypted disk without decrypting it.
+    /// Reads a file from an encrypted disk without decrypting it (only for encrypted disks).
     virtual std::unique_ptr<ReadBufferFromFileBase> readEncryptedFile(
         const String & path,
         const ReadSettings & settings = ReadSettings{},
         std::optional<size_t> read_hint = {},
         std::optional<size_t> file_size = {}) const;
 
-    /// Writes an already encrypted file to an encrypted disk.
+    /// Writes an already encrypted file to the disk (only for encrypted disks).
     virtual std::unique_ptr<WriteBufferFromFileBase> writeEncryptedFile(
         const String & path,
         size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
         WriteMode mode = WriteMode::Rewrite,
         const WriteSettings & settings = {}) const;
 
-    /// Returns the size of encrypted file on an encrypted disk.
+    /// Returns the size of an encrypted file (only for encrypted disks).
     virtual size_t getEncryptedFileSize(const String & path) const;
+    virtual size_t getEncryptedFileSize(size_t unencrypted_size) const;
+
+    /// Returns IV of an encrypted file (only for encrypted disks).
+    virtual UInt128 getEncryptedFileIV(const String & path) const;
 
     virtual const String & getCacheName() const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There is no cache"); }
 
diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py
index b5ac34f0b46..2f60575b634 100644
--- a/tests/integration/test_backup_restore_s3/test.py
+++ b/tests/integration/test_backup_restore_s3/test.py
@@ -141,8 +141,8 @@ def test_backup_to_s3_native_copy():
         f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', 'minio123')"
     )
     check_backup_and_restore(storage_policy, backup_destination)
-    assert node.contains_in_log("copyS3FileFromDisk.*using native copy")
-    assert node.contains_in_log("copyS3FileToDisk.*using native copy")
+    assert node.contains_in_log("BackupWriterS3.*using native copy")
+    assert node.contains_in_log("BackupReaderS3.*using native copy")
     assert node.contains_in_log(
         f"copyS3File: Single operation copy has completed. Bucket: root, Key: data/backups/{backup_name}"
     )
@@ -155,8 +155,8 @@ def test_backup_to_s3_native_copy_other_bucket():
         f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', 'minio123')"
     )
     check_backup_and_restore(storage_policy, backup_destination)
-    assert node.contains_in_log("copyS3FileFromDisk.*using native copy")
-    assert node.contains_in_log("copyS3FileToDisk.*using native copy")
+    assert node.contains_in_log("BackupWriterS3.*using native copy")
+    assert node.contains_in_log("BackupReaderS3.*using native copy")
     assert node.contains_in_log(
         f"copyS3File: Single operation copy has completed. Bucket: root, Key: data/backups/{backup_name}"
     )
@@ -167,8 +167,8 @@ def test_backup_to_s3_native_copy_multipart():
     backup_name = new_backup_name()
     backup_destination = f"S3('http://minio1:9001/root/data/backups/multipart/{backup_name}', 'minio', 'minio123')"
     check_backup_and_restore(storage_policy, backup_destination, size=1000000)
-    assert node.contains_in_log("copyS3FileFromDisk.*using native copy")
-    assert node.contains_in_log("copyS3FileToDisk.*using native copy")
+    assert node.contains_in_log("BackupWriterS3.*using native copy")
+    assert node.contains_in_log("BackupReaderS3.*using native copy")
     assert node.contains_in_log(
         f"copyS3File: Multipart upload has completed. Bucket: root, Key: data/backups/multipart/{backup_name}/"
     )

From d0fd81fa27514ff18001a25bd2a3e2b631ca9e0d Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Tue, 2 May 2023 01:55:56 +0200
Subject: [PATCH 564/689] Add tests.

---
 .../configs/allow_backup_path.xml             |   5 +
 tests/integration/test_encrypted_disk/test.py | 126 +++++++++++++++++-
 2 files changed, 130 insertions(+), 1 deletion(-)
 create mode 100644 tests/integration/test_encrypted_disk/configs/allow_backup_path.xml

diff --git a/tests/integration/test_encrypted_disk/configs/allow_backup_path.xml b/tests/integration/test_encrypted_disk/configs/allow_backup_path.xml
new file mode 100644
index 00000000000..eade3bfb744
--- /dev/null
+++ b/tests/integration/test_encrypted_disk/configs/allow_backup_path.xml
@@ -0,0 +1,5 @@
+<clickhouse>
+    <backups>
+        <allowed_path>/backups/</allowed_path>
+    </backups>
+</clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_encrypted_disk/test.py b/tests/integration/test_encrypted_disk/test.py
index 8187f2ff6a8..90da5849c7f 100644
--- a/tests/integration/test_encrypted_disk/test.py
+++ b/tests/integration/test_encrypted_disk/test.py
@@ -1,6 +1,7 @@
 import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.client import QueryRuntimeException
+import os.path
 from helpers.test_tools import assert_eq_with_retry
 
 
@@ -9,8 +10,9 @@ FIRST_PART_NAME = "all_1_1_0"
 cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance(
     "node",
-    main_configs=["configs/storage.xml"],
+    main_configs=["configs/storage.xml", "configs/allow_backup_path.xml"],
     tmpfs=["/disk:size=100M"],
+    external_dirs=["/backups/"],
     with_minio=True,
     stay_alive=True,
 )
@@ -33,6 +35,15 @@ def cleanup_after_test():
         node.query("DROP TABLE IF EXISTS encrypted_test SYNC")
 
 
+backup_id_counter = 0
+
+
+def new_backup_name():
+    global backup_id_counter
+    backup_id_counter += 1
+    return f"backup{backup_id_counter}"
+
+
 @pytest.mark.parametrize(
     "policy",
     ["encrypted_policy", "encrypted_policy_key192b", "local_policy", "s3_policy"],
@@ -295,3 +306,116 @@ def test_restart():
         assert node.query(select_query) == "(0,'data'),(1,'data')"
 
         node.query("DROP TABLE encrypted_test SYNC;")
+
+
+@pytest.mark.parametrize(
+    "storage_policy,backup_type,storage_policy2",
+    [
+        ("encrypted_policy", "S3", "encrypted_policy"),
+        ("encrypted_policy", "S3", "s3_encrypted_default_path"),
+        ("s3_encrypted_default_path", "S3", "s3_encrypted_default_path"),
+        ("s3_encrypted_default_path", "S3", "encrypted_policy"),
+        ("s3_encrypted_default_path", "File", "encrypted_policy"),
+        ("local_policy", "File", "encrypted_policy"),
+    ],
+)
+def test_backup_restore(storage_policy, backup_type, storage_policy2):
+    node.query(
+        f"""
+        CREATE TABLE encrypted_test (
+            id Int64,
+            data String
+        ) ENGINE=MergeTree()
+        ORDER BY id
+        SETTINGS storage_policy='{storage_policy}'
+        """
+    )
+
+    node.query("INSERT INTO encrypted_test VALUES (0,'data'),(1,'data')")
+    select_query = "SELECT * FROM encrypted_test ORDER BY id FORMAT Values"
+    assert node.query(select_query) == "(0,'data'),(1,'data')"
+
+    backup_name = new_backup_name()
+    if backup_type == "S3":
+        backup_destination = (
+            f"S3('http://minio1:9001/root/backups/{backup_name}', 'minio', 'minio123')"
+        )
+    elif backup_type == "File":
+        backup_destination = f"File('/backups/{backup_name}/')"
+
+    node.query(f"BACKUP TABLE encrypted_test TO {backup_destination}")
+
+    if backup_type == "File" and storage_policy.find("encrypted") != -1:
+        root_path = os.path.join(node.cluster.instances_dir, "backups", backup_name)
+        with open(
+            f"{root_path}/data/default/encrypted_test/all_1_1_0/data.bin", "rb"
+        ) as file:
+            assert file.read().startswith(b"ENC")
+        with open(f"{root_path}/metadata/default/encrypted_test.sql") as file:
+            assert file.read().startswith("CREATE TABLE default.encrypted_test")
+        with open(f"{root_path}/.backup") as file:
+            assert file.read().find("<encrypted_by_disk>true</encrypted_by_disk>") != -1
+
+    node.query(f"DROP TABLE encrypted_test SYNC")
+
+    if storage_policy != storage_policy2:
+        node.query(
+            f"""
+            CREATE TABLE encrypted_test (
+                id Int64,
+                data String
+            ) ENGINE=MergeTree()
+            ORDER BY id
+            SETTINGS storage_policy='{storage_policy2}'
+            """
+        )
+
+    node.query(
+        f"RESTORE TABLE encrypted_test FROM {backup_destination} SETTINGS allow_different_table_def={int(storage_policy != storage_policy2)}"
+    )
+
+    assert node.query(select_query) == "(0,'data'),(1,'data')"
+
+
+def test_cannot_restore_encrypted_files_to_unencrypted_disk():
+    node.query(
+        """
+        CREATE TABLE encrypted_test (
+            id Int64,
+            data String
+        ) ENGINE=MergeTree()
+        ORDER BY id
+        SETTINGS storage_policy='encrypted_policy'
+        """
+    )
+
+    node.query("INSERT INTO encrypted_test VALUES (0,'data'),(1,'data')")
+    assert (
+        node.query("SELECT * FROM encrypted_test ORDER BY id FORMAT Values")
+        == "(0,'data'),(1,'data')"
+    )
+
+    backup_name = new_backup_name()
+    backup_destination = (
+        f"S3('http://minio1:9001/root/backups/{backup_name}', 'minio', 'minio123')"
+    )
+    node.query(f"BACKUP TABLE encrypted_test TO {backup_destination}")
+
+    node.query(f"DROP TABLE encrypted_test SYNC")
+
+    node.query(
+        f"""
+        CREATE TABLE encrypted_test (
+            id Int64,
+            data String
+        ) ENGINE=MergeTree()
+        ORDER BY id
+        SETTINGS storage_policy='local_policy'
+        """
+    )
+
+    expected_error = "can be restored only to an encrypted disk"
+    assert expected_error in node.query_and_get_error(
+        f"RESTORE TABLE encrypted_test FROM {backup_destination} SETTINGS allow_different_table_def=1"
+    )
+>>>>>>> 9c08fb30995 (Add tests.)

From 7cea26423010a09ef6777c17ac210cd11cfb14c7 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Wed, 3 May 2023 12:33:35 +0200
Subject: [PATCH 565/689] Fix whitespaces.

---
 src/Backups/BackupIO.h         | 4 ++--
 src/Backups/BackupIO_Default.h | 4 ++--
 src/Backups/BackupIO_Disk.cpp  | 5 -----
 src/Backups/BackupIO_File.cpp  | 5 +++++
 src/Backups/IBackupEntry.h     | 2 +-
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/Backups/BackupIO.h b/src/Backups/BackupIO.h
index d522387deba..e4a82a604e8 100644
--- a/src/Backups/BackupIO.h
+++ b/src/Backups/BackupIO.h
@@ -28,10 +28,10 @@ public:
     /// The function copyFileToDisk() can be much faster than reading the file with readFile() and then writing it to some disk.
     /// (especially for S3 where it can use CopyObject to copy objects inside S3 instead of downloading and uploading them).
     /// Parameters:
-    /// `encrypted_in_backup` specify if this file is encrypted in the backup,  so it shouldn't be encrypted again while restoring to an encrypted disk.
+    /// `encrypted_in_backup` specify if this file is encrypted in the backup, so it shouldn't be encrypted again while restoring to an encrypted disk.
     virtual void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
                                 DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) = 0;
-    
+
     virtual const ReadSettings & getReadSettings() const = 0;
     virtual const WriteSettings & getWriteSettings() const = 0;
     virtual size_t getWriteBufferSize() const = 0;
diff --git a/src/Backups/BackupIO_Default.h b/src/Backups/BackupIO_Default.h
index 0fc510f9361..ad7bdf15d9f 100644
--- a/src/Backups/BackupIO_Default.h
+++ b/src/Backups/BackupIO_Default.h
@@ -25,10 +25,10 @@ public:
     /// The function copyFileToDisk() can be much faster than reading the file with readFile() and then writing it to some disk.
     /// (especially for S3 where it can use CopyObject to copy objects inside S3 instead of downloading and uploading them).
     /// Parameters:
-    /// `encrypted_in_backup` specify if this file is encrypted in the backup,  so it shouldn't be encrypted again while restoring to an encrypted disk.
+    /// `encrypted_in_backup` specify if this file is encrypted in the backup, so it shouldn't be encrypted again while restoring to an encrypted disk.
     void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
                         DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) override;
-    
+
     const ReadSettings & getReadSettings() const override { return read_settings; }
     const WriteSettings & getWriteSettings() const override { return write_settings; }
     size_t getWriteBufferSize() const override { return write_buffer_size; }
diff --git a/src/Backups/BackupIO_Disk.cpp b/src/Backups/BackupIO_Disk.cpp
index 1e1af63cdc2..b58aa1832a5 100644
--- a/src/Backups/BackupIO_Disk.cpp
+++ b/src/Backups/BackupIO_Disk.cpp
@@ -8,11 +8,6 @@
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int LOGICAL_ERROR;
-}
-
 BackupReaderDisk::BackupReaderDisk(const DiskPtr & disk_, const String & root_path_, const ContextPtr & context_)
     : BackupReaderDefault(&Poco::Logger::get("BackupReaderDisk"), context_)
     , disk(disk_)
diff --git a/src/Backups/BackupIO_File.cpp b/src/Backups/BackupIO_File.cpp
index 649637e567d..eb079623919 100644
--- a/src/Backups/BackupIO_File.cpp
+++ b/src/Backups/BackupIO_File.cpp
@@ -11,6 +11,11 @@ namespace fs = std::filesystem;
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
 BackupReaderFile::BackupReaderFile(const String & root_path_, const ContextPtr & context_)
     : BackupReaderDefault(&Poco::Logger::get("BackupReaderFile"), context_)
     , root_path(root_path_)
diff --git a/src/Backups/IBackupEntry.h b/src/Backups/IBackupEntry.h
index 249c290cbe7..ca176c740c0 100644
--- a/src/Backups/IBackupEntry.h
+++ b/src/Backups/IBackupEntry.h
@@ -33,7 +33,7 @@ public:
     /// Returns true if the data returned by getReadBuffer() is encrypted by an encrypted disk.
     virtual bool isEncryptedByDisk() const { return false; }
 
-    /// Returns information about disk and file if this backup entry is generated from a file.  
+    /// Returns information about disk and file if this backup entry is generated from a file.
     virtual bool isFromFile() const { return false; }
     virtual bool isFromImmutableFile() const { return false; }
     virtual String getFilePath() const { return ""; }

From 5198997fd84dc3d2071f4b80b6888ebf8f800b81 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Wed, 3 May 2023 13:51:36 +0200
Subject: [PATCH 566/689] Remove ReadSettings from backup entries.

---
 src/Backups/BackupEntryFromAppendOnlyFile.cpp        | 9 ++++-----
 src/Backups/BackupEntryFromAppendOnlyFile.h          | 4 +---
 src/Backups/BackupEntryFromImmutableFile.cpp         | 8 +++-----
 src/Backups/BackupEntryFromImmutableFile.h           | 6 ++----
 src/Backups/BackupEntryFromMemory.cpp                | 2 +-
 src/Backups/BackupEntryFromMemory.h                  | 2 +-
 src/Backups/BackupEntryFromSmallFile.cpp             | 2 +-
 src/Backups/BackupEntryFromSmallFile.h               | 2 +-
 src/Backups/BackupEntryWithChecksumCalculation.cpp   | 4 ++--
 src/Backups/BackupEntryWrappedWith.h                 | 2 +-
 src/Backups/BackupImpl.cpp                           | 4 ++--
 src/Backups/IBackupEntriesLazyBatch.cpp              | 2 +-
 src/Backups/IBackupEntry.h                           | 2 +-
 src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp | 3 +--
 src/Storages/MergeTree/DataPartStorageOnDiskBase.h   | 1 -
 src/Storages/MergeTree/IDataPartStorage.h            | 1 -
 src/Storages/MergeTree/MergeTreeData.cpp             | 2 --
 src/Storages/StorageLog.cpp                          | 8 +++-----
 src/Storages/StorageMemory.cpp                       | 6 ++----
 src/Storages/StorageStripeLog.cpp                    | 8 +++-----
 20 files changed, 30 insertions(+), 48 deletions(-)

diff --git a/src/Backups/BackupEntryFromAppendOnlyFile.cpp b/src/Backups/BackupEntryFromAppendOnlyFile.cpp
index 2c28abc2e2f..331a4a69d06 100644
--- a/src/Backups/BackupEntryFromAppendOnlyFile.cpp
+++ b/src/Backups/BackupEntryFromAppendOnlyFile.cpp
@@ -21,24 +21,23 @@ namespace
 }
 
 BackupEntryFromAppendOnlyFile::BackupEntryFromAppendOnlyFile(
-    const DiskPtr & disk_, const String & file_path_, const ReadSettings & settings_, const std::optional<UInt64> & file_size_)
+    const DiskPtr & disk_, const String & file_path_, const std::optional<UInt64> & file_size_)
     : disk(disk_)
     , file_path(file_path_)
     , data_source_description(disk->getDataSourceDescription())
     , size(calculateSize(disk_, file_path_, data_source_description.is_encrypted, file_size_))
-    , settings(settings_.adjustBufferSize(size))
 {
 }
 
 BackupEntryFromAppendOnlyFile::~BackupEntryFromAppendOnlyFile() = default;
 
-std::unique_ptr<SeekableReadBuffer> BackupEntryFromAppendOnlyFile::getReadBuffer() const
+std::unique_ptr<SeekableReadBuffer> BackupEntryFromAppendOnlyFile::getReadBuffer(const ReadSettings & read_settings) const
 {
     std::unique_ptr<SeekableReadBuffer> buf;
     if (data_source_description.is_encrypted)
-        buf = disk->readEncryptedFile(file_path, settings);
+        buf = disk->readEncryptedFile(file_path, read_settings.adjustBufferSize(size));
     else
-        buf = disk->readFile(file_path, settings);
+        buf = disk->readFile(file_path, read_settings.adjustBufferSize(size));
     return std::make_unique<LimitSeekableReadBuffer>(std::move(buf), 0, size);
 }
 
diff --git a/src/Backups/BackupEntryFromAppendOnlyFile.h b/src/Backups/BackupEntryFromAppendOnlyFile.h
index fac8c319d8c..8a78478dcc5 100644
--- a/src/Backups/BackupEntryFromAppendOnlyFile.h
+++ b/src/Backups/BackupEntryFromAppendOnlyFile.h
@@ -15,12 +15,11 @@ public:
     BackupEntryFromAppendOnlyFile(
         const DiskPtr & disk_,
         const String & file_path_,
-        const ReadSettings & settings_,
         const std::optional<UInt64> & file_size_ = {});
 
     ~BackupEntryFromAppendOnlyFile() override;
 
-    std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override;
+    std::unique_ptr<SeekableReadBuffer> getReadBuffer(const ReadSettings & read_settings) const override;
     UInt64 getSize() const override { return size; }
 
     DataSourceDescription getDataSourceDescription() const override { return data_source_description; }
@@ -35,7 +34,6 @@ private:
     const String file_path;
     const DataSourceDescription data_source_description;
     const UInt64 size;
-    const ReadSettings settings;
 };
 
 }
diff --git a/src/Backups/BackupEntryFromImmutableFile.cpp b/src/Backups/BackupEntryFromImmutableFile.cpp
index f377e54d9b0..24965f09356 100644
--- a/src/Backups/BackupEntryFromImmutableFile.cpp
+++ b/src/Backups/BackupEntryFromImmutableFile.cpp
@@ -20,13 +20,11 @@ namespace
 BackupEntryFromImmutableFile::BackupEntryFromImmutableFile(
     const DiskPtr & disk_,
     const String & file_path_,
-    const ReadSettings & settings_,
     const std::optional<UInt64> & file_size_,
     const std::optional<UInt128> & checksum_)
     : disk(disk_)
     , file_path(file_path_)
     , data_source_description(disk->getDataSourceDescription())
-    , settings(settings_)
     , file_size(file_size_)
     , checksum(checksum_)
 {
@@ -34,12 +32,12 @@ BackupEntryFromImmutableFile::BackupEntryFromImmutableFile(
 
 BackupEntryFromImmutableFile::~BackupEntryFromImmutableFile() = default;
 
-std::unique_ptr<SeekableReadBuffer> BackupEntryFromImmutableFile::getReadBuffer() const
+std::unique_ptr<SeekableReadBuffer> BackupEntryFromImmutableFile::getReadBuffer(const ReadSettings & read_settings) const
 {
     if (data_source_description.is_encrypted)
-        return disk->readEncryptedFile(file_path, settings);
+        return disk->readEncryptedFile(file_path, read_settings);
     else
-        return disk->readFile(file_path, settings);
+        return disk->readFile(file_path, read_settings);
 }
 
 UInt64 BackupEntryFromImmutableFile::getSize() const
diff --git a/src/Backups/BackupEntryFromImmutableFile.h b/src/Backups/BackupEntryFromImmutableFile.h
index 23f48e8335a..5cdb4adc7c4 100644
--- a/src/Backups/BackupEntryFromImmutableFile.h
+++ b/src/Backups/BackupEntryFromImmutableFile.h
@@ -1,10 +1,10 @@
 #pragma once
 
 #include <Backups/BackupEntryWithChecksumCalculation.h>
-#include <IO/ReadSettings.h>
 #include <base/defines.h>
 #include <mutex>
 
+
 namespace DB
 {
 class IDisk;
@@ -18,13 +18,12 @@ public:
     BackupEntryFromImmutableFile(
         const DiskPtr & disk_,
         const String & file_path_,
-        const ReadSettings & settings_,
         const std::optional<UInt64> & file_size_ = {},
         const std::optional<UInt128> & checksum_ = {});
 
     ~BackupEntryFromImmutableFile() override;
 
-    std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override;
+    std::unique_ptr<SeekableReadBuffer> getReadBuffer(const ReadSettings & read_settings) const override;
 
     UInt64 getSize() const override;
     UInt128 getChecksum() const override;
@@ -42,7 +41,6 @@ private:
     const DiskPtr disk;
     const String file_path;
     const DataSourceDescription data_source_description;
-    const ReadSettings settings;
     mutable std::optional<UInt64> file_size;
     mutable std::optional<UInt64> checksum;
     mutable bool file_size_adjusted = false;
diff --git a/src/Backups/BackupEntryFromMemory.cpp b/src/Backups/BackupEntryFromMemory.cpp
index 82e033caca0..1dd911afbbb 100644
--- a/src/Backups/BackupEntryFromMemory.cpp
+++ b/src/Backups/BackupEntryFromMemory.cpp
@@ -14,7 +14,7 @@ BackupEntryFromMemory::BackupEntryFromMemory(String data_) : data(std::move(data
 {
 }
 
-std::unique_ptr<SeekableReadBuffer> BackupEntryFromMemory::getReadBuffer() const
+std::unique_ptr<SeekableReadBuffer> BackupEntryFromMemory::getReadBuffer(const ReadSettings &) const
 {
     return std::make_unique<ReadBufferFromString>(data);
 }
diff --git a/src/Backups/BackupEntryFromMemory.h b/src/Backups/BackupEntryFromMemory.h
index 19b72c16e53..d8bc0eb966d 100644
--- a/src/Backups/BackupEntryFromMemory.h
+++ b/src/Backups/BackupEntryFromMemory.h
@@ -14,7 +14,7 @@ public:
     BackupEntryFromMemory(const void * data_, size_t size_);
     explicit BackupEntryFromMemory(String data_);
 
-    std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override;
+    std::unique_ptr<SeekableReadBuffer> getReadBuffer(const ReadSettings &) const override;
     UInt64 getSize() const override { return data.size(); }
 
     DataSourceDescription getDataSourceDescription() const override { return DataSourceDescription{DataSourceType::RAM, "", false, false}; }
diff --git a/src/Backups/BackupEntryFromSmallFile.cpp b/src/Backups/BackupEntryFromSmallFile.cpp
index 3662522c243..3dcee7147fc 100644
--- a/src/Backups/BackupEntryFromSmallFile.cpp
+++ b/src/Backups/BackupEntryFromSmallFile.cpp
@@ -44,7 +44,7 @@ BackupEntryFromSmallFile::BackupEntryFromSmallFile(const DiskPtr & disk_, const
 {
 }
 
-std::unique_ptr<SeekableReadBuffer> BackupEntryFromSmallFile::getReadBuffer() const
+std::unique_ptr<SeekableReadBuffer> BackupEntryFromSmallFile::getReadBuffer(const ReadSettings &) const
 {
     return std::make_unique<ReadBufferFromString>(data);
 }
diff --git a/src/Backups/BackupEntryFromSmallFile.h b/src/Backups/BackupEntryFromSmallFile.h
index 6846ca71a55..4f936718fbb 100644
--- a/src/Backups/BackupEntryFromSmallFile.h
+++ b/src/Backups/BackupEntryFromSmallFile.h
@@ -16,7 +16,7 @@ public:
     explicit BackupEntryFromSmallFile(const String & file_path_);
     BackupEntryFromSmallFile(const DiskPtr & disk_, const String & file_path_);
 
-    std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override;
+    std::unique_ptr<SeekableReadBuffer> getReadBuffer(const ReadSettings &) const override;
     UInt64 getSize() const override { return data.size(); }
 
     DataSourceDescription getDataSourceDescription() const override { return data_source_description; }
diff --git a/src/Backups/BackupEntryWithChecksumCalculation.cpp b/src/Backups/BackupEntryWithChecksumCalculation.cpp
index bc97a907cff..2c0c2eab8ff 100644
--- a/src/Backups/BackupEntryWithChecksumCalculation.cpp
+++ b/src/Backups/BackupEntryWithChecksumCalculation.cpp
@@ -11,7 +11,7 @@ UInt128 BackupEntryWithChecksumCalculation<Base>::getChecksum() const
     std::lock_guard lock{checksum_calculation_mutex};
     if (!calculated_checksum)
     {
-        auto read_buffer = this->getReadBuffer();
+        auto read_buffer = this->getReadBuffer(ReadSettings{}.adjustBufferSize(this->getSize()));
         HashingReadBuffer hashing_read_buffer(*read_buffer);
         hashing_read_buffer.ignoreAll();
         calculated_checksum = hashing_read_buffer.getHash();
@@ -28,7 +28,7 @@ std::optional<UInt128> BackupEntryWithChecksumCalculation<Base>::getPartialCheck
     if (prefix_length >= this->getSize())
         return this->getChecksum();
 
-    auto read_buffer = this->getReadBuffer();
+    auto read_buffer = this->getReadBuffer(ReadSettings{}.adjustBufferSize(prefix_length));
     HashingReadBuffer hashing_read_buffer(*read_buffer);
     hashing_read_buffer.ignore(prefix_length);
     auto partial_checksum = hashing_read_buffer.getHash();
diff --git a/src/Backups/BackupEntryWrappedWith.h b/src/Backups/BackupEntryWrappedWith.h
index 01b6163039b..f865d529206 100644
--- a/src/Backups/BackupEntryWrappedWith.h
+++ b/src/Backups/BackupEntryWrappedWith.h
@@ -15,7 +15,7 @@ public:
     BackupEntryWrappedWith(BackupEntryPtr entry_, T && custom_value_) : entry(entry_), custom_value(std::move(custom_value_)) { }
     ~BackupEntryWrappedWith() override = default;
 
-    std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override { return entry->getReadBuffer(); }
+    std::unique_ptr<SeekableReadBuffer> getReadBuffer(const ReadSettings & read_settings) const override { return entry->getReadBuffer(read_settings); }
     UInt64 getSize() const override { return entry->getSize(); }
     UInt128 getChecksum() const override { return entry->getChecksum(); }
     std::optional<UInt128> getPartialChecksum(size_t prefix_length) const override { return entry->getPartialChecksum(prefix_length); }
diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp
index 715fe9e0d13..7fcb42ec378 100644
--- a/src/Backups/BackupImpl.cpp
+++ b/src/Backups/BackupImpl.cpp
@@ -868,7 +868,7 @@ void BackupImpl::writeFile(const BackupFileInfo & info, BackupEntryPtr entry)
     {
         LOG_TRACE(log, "Writing backup for file {} from {}: data file #{}, adding to archive", info.data_file_name, src_file_desc, info.data_file_index);
         auto out = archive_writer->writeFile(info.data_file_name);
-        auto read_buffer = entry->getReadBuffer();
+        auto read_buffer = entry->getReadBuffer(writer->getReadSettings());
         if (info.base_size != 0)
             read_buffer->seek(info.base_size, SEEK_SET);
         copyData(*read_buffer, *out);
@@ -882,7 +882,7 @@ void BackupImpl::writeFile(const BackupFileInfo & info, BackupEntryPtr entry)
     else
     {
         LOG_TRACE(log, "Writing backup for file {} from {}: data file #{}", info.data_file_name, src_file_desc, info.data_file_index);
-        auto create_read_buffer = [entry] { return entry->getReadBuffer(); };
+        auto create_read_buffer = [entry, read_settings = writer->getReadSettings()] { return entry->getReadBuffer(read_settings); };
         writer->copyDataToFile(info.data_file_name, create_read_buffer, info.base_size, info.size - info.base_size);
     }
 
diff --git a/src/Backups/IBackupEntriesLazyBatch.cpp b/src/Backups/IBackupEntriesLazyBatch.cpp
index 8913d60f2eb..7c6bb891981 100644
--- a/src/Backups/IBackupEntriesLazyBatch.cpp
+++ b/src/Backups/IBackupEntriesLazyBatch.cpp
@@ -17,7 +17,7 @@ class IBackupEntriesLazyBatch::BackupEntryFromBatch : public IBackupEntry
 public:
     BackupEntryFromBatch(const std::shared_ptr<IBackupEntriesLazyBatch> & batch_, size_t index_) : batch(batch_), index(index_) { }
 
-    std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override { return getInternalBackupEntry()->getReadBuffer(); }
+    std::unique_ptr<SeekableReadBuffer> getReadBuffer(const ReadSettings & read_settings) const override { return getInternalBackupEntry()->getReadBuffer(read_settings); }
     UInt64 getSize() const override { return getInternalBackupEntry()->getSize(); }
     UInt128 getChecksum() const override { return getInternalBackupEntry()->getChecksum(); }
     std::optional<UInt128> getPartialChecksum(size_t prefix_length) const override { return getInternalBackupEntry()->getPartialChecksum(prefix_length); }
diff --git a/src/Backups/IBackupEntry.h b/src/Backups/IBackupEntry.h
index ca176c740c0..7e952e9b568 100644
--- a/src/Backups/IBackupEntry.h
+++ b/src/Backups/IBackupEntry.h
@@ -28,7 +28,7 @@ public:
     virtual std::optional<UInt128> getPartialChecksum(size_t /* prefix_length */) const { return {}; }
 
     /// Returns a read buffer for reading the data.
-    virtual std::unique_ptr<SeekableReadBuffer> getReadBuffer() const = 0;
+    virtual std::unique_ptr<SeekableReadBuffer> getReadBuffer(const ReadSettings & read_settings) const = 0;
 
     /// Returns true if the data returned by getReadBuffer() is encrypted by an encrypted disk.
     virtual bool isEncryptedByDisk() const { return false; }
diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
index 4df490f41fe..c1e2c5b8cf8 100644
--- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
+++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
@@ -319,7 +319,6 @@ DataPartStorageOnDiskBase::getReplicatedFilesDescriptionForRemoteDisk(const Name
 }
 
 void DataPartStorageOnDiskBase::backup(
-    const ReadSettings & read_settings,
     const MergeTreeDataPartChecksums & checksums,
     const NameSet & files_without_checksums,
     const String & path_in_backup,
@@ -393,7 +392,7 @@ void DataPartStorageOnDiskBase::backup(
             file_hash = {it->second.file_hash.first, it->second.file_hash.second};
         }
 
-        BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromImmutableFile>(disk, filepath_on_disk, read_settings, file_size, file_hash);
+        BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromImmutableFile>(disk, filepath_on_disk, file_size, file_hash);
 
         if (temp_dir_owner)
             backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h
index fcc771f1250..52544bb2457 100644
--- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h
+++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h
@@ -50,7 +50,6 @@ public:
     ReplicatedFilesDescription getReplicatedFilesDescriptionForRemoteDisk(const NameSet & file_names) const override;
 
     void backup(
-        const ReadSettings & read_settings,
         const MergeTreeDataPartChecksums & checksums,
         const NameSet & files_without_checksums,
         const String & path_in_backup,
diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h
index b0b42b331cd..9c267d94e63 100644
--- a/src/Storages/MergeTree/IDataPartStorage.h
+++ b/src/Storages/MergeTree/IDataPartStorage.h
@@ -197,7 +197,6 @@ public:
     /// Also creates a new tmp_dir for internal disk (if disk is mentioned the first time).
     using TemporaryFilesOnDisks = std::map<DiskPtr, std::shared_ptr<TemporaryFileOnDisk>>;
     virtual void backup(
-        const ReadSettings & read_settings,
         const MergeTreeDataPartChecksums & checksums,
         const NameSet & files_without_checksums,
         const String & path_in_backup,
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index a53eb417fcd..e36bc8baeb4 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -5086,7 +5086,6 @@ BackupEntries MergeTreeData::backupParts(const DataPartsVector & data_parts, con
 
         BackupEntries backup_entries_from_part;
         part->getDataPartStorage().backup(
-            read_settings,
             part->checksums,
             part->getFileNamesWithoutChecksums(),
             data_path_in_backup,
@@ -5098,7 +5097,6 @@ BackupEntries MergeTreeData::backupParts(const DataPartsVector & data_parts, con
         for (const auto & [projection_name, projection_part] : projection_parts)
         {
             projection_part->getDataPartStorage().backup(
-                read_settings,
                 projection_part->checksums,
                 projection_part->getFileNamesWithoutChecksums(),
                 fs::path{data_path_in_backup} / part->name,
diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp
index 31f499a7d96..6d77d2b97b0 100644
--- a/src/Storages/StorageLog.cpp
+++ b/src/Storages/StorageLog.cpp
@@ -928,10 +928,8 @@ std::optional<UInt64> StorageLog::totalBytes(const Settings &) const
 
 void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional<ASTs> & /* partitions */)
 {
-    auto local_context = backup_entries_collector.getContext();
-    ReadSettings read_settings = local_context->getBackupReadSettings();
+    auto lock_timeout = getLockTimeout(backup_entries_collector.getContext());
 
-    auto lock_timeout = getLockTimeout(local_context);
     loadMarks(lock_timeout);
 
     ReadLock lock{rwlock, lock_timeout};
@@ -954,7 +952,7 @@ void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, c
         String hardlink_file_path = temp_dir / data_file_name;
         disk->createHardLink(data_file.path, hardlink_file_path);
         BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromAppendOnlyFile>(
-            disk, hardlink_file_path, read_settings, file_checker.getFileSize(data_file.path));
+            disk, hardlink_file_path, file_checker.getFileSize(data_file.path));
         backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
         backup_entries_collector.addBackupEntry(data_path_in_backup_fs / data_file_name, std::move(backup_entry));
     }
@@ -967,7 +965,7 @@ void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, c
         String hardlink_file_path = temp_dir / marks_file_name;
         disk->createHardLink(marks_file_path, hardlink_file_path);
         BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromAppendOnlyFile>(
-            disk, hardlink_file_path, read_settings, file_checker.getFileSize(marks_file_path));
+            disk, hardlink_file_path, file_checker.getFileSize(marks_file_path));
         backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
         backup_entries_collector.addBackupEntry(data_path_in_backup_fs / marks_file_name, std::move(backup_entry));
     }
diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp
index 68c888a2d23..ebc780f5ab1 100644
--- a/src/Storages/StorageMemory.cpp
+++ b/src/Storages/StorageMemory.cpp
@@ -308,8 +308,6 @@ namespace
 
         BackupEntries generate() override
         {
-            ReadSettings read_settings = context->getBackupReadSettings();
-
             BackupEntries backup_entries;
             backup_entries.resize(file_paths.size());
 
@@ -326,7 +324,7 @@ namespace
                 NativeWriter block_out{data_out, 0, metadata_snapshot->getSampleBlock(), false, &index};
                 for (const auto & block : *blocks)
                     block_out.write(block);
-                backup_entries[data_bin_pos] = {file_paths[data_bin_pos], std::make_shared<BackupEntryFromImmutableFile>(temp_disk, data_file_path, read_settings)};
+                backup_entries[data_bin_pos] = {file_paths[data_bin_pos], std::make_shared<BackupEntryFromImmutableFile>(temp_disk, data_file_path)};
             }
 
             /// Writing index.mrk
@@ -335,7 +333,7 @@ namespace
                 auto index_mrk_out_compressed = temp_disk->writeFile(index_mrk_path);
                 CompressedWriteBuffer index_mrk_out{*index_mrk_out_compressed};
                 index.write(index_mrk_out);
-                backup_entries[index_mrk_pos] = {file_paths[index_mrk_pos], std::make_shared<BackupEntryFromImmutableFile>(temp_disk, index_mrk_path, read_settings)};
+                backup_entries[index_mrk_pos] = {file_paths[index_mrk_pos], std::make_shared<BackupEntryFromImmutableFile>(temp_disk, index_mrk_path)};
             }
 
             /// Writing columns.txt
diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp
index 5b22db91631..23bf88b9db5 100644
--- a/src/Storages/StorageStripeLog.cpp
+++ b/src/Storages/StorageStripeLog.cpp
@@ -529,10 +529,8 @@ std::optional<UInt64> StorageStripeLog::totalBytes(const Settings &) const
 
 void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional<ASTs> & /* partitions */)
 {
-    auto local_context = backup_entries_collector.getContext();
-    ReadSettings read_settings = local_context->getBackupReadSettings();
+    auto lock_timeout = getLockTimeout(backup_entries_collector.getContext());
 
-    auto lock_timeout = getLockTimeout(local_context);
     loadIndices(lock_timeout);
 
     ReadLock lock{rwlock, lock_timeout};
@@ -554,7 +552,7 @@ void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collec
         String hardlink_file_path = temp_dir / data_file_name;
         disk->createHardLink(data_file_path, hardlink_file_path);
         BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromAppendOnlyFile>(
-            disk, hardlink_file_path, read_settings, file_checker.getFileSize(data_file_path));
+            disk, hardlink_file_path, file_checker.getFileSize(data_file_path));
         backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
         backup_entries_collector.addBackupEntry(data_path_in_backup_fs / data_file_name, std::move(backup_entry));
     }
@@ -566,7 +564,7 @@ void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collec
         String hardlink_file_path = temp_dir / index_file_name;
         disk->createHardLink(index_file_path, hardlink_file_path);
         BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromAppendOnlyFile>(
-            disk, hardlink_file_path, read_settings, file_checker.getFileSize(index_file_path));
+            disk, hardlink_file_path, file_checker.getFileSize(index_file_path));
         backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
         backup_entries_collector.addBackupEntry(data_path_in_backup_fs / index_file_name, std::move(backup_entry));
     }

From 019493efa370f4c7aceadc6cb6a33485bf022254 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Wed, 3 May 2023 15:56:28 +0200
Subject: [PATCH 567/689] Fix throttling in backups.

---
 src/Backups/BackupIO_Disk.cpp | 39 ++++++++++++++++++++++++-----------
 src/Backups/BackupIO_Disk.h   |  3 +++
 src/Backups/BackupIO_File.cpp | 18 +++++++++-------
 src/Backups/BackupIO_File.h   |  1 -
 src/Backups/BackupIO_S3.cpp   | 11 ++++++----
 5 files changed, 47 insertions(+), 25 deletions(-)

diff --git a/src/Backups/BackupIO_Disk.cpp b/src/Backups/BackupIO_Disk.cpp
index b58aa1832a5..3b1651bb223 100644
--- a/src/Backups/BackupIO_Disk.cpp
+++ b/src/Backups/BackupIO_Disk.cpp
@@ -12,6 +12,7 @@ BackupReaderDisk::BackupReaderDisk(const DiskPtr & disk_, const String & root_pa
     : BackupReaderDefault(&Poco::Logger::get("BackupReaderDisk"), context_)
     , disk(disk_)
     , root_path(root_path_)
+    , data_source_description(disk->getDataSourceDescription())
 {
 }
 
@@ -35,12 +36,18 @@ std::unique_ptr<SeekableReadBuffer> BackupReaderDisk::readFile(const String & fi
 void BackupReaderDisk::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
                                       DiskPtr destination_disk, const String & destination_path, WriteMode write_mode)
 {
-    if ((write_mode == WriteMode::Rewrite) && !encrypted_in_backup)
+    /// Use IDisk::copyFile() as a more optimal way to copy a file if it's possible.
+    bool has_throttling = disk->isRemote() ? static_cast<bool>(read_settings.remote_throttler) : static_cast<bool>(read_settings.local_throttler);
+    if (!has_throttling && (write_mode == WriteMode::Rewrite) && !encrypted_in_backup)
     {
-        /// Use more optimal way.
-        LOG_TRACE(log, "Copying file {} from disk {} to disk {}", path_in_backup, disk->getName(), destination_disk->getName());
-        disk->copyFile(root_path / path_in_backup, *destination_disk, destination_path, write_settings);
-        return;
+        auto destination_data_source_description = destination_disk->getDataSourceDescription();
+        if (destination_data_source_description.sameKind(data_source_description) && !data_source_description.is_encrypted)
+        {
+            /// Use more optimal way.
+            LOG_TRACE(log, "Copying file {} from disk {} to disk {}", path_in_backup, disk->getName(), destination_disk->getName());
+            disk->copyFile(root_path / path_in_backup, *destination_disk, destination_path, write_settings);
+            return; /// copied!
+        }
     }
 
     /// Fallback to copy through buffers.
@@ -52,6 +59,7 @@ BackupWriterDisk::BackupWriterDisk(const DiskPtr & disk_, const String & root_pa
     : BackupWriterDefault(&Poco::Logger::get("BackupWriterDisk"), context_)
     , disk(disk_)
     , root_path(root_path_)
+    , data_source_description(disk->getDataSourceDescription())
 {
 }
 
@@ -97,14 +105,21 @@ void BackupWriterDisk::removeFiles(const Strings & file_names)
 void BackupWriterDisk::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
                                         bool copy_encrypted, UInt64 start_pos, UInt64 length)
 {
-    if (!copy_encrypted && !start_pos && (length == src_disk->getFileSize(src_path)))
+    /// Use IDisk::copyFile() as a more optimal way to copy a file if it's possible.
+    bool has_throttling = src_disk->isRemote() ? static_cast<bool>(read_settings.remote_throttler) : static_cast<bool>(read_settings.local_throttler);
+    if (!has_throttling && !start_pos && !copy_encrypted)
     {
-        /// Use more optimal way.
-        LOG_TRACE(log, "Copying file {} from disk {} to disk {}", src_path, src_disk->getName(), disk->getName());
-        auto dest_file_path = root_path / path_in_backup;
-        disk->createDirectories(dest_file_path.parent_path());
-        src_disk->copyFile(src_path, *disk, dest_file_path, write_settings);
-        return;
+        auto source_data_source_description = src_disk->getDataSourceDescription();
+        if (source_data_source_description.sameKind(data_source_description) && !source_data_source_description.is_encrypted
+            && (length == src_disk->getFileSize(src_path)))
+        {
+            /// Use more optimal way.
+            LOG_TRACE(log, "Copying file {} from disk {} to disk {}", src_path, src_disk->getName(), disk->getName());
+            auto dest_file_path = root_path / path_in_backup;
+            disk->createDirectories(dest_file_path.parent_path());
+            src_disk->copyFile(src_path, *disk, dest_file_path, write_settings);
+            return; /// copied!
+        }
     }
 
     /// Fallback to copy through buffers.
diff --git a/src/Backups/BackupIO_Disk.h b/src/Backups/BackupIO_Disk.h
index 399ebeaa227..faf4ef03447 100644
--- a/src/Backups/BackupIO_Disk.h
+++ b/src/Backups/BackupIO_Disk.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Backups/BackupIO_Default.h>
+#include <Disks/DiskType.h>
 #include <filesystem>
 
 
@@ -26,6 +27,7 @@ public:
 private:
     const DiskPtr disk;
     const std::filesystem::path root_path;
+    const DataSourceDescription data_source_description;
 };
 
 class BackupWriterDisk : public BackupWriterDefault
@@ -50,6 +52,7 @@ private:
 
     const DiskPtr disk;
     const std::filesystem::path root_path;
+    const DataSourceDescription data_source_description;
 };
 
 }
diff --git a/src/Backups/BackupIO_File.cpp b/src/Backups/BackupIO_File.cpp
index eb079623919..7ffae26d16f 100644
--- a/src/Backups/BackupIO_File.cpp
+++ b/src/Backups/BackupIO_File.cpp
@@ -41,7 +41,9 @@ std::unique_ptr<SeekableReadBuffer> BackupReaderFile::readFile(const String & fi
 void BackupReaderFile::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
                                       DiskPtr destination_disk, const String & destination_path, WriteMode write_mode)
 {
-    if (write_mode == WriteMode::Rewrite)
+    /// std::filesystem::copy() can copy from the filesystem only, and it can't do the throttling or appending.
+    bool has_throttling = static_cast<bool>(read_settings.local_throttler);
+    if (!has_throttling && (write_mode == WriteMode::Rewrite))
     {
         auto destination_data_source_description = destination_disk->getDataSourceDescription();
         if (destination_data_source_description.sameKind(data_source_description)
@@ -50,10 +52,10 @@ void BackupReaderFile::copyFileToDisk(const String & path_in_backup, size_t file
             /// Use more optimal way.
             LOG_TRACE(log, "Copying file {} to disk {} locally", path_in_backup, destination_disk->getName());
 
-            auto write_blob_function
-                = [abs_source_path = root_path / path_in_backup, file_size](
-                      const Strings & blob_path, WriteMode mode, const std::optional<ObjectAttributes> &) -> size_t
+            auto write_blob_function = [abs_source_path = root_path / path_in_backup, file_size](
+                                           const Strings & blob_path, WriteMode mode, const std::optional<ObjectAttributes> &) -> size_t
             {
+                /// For local disks the size of a blob path is expected to be 1.
                 if (blob_path.size() != 1 || mode != WriteMode::Rewrite)
                     throw Exception(ErrorCodes::LOGICAL_ERROR,
                                     "Blob writing function called with unexpected blob_path.size={} or mode={}",
@@ -63,7 +65,7 @@ void BackupReaderFile::copyFileToDisk(const String & path_in_backup, size_t file
             };
 
             destination_disk->writeFileUsingBlobWritingFunction(destination_path, write_mode, write_blob_function);
-            return;
+            return; /// copied!
         }
     }
 
@@ -76,7 +78,6 @@ BackupWriterFile::BackupWriterFile(const String & root_path_, const ContextPtr &
     : BackupWriterDefault(&Poco::Logger::get("BackupWriterFile"), context_)
     , root_path(root_path_)
     , data_source_description(DiskLocal::getLocalDataSourceDescription(root_path))
-    , has_throttling(static_cast<bool>(read_settings.local_throttler))
 {
 }
 
@@ -120,7 +121,8 @@ void BackupWriterFile::removeFiles(const Strings & file_names)
 void BackupWriterFile::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
                                         bool copy_encrypted, UInt64 start_pos, UInt64 length)
 {
-    /// std::filesystem::copy() can copy from the filesystem only, and it cannot do the throttling.
+    /// std::filesystem::copy() can copy from the filesystem only, and it can't do the throttling or copy a part of the file.
+    bool has_throttling = static_cast<bool>(read_settings.local_throttler);
     if (!has_throttling)
     {
         auto source_data_source_description = src_disk->getDataSourceDescription();
@@ -140,7 +142,7 @@ void BackupWriterFile::copyFileFromDisk(const String & path_in_backup, DiskPtr s
                     auto abs_dest_path = root_path / path_in_backup;
                     fs::create_directories(abs_dest_path.parent_path());
                     fs::copy(abs_source_path, abs_dest_path, fs::copy_options::overwrite_existing);
-                    return;
+                    return; /// copied!
                 }
             }
         }
diff --git a/src/Backups/BackupIO_File.h b/src/Backups/BackupIO_File.h
index 45fc0d47115..fd2c0b07158 100644
--- a/src/Backups/BackupIO_File.h
+++ b/src/Backups/BackupIO_File.h
@@ -46,7 +46,6 @@ private:
 
     const std::filesystem::path root_path;
     const DataSourceDescription data_source_description;
-    const bool has_throttling;
 };
 
 }
diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp
index 09d2bdbcfaf..69f56078f9d 100644
--- a/src/Backups/BackupIO_S3.cpp
+++ b/src/Backups/BackupIO_S3.cpp
@@ -135,6 +135,8 @@ std::unique_ptr<SeekableReadBuffer> BackupReaderS3::readFile(const String & file
 void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
                                     DiskPtr destination_disk, const String & destination_path, WriteMode write_mode)
 {
+    /// Use the native copy as a more optimal way to copy a file from S3 to S3 if it's possible.
+    /// We don't check for `has_throttling` here because the native copy almost doesn't use network.
     auto destination_data_source_description = destination_disk->getDataSourceDescription();
     if (destination_data_source_description.sameKind(data_source_description)
         && (destination_data_source_description.is_encrypted == encrypted_in_backup))
@@ -166,7 +168,7 @@ void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_s
         };
 
         destination_disk->writeFileUsingBlobWritingFunction(destination_path, write_mode, write_blob_function);
-        return;
+        return; /// copied!
     }
 
     /// Fallback to copy through buffers.
@@ -189,9 +191,10 @@ BackupWriterS3::BackupWriterS3(
 void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
                                       bool copy_encrypted, UInt64 start_pos, UInt64 length)
 {
+    /// Use the native copy as a more optimal way to copy a file from S3 to S3 if it's possible.
+    /// We don't check for `has_throttling` here because the native copy almost doesn't use network.
     auto source_data_source_description = src_disk->getDataSourceDescription();
-    if (source_data_source_description.sameKind(data_source_description)
-        && (source_data_source_description.is_encrypted == copy_encrypted))
+    if (source_data_source_description.sameKind(data_source_description) && (source_data_source_description.is_encrypted == copy_encrypted))
     {
         /// getBlobPath() can return more than 2 elements if the file is stored as multiple objects in S3 bucket.
         /// In this case we can't use the native copy.
@@ -210,7 +213,7 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src
                 request_settings,
                 {},
                 threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupWriterS3"));
-            return;
+            return; /// copied!
         }
     }
 

From 943707963feff55dbf07ae47a7955253cc6adde7 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Thu, 4 May 2023 01:27:16 +0200
Subject: [PATCH 568/689] Add backup setting
 "decrypt_files_from_encrypted_disks"

---
 src/Backups/BackupEntryFromAppendOnlyFile.cpp |  15 +--
 src/Backups/BackupEntryFromAppendOnlyFile.h   |   4 +-
 src/Backups/BackupEntryFromImmutableFile.cpp  |  11 +-
 src/Backups/BackupEntryFromImmutableFile.h    |   4 +-
 src/Backups/BackupEntryFromSmallFile.cpp      |   9 +-
 src/Backups/BackupEntryFromSmallFile.h        |   5 +-
 .../BackupEntryWithChecksumCalculation.cpp    |  12 +-
 src/Backups/BackupIO_Default.cpp              |   6 +-
 src/Backups/BackupSettings.cpp                |   1 +
 src/Backups/BackupSettings.h                  |   3 +
 src/Disks/IDisk.h                             |   4 +-
 .../MergeTree/DataPartStorageOnDiskBase.cpp   |  10 +-
 .../MergeTree/DataPartStorageOnDiskBase.h     |   3 +-
 src/Storages/MergeTree/IDataPartStorage.h     |   4 +-
 src/Storages/MergeTree/MergeTreeData.cpp      |  12 +-
 src/Storages/MergeTree/MergeTreeData.h        |   2 +-
 src/Storages/StorageLog.cpp                   |   8 +-
 src/Storages/StorageMemory.cpp                |  26 ++--
 src/Storages/StorageMergeTree.cpp             |   3 +-
 src/Storages/StorageReplicatedMergeTree.cpp   |   4 +-
 src/Storages/StorageStripeLog.cpp             |   8 +-
 tests/integration/test_encrypted_disk/test.py | 119 ++++++++----------
 22 files changed, 152 insertions(+), 121 deletions(-)

diff --git a/src/Backups/BackupEntryFromAppendOnlyFile.cpp b/src/Backups/BackupEntryFromAppendOnlyFile.cpp
index 331a4a69d06..1d73ab52820 100644
--- a/src/Backups/BackupEntryFromAppendOnlyFile.cpp
+++ b/src/Backups/BackupEntryFromAppendOnlyFile.cpp
@@ -1,5 +1,5 @@
 #include <Backups/BackupEntryFromAppendOnlyFile.h>
-#include <Disks/DiskEncrypted.h>
+#include <Disks/IDisk.h>
 #include <IO/LimitSeekableReadBuffer.h>
 
 
@@ -9,11 +9,11 @@ namespace DB
 namespace
 {
     /// For append-only files we must calculate its size on the construction of a backup entry.
-    UInt64 calculateSize(const DiskPtr & disk, const String & file_path, bool is_encrypted, std::optional<UInt64> unencrypted_file_size)
+    UInt64 calculateSize(const DiskPtr & disk, const String & file_path, bool copy_encrypted, std::optional<UInt64> unencrypted_file_size)
     {
         if (!unencrypted_file_size)
-            return is_encrypted ? disk->getEncryptedFileSize(file_path) : disk->getFileSize(file_path);
-        else if (is_encrypted)
+            return copy_encrypted ? disk->getEncryptedFileSize(file_path) : disk->getFileSize(file_path);
+        else if (copy_encrypted)
             return disk->getEncryptedFileSize(*unencrypted_file_size);
         else
             return *unencrypted_file_size;
@@ -21,11 +21,12 @@ namespace
 }
 
 BackupEntryFromAppendOnlyFile::BackupEntryFromAppendOnlyFile(
-    const DiskPtr & disk_, const String & file_path_, const std::optional<UInt64> & file_size_)
+    const DiskPtr & disk_, const String & file_path_, bool copy_encrypted_, const std::optional<UInt64> & file_size_)
     : disk(disk_)
     , file_path(file_path_)
     , data_source_description(disk->getDataSourceDescription())
-    , size(calculateSize(disk_, file_path_, data_source_description.is_encrypted, file_size_))
+    , copy_encrypted(copy_encrypted_ && data_source_description.is_encrypted)
+    , size(calculateSize(disk_, file_path_, copy_encrypted, file_size_))
 {
 }
 
@@ -34,7 +35,7 @@ BackupEntryFromAppendOnlyFile::~BackupEntryFromAppendOnlyFile() = default;
 std::unique_ptr<SeekableReadBuffer> BackupEntryFromAppendOnlyFile::getReadBuffer(const ReadSettings & read_settings) const
 {
     std::unique_ptr<SeekableReadBuffer> buf;
-    if (data_source_description.is_encrypted)
+    if (copy_encrypted)
         buf = disk->readEncryptedFile(file_path, read_settings.adjustBufferSize(size));
     else
         buf = disk->readFile(file_path, read_settings.adjustBufferSize(size));
diff --git a/src/Backups/BackupEntryFromAppendOnlyFile.h b/src/Backups/BackupEntryFromAppendOnlyFile.h
index 8a78478dcc5..257c392f24c 100644
--- a/src/Backups/BackupEntryFromAppendOnlyFile.h
+++ b/src/Backups/BackupEntryFromAppendOnlyFile.h
@@ -15,6 +15,7 @@ public:
     BackupEntryFromAppendOnlyFile(
         const DiskPtr & disk_,
         const String & file_path_,
+        bool copy_encrypted_ = false,
         const std::optional<UInt64> & file_size_ = {});
 
     ~BackupEntryFromAppendOnlyFile() override;
@@ -23,7 +24,7 @@ public:
     UInt64 getSize() const override { return size; }
 
     DataSourceDescription getDataSourceDescription() const override { return data_source_description; }
-    bool isEncryptedByDisk() const override { return data_source_description.is_encrypted; }
+    bool isEncryptedByDisk() const override { return copy_encrypted; }
 
     bool isFromFile() const override { return true; }
     DiskPtr getDisk() const override { return disk; }
@@ -33,6 +34,7 @@ private:
     const DiskPtr disk;
     const String file_path;
     const DataSourceDescription data_source_description;
+    const bool copy_encrypted;
     const UInt64 size;
 };
 
diff --git a/src/Backups/BackupEntryFromImmutableFile.cpp b/src/Backups/BackupEntryFromImmutableFile.cpp
index 24965f09356..cc635dd8541 100644
--- a/src/Backups/BackupEntryFromImmutableFile.cpp
+++ b/src/Backups/BackupEntryFromImmutableFile.cpp
@@ -1,6 +1,5 @@
 #include <Backups/BackupEntryFromImmutableFile.h>
 #include <Disks/IDisk.h>
-#include <Disks/DiskEncrypted.h>
 
 
 namespace DB
@@ -20,11 +19,13 @@ namespace
 BackupEntryFromImmutableFile::BackupEntryFromImmutableFile(
     const DiskPtr & disk_,
     const String & file_path_,
+    bool copy_encrypted_,
     const std::optional<UInt64> & file_size_,
     const std::optional<UInt128> & checksum_)
     : disk(disk_)
     , file_path(file_path_)
     , data_source_description(disk->getDataSourceDescription())
+    , copy_encrypted(copy_encrypted_ && data_source_description.is_encrypted)
     , file_size(file_size_)
     , checksum(checksum_)
 {
@@ -34,7 +35,7 @@ BackupEntryFromImmutableFile::~BackupEntryFromImmutableFile() = default;
 
 std::unique_ptr<SeekableReadBuffer> BackupEntryFromImmutableFile::getReadBuffer(const ReadSettings & read_settings) const
 {
-    if (data_source_description.is_encrypted)
+    if (copy_encrypted)
         return disk->readEncryptedFile(file_path, read_settings);
     else
         return disk->readFile(file_path, read_settings);
@@ -46,8 +47,8 @@ UInt64 BackupEntryFromImmutableFile::getSize() const
     if (!file_size_adjusted)
     {
         if (!file_size)
-            file_size = data_source_description.is_encrypted ? disk->getEncryptedFileSize(file_path) : disk->getFileSize(file_path);
-        else if (data_source_description.is_encrypted)
+            file_size = copy_encrypted ? disk->getEncryptedFileSize(file_path) : disk->getFileSize(file_path);
+        else if (copy_encrypted)
             file_size = disk->getEncryptedFileSize(*file_size);
         file_size_adjusted = true;
     }
@@ -61,7 +62,7 @@ UInt128 BackupEntryFromImmutableFile::getChecksum() const
     {
         if (!checksum)
             checksum = BackupEntryWithChecksumCalculation<IBackupEntry>::getChecksum();
-        else if (data_source_description.is_encrypted)
+        else if (copy_encrypted)
             checksum = combineChecksums(*checksum, disk->getEncryptedFileIV(file_path));
         checksum_adjusted = true;
     }
diff --git a/src/Backups/BackupEntryFromImmutableFile.h b/src/Backups/BackupEntryFromImmutableFile.h
index 5cdb4adc7c4..850a86a3264 100644
--- a/src/Backups/BackupEntryFromImmutableFile.h
+++ b/src/Backups/BackupEntryFromImmutableFile.h
@@ -18,6 +18,7 @@ public:
     BackupEntryFromImmutableFile(
         const DiskPtr & disk_,
         const String & file_path_,
+        bool copy_encrypted_ = false,
         const std::optional<UInt64> & file_size_ = {},
         const std::optional<UInt128> & checksum_ = {});
 
@@ -30,7 +31,7 @@ public:
     std::optional<UInt128> getPartialChecksum(size_t prefix_length) const override;
 
     DataSourceDescription getDataSourceDescription() const override { return data_source_description; }
-    bool isEncryptedByDisk() const override { return data_source_description.is_encrypted; }
+    bool isEncryptedByDisk() const override { return copy_encrypted; }
 
     bool isFromFile() const override { return true; }
     bool isFromImmutableFile() const override { return true; }
@@ -41,6 +42,7 @@ private:
     const DiskPtr disk;
     const String file_path;
     const DataSourceDescription data_source_description;
+    const bool copy_encrypted;
     mutable std::optional<UInt64> file_size;
     mutable std::optional<UInt64> checksum;
     mutable bool file_size_adjusted = false;
diff --git a/src/Backups/BackupEntryFromSmallFile.cpp b/src/Backups/BackupEntryFromSmallFile.cpp
index 3dcee7147fc..22487767689 100644
--- a/src/Backups/BackupEntryFromSmallFile.cpp
+++ b/src/Backups/BackupEntryFromSmallFile.cpp
@@ -19,9 +19,9 @@ namespace
         return s;
     }
 
-    String readFile(const DiskPtr & disk, const String & file_path, bool read_encrypted)
+    String readFile(const DiskPtr & disk, const String & file_path, bool copy_encrypted)
     {
-        auto buf = read_encrypted ? disk->readEncryptedFile(file_path) : disk->readFile(file_path);
+        auto buf = copy_encrypted ? disk->readEncryptedFile(file_path) : disk->readFile(file_path);
         String s;
         readStringUntilEOF(s, *buf);
         return s;
@@ -36,11 +36,12 @@ BackupEntryFromSmallFile::BackupEntryFromSmallFile(const String & file_path_)
 {
 }
 
-BackupEntryFromSmallFile::BackupEntryFromSmallFile(const DiskPtr & disk_, const String & file_path_)
+BackupEntryFromSmallFile::BackupEntryFromSmallFile(const DiskPtr & disk_, const String & file_path_, bool copy_encrypted_)
     : disk(disk_)
     , file_path(file_path_)
     , data_source_description(disk_->getDataSourceDescription())
-    , data(readFile(disk_, file_path, data_source_description.is_encrypted))
+    , copy_encrypted(copy_encrypted_ && data_source_description.is_encrypted)
+    , data(readFile(disk_, file_path, copy_encrypted))
 {
 }
 
diff --git a/src/Backups/BackupEntryFromSmallFile.h b/src/Backups/BackupEntryFromSmallFile.h
index 4f936718fbb..d6651ab8cb5 100644
--- a/src/Backups/BackupEntryFromSmallFile.h
+++ b/src/Backups/BackupEntryFromSmallFile.h
@@ -14,13 +14,13 @@ class BackupEntryFromSmallFile : public BackupEntryWithChecksumCalculation<IBack
 {
 public:
     explicit BackupEntryFromSmallFile(const String & file_path_);
-    BackupEntryFromSmallFile(const DiskPtr & disk_, const String & file_path_);
+    BackupEntryFromSmallFile(const DiskPtr & disk_, const String & file_path_, bool copy_encrypted_ = false);
 
     std::unique_ptr<SeekableReadBuffer> getReadBuffer(const ReadSettings &) const override;
     UInt64 getSize() const override { return data.size(); }
 
     DataSourceDescription getDataSourceDescription() const override { return data_source_description; }
-    bool isEncryptedByDisk() const override { return data_source_description.is_encrypted; }
+    bool isEncryptedByDisk() const override { return copy_encrypted; }
 
     bool isFromFile() const override { return true; }
     DiskPtr getDisk() const override { return disk; }
@@ -30,6 +30,7 @@ private:
     const DiskPtr disk;
     const String file_path;
     const DataSourceDescription data_source_description;
+    const bool copy_encrypted = false;
     const String data;
 };
 
diff --git a/src/Backups/BackupEntryWithChecksumCalculation.cpp b/src/Backups/BackupEntryWithChecksumCalculation.cpp
index 2c0c2eab8ff..1e634e6bb73 100644
--- a/src/Backups/BackupEntryWithChecksumCalculation.cpp
+++ b/src/Backups/BackupEntryWithChecksumCalculation.cpp
@@ -25,15 +25,21 @@ std::optional<UInt128> BackupEntryWithChecksumCalculation<Base>::getPartialCheck
     if (prefix_length == 0)
         return 0;
 
-    if (prefix_length >= this->getSize())
+    size_t size = this->getSize();
+    if (prefix_length >= size)
         return this->getChecksum();
 
-    auto read_buffer = this->getReadBuffer(ReadSettings{}.adjustBufferSize(prefix_length));
+    std::lock_guard lock{checksum_calculation_mutex};
+
+    ReadSettings read_settings;
+    if (calculated_checksum)
+        read_settings.adjustBufferSize(calculated_checksum ? prefix_length : size);
+
+    auto read_buffer = this->getReadBuffer(read_settings);
     HashingReadBuffer hashing_read_buffer(*read_buffer);
     hashing_read_buffer.ignore(prefix_length);
     auto partial_checksum = hashing_read_buffer.getHash();
 
-    std::lock_guard lock{checksum_calculation_mutex};
     if (!calculated_checksum)
     {
         hashing_read_buffer.ignoreAll();
diff --git a/src/Backups/BackupIO_Default.cpp b/src/Backups/BackupIO_Default.cpp
index 3b4851e9441..f7ba061cf3a 100644
--- a/src/Backups/BackupIO_Default.cpp
+++ b/src/Backups/BackupIO_Default.cpp
@@ -82,12 +82,12 @@ void BackupWriterDefault::copyFileFromDisk(const String & path_in_backup, DiskPt
 {
     LOG_TRACE(log, "Copying file {} from disk {} through buffers", src_path, src_disk->getName());
 
-    auto create_read_buffer = [this, src_disk, src_path, file_size = start_pos + length, copy_encrypted]
+    auto create_read_buffer = [src_disk, src_path, copy_encrypted, settings = read_settings.adjustBufferSize(start_pos + length)]
     {
         if (copy_encrypted)
-            return src_disk->readEncryptedFile(src_path, read_settings, {}, file_size);
+            return src_disk->readEncryptedFile(src_path, settings);
         else
-            return src_disk->readFile(src_path, read_settings, {}, file_size);
+            return src_disk->readFile(src_path, settings);
     };
 
     copyDataToFile(path_in_backup, create_read_buffer, start_pos, length);
diff --git a/src/Backups/BackupSettings.cpp b/src/Backups/BackupSettings.cpp
index 57d85305e25..882342467fe 100644
--- a/src/Backups/BackupSettings.cpp
+++ b/src/Backups/BackupSettings.cpp
@@ -23,6 +23,7 @@ namespace ErrorCodes
     M(String, password) \
     M(Bool, structure_only) \
     M(Bool, async) \
+    M(Bool, decrypt_files_from_encrypted_disks) \
     M(Bool, deduplicate_files) \
     M(UInt64, shard_num) \
     M(UInt64, replica_num) \
diff --git a/src/Backups/BackupSettings.h b/src/Backups/BackupSettings.h
index 1b97256c75b..2c899687e6e 100644
--- a/src/Backups/BackupSettings.h
+++ b/src/Backups/BackupSettings.h
@@ -32,6 +32,9 @@ struct BackupSettings
     /// Whether the BACKUP command must return immediately without waiting until the backup has completed.
     bool async = false;
 
+    /// Whether the BACKUP command should decrypt files stored on encrypted disks.
+    bool decrypt_files_from_encrypted_disks = false;
+
     /// Whether the BACKUP will omit similar files (within one backup only).
     bool deduplicate_files = true;
 
diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h
index 47b4ed80ebf..4d74fe8bbab 100644
--- a/src/Disks/IDisk.h
+++ b/src/Disks/IDisk.h
@@ -253,14 +253,14 @@ public:
     virtual void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) = 0;
 
     /// Reads a file from an encrypted disk without decrypting it (only for encrypted disks).
-    virtual std::unique_ptr<ReadBufferFromFileBase> readEncryptedFile(
+    virtual std::unique_ptr<ReadBufferFromFileBase> readEncryptedFile( /// NOLINT
         const String & path,
         const ReadSettings & settings = ReadSettings{},
         std::optional<size_t> read_hint = {},
         std::optional<size_t> file_size = {}) const;
 
     /// Writes an already encrypted file to the disk (only for encrypted disks).
-    virtual std::unique_ptr<WriteBufferFromFileBase> writeEncryptedFile(
+    virtual std::unique_ptr<WriteBufferFromFileBase> writeEncryptedFile( /// NOLINT
         const String & path,
         size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
         WriteMode mode = WriteMode::Rewrite,
diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
index c1e2c5b8cf8..ebe55ea7dc7 100644
--- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
+++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
@@ -10,6 +10,7 @@
 #include <Backups/BackupEntryFromSmallFile.h>
 #include <Backups/BackupEntryFromImmutableFile.h>
 #include <Backups/BackupEntryWrappedWith.h>
+#include <Backups/BackupSettings.h>
 #include <Disks/SingleDiskVolume.h>
 #include <Storages/MergeTree/IMergeTreeDataPart.h>
 
@@ -322,8 +323,9 @@ void DataPartStorageOnDiskBase::backup(
     const MergeTreeDataPartChecksums & checksums,
     const NameSet & files_without_checksums,
     const String & path_in_backup,
-    BackupEntries & backup_entries,
+    const BackupSettings & backup_settings,
     bool make_temporary_hard_links,
+    BackupEntries & backup_entries,
     TemporaryFilesOnDisks * temp_dirs) const
 {
     fs::path part_path_on_disk = fs::path{root_path} / part_dir;
@@ -364,6 +366,8 @@ void DataPartStorageOnDiskBase::backup(
 
     files_to_backup = getActualFileNamesOnDisk(files_to_backup);
 
+    bool copy_encrypted = !backup_settings.decrypt_files_from_encrypted_disks;
+
     for (const auto & filepath : files_to_backup)
     {
         auto filepath_on_disk = part_path_on_disk / filepath;
@@ -371,7 +375,7 @@ void DataPartStorageOnDiskBase::backup(
 
         if (files_without_checksums.contains(filepath))
         {
-            backup_entries.emplace_back(filepath_in_backup, std::make_unique<BackupEntryFromSmallFile>(disk, filepath_on_disk));
+            backup_entries.emplace_back(filepath_in_backup, std::make_unique<BackupEntryFromSmallFile>(disk, filepath_on_disk, copy_encrypted));
             continue;
         }
 
@@ -392,7 +396,7 @@ void DataPartStorageOnDiskBase::backup(
             file_hash = {it->second.file_hash.first, it->second.file_hash.second};
         }
 
-        BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromImmutableFile>(disk, filepath_on_disk, file_size, file_hash);
+        BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromImmutableFile>(disk, filepath_on_disk, copy_encrypted, file_size, file_hash);
 
         if (temp_dir_owner)
             backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h
index 52544bb2457..09eb7f008bc 100644
--- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h
+++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h
@@ -53,8 +53,9 @@ public:
         const MergeTreeDataPartChecksums & checksums,
         const NameSet & files_without_checksums,
         const String & path_in_backup,
-        BackupEntries & backup_entries,
+        const BackupSettings & backup_settings,
         bool make_temporary_hard_links,
+        BackupEntries & backup_entries,
         TemporaryFilesOnDisks * temp_dirs) const override;
 
     MutableDataPartStoragePtr freeze(
diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h
index 9c267d94e63..7c85469d890 100644
--- a/src/Storages/MergeTree/IDataPartStorage.h
+++ b/src/Storages/MergeTree/IDataPartStorage.h
@@ -65,6 +65,7 @@ using SyncGuardPtr = std::unique_ptr<ISyncGuard>;
 class IBackupEntry;
 using BackupEntryPtr = std::shared_ptr<const IBackupEntry>;
 using BackupEntries = std::vector<std::pair<String, BackupEntryPtr>>;
+struct BackupSettings;
 
 struct WriteSettings;
 
@@ -200,8 +201,9 @@ public:
         const MergeTreeDataPartChecksums & checksums,
         const NameSet & files_without_checksums,
         const String & path_in_backup,
-        BackupEntries & backup_entries,
+        const BackupSettings & backup_settings,
         bool make_temporary_hard_links,
+        BackupEntries & backup_entries,
         TemporaryFilesOnDisks * temp_dirs) const = 0;
 
     /// Creates hardlinks into 'to/dir_path' for every file in data part.
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index e36bc8baeb4..b8208052f19 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -5051,7 +5051,11 @@ Pipe MergeTreeData::alterPartition(
 }
 
 
-BackupEntries MergeTreeData::backupParts(const DataPartsVector & data_parts, const String & data_path_in_backup, const ContextPtr & local_context)
+BackupEntries MergeTreeData::backupParts(
+    const DataPartsVector & data_parts,
+    const String & data_path_in_backup,
+    const BackupSettings & backup_settings,
+    const ContextPtr & local_context)
 {
     BackupEntries backup_entries;
     std::map<DiskPtr, std::shared_ptr<TemporaryFileOnDisk>> temp_dirs;
@@ -5089,8 +5093,9 @@ BackupEntries MergeTreeData::backupParts(const DataPartsVector & data_parts, con
             part->checksums,
             part->getFileNamesWithoutChecksums(),
             data_path_in_backup,
-            backup_entries_from_part,
+            backup_settings,
             make_temporary_hard_links,
+            backup_entries_from_part,
             &temp_dirs);
 
         auto projection_parts = part->getProjectionParts();
@@ -5100,8 +5105,9 @@ BackupEntries MergeTreeData::backupParts(const DataPartsVector & data_parts, con
                 projection_part->checksums,
                 projection_part->getFileNamesWithoutChecksums(),
                 fs::path{data_path_in_backup} / part->name,
-                backup_entries_from_part,
+                backup_settings,
                 make_temporary_hard_links,
+                backup_entries_from_part,
                 &temp_dirs);
         }
 
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index ce5c6a730e9..04b008b623c 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -1322,7 +1322,7 @@ protected:
     MovePartsOutcome movePartsToSpace(const DataPartsVector & parts, SpacePtr space);
 
     /// Makes backup entries to backup the parts of this table.
-    BackupEntries backupParts(const DataPartsVector & data_parts, const String & data_path_in_backup, const ContextPtr & local_context);
+    BackupEntries backupParts(const DataPartsVector & data_parts, const String & data_path_in_backup, const BackupSettings & backup_settings, const ContextPtr & local_context);
 
     class RestoredPartsHolder;
 
diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp
index 6d77d2b97b0..f698f1881fa 100644
--- a/src/Storages/StorageLog.cpp
+++ b/src/Storages/StorageLog.cpp
@@ -944,6 +944,8 @@ void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, c
     fs::path temp_dir = temp_dir_owner->getPath();
     disk->createDirectories(temp_dir);
 
+    bool copy_encrypted = !backup_entries_collector.getBackupSettings().decrypt_files_from_encrypted_disks;
+
     /// *.bin
     for (const auto & data_file : data_files)
     {
@@ -952,7 +954,7 @@ void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, c
         String hardlink_file_path = temp_dir / data_file_name;
         disk->createHardLink(data_file.path, hardlink_file_path);
         BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromAppendOnlyFile>(
-            disk, hardlink_file_path, file_checker.getFileSize(data_file.path));
+            disk, hardlink_file_path, copy_encrypted, file_checker.getFileSize(data_file.path));
         backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
         backup_entries_collector.addBackupEntry(data_path_in_backup_fs / data_file_name, std::move(backup_entry));
     }
@@ -965,7 +967,7 @@ void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, c
         String hardlink_file_path = temp_dir / marks_file_name;
         disk->createHardLink(marks_file_path, hardlink_file_path);
         BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromAppendOnlyFile>(
-            disk, hardlink_file_path, file_checker.getFileSize(marks_file_path));
+            disk, hardlink_file_path, copy_encrypted, file_checker.getFileSize(marks_file_path));
         backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
         backup_entries_collector.addBackupEntry(data_path_in_backup_fs / marks_file_name, std::move(backup_entry));
     }
@@ -973,7 +975,7 @@ void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, c
     /// sizes.json
     String files_info_path = file_checker.getPath();
     backup_entries_collector.addBackupEntry(
-        data_path_in_backup_fs / fileName(files_info_path), std::make_unique<BackupEntryFromSmallFile>(disk, files_info_path));
+        data_path_in_backup_fs / fileName(files_info_path), std::make_unique<BackupEntryFromSmallFile>(disk, files_info_path, copy_encrypted));
 
     /// columns.txt
     backup_entries_collector.addBackupEntry(
diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp
index ebc780f5ab1..c9654cfd105 100644
--- a/src/Storages/StorageMemory.cpp
+++ b/src/Storages/StorageMemory.cpp
@@ -23,7 +23,7 @@
 #include <Compression/CompressedReadBufferFromFile.h>
 #include <Compression/CompressedWriteBuffer.h>
 #include <Backups/BackupEntriesCollector.h>
-#include <Backups/BackupEntryFromImmutableFile.h>
+#include <Backups/BackupEntryFromAppendOnlyFile.h>
 #include <Backups/BackupEntryFromMemory.h>
 #include <Backups/BackupEntryFromSmallFile.h>
 #include <Backups/IBackup.h>
@@ -319,21 +319,25 @@ namespace
             IndexForNativeFormat index;
             {
                 auto data_file_path = temp_dir / fs::path{file_paths[data_bin_pos]}.filename();
-                auto data_out_compressed = temp_disk->writeFile(data_file_path);
-                CompressedWriteBuffer data_out{*data_out_compressed, CompressionCodecFactory::instance().getDefaultCodec(), max_compress_block_size};
-                NativeWriter block_out{data_out, 0, metadata_snapshot->getSampleBlock(), false, &index};
-                for (const auto & block : *blocks)
-                    block_out.write(block);
-                backup_entries[data_bin_pos] = {file_paths[data_bin_pos], std::make_shared<BackupEntryFromImmutableFile>(temp_disk, data_file_path)};
+                {
+                    auto data_out_compressed = temp_disk->writeFile(data_file_path);
+                    CompressedWriteBuffer data_out{*data_out_compressed, CompressionCodecFactory::instance().getDefaultCodec(), max_compress_block_size};
+                    NativeWriter block_out{data_out, 0, metadata_snapshot->getSampleBlock(), false, &index};
+                    for (const auto & block : *blocks)
+                        block_out.write(block);
+                }
+                backup_entries[data_bin_pos] = {file_paths[data_bin_pos], std::make_shared<BackupEntryFromAppendOnlyFile>(temp_disk, data_file_path)};
             }
 
             /// Writing index.mrk
             {
                 auto index_mrk_path = temp_dir / fs::path{file_paths[index_mrk_pos]}.filename();
-                auto index_mrk_out_compressed = temp_disk->writeFile(index_mrk_path);
-                CompressedWriteBuffer index_mrk_out{*index_mrk_out_compressed};
-                index.write(index_mrk_out);
-                backup_entries[index_mrk_pos] = {file_paths[index_mrk_pos], std::make_shared<BackupEntryFromImmutableFile>(temp_disk, index_mrk_path)};
+                {
+                    auto index_mrk_out_compressed = temp_disk->writeFile(index_mrk_path);
+                    CompressedWriteBuffer index_mrk_out{*index_mrk_out_compressed};
+                    index.write(index_mrk_out);
+                }
+                backup_entries[index_mrk_pos] = {file_paths[index_mrk_pos], std::make_shared<BackupEntryFromAppendOnlyFile>(temp_disk, index_mrk_path)};
             }
 
             /// Writing columns.txt
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index b4dc2830bd6..2c19d3ba122 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -2142,6 +2142,7 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_
 
 void StorageMergeTree::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional<ASTs> & partitions)
 {
+    const auto & backup_settings = backup_entries_collector.getBackupSettings();
     auto local_context = backup_entries_collector.getContext();
 
     DataPartsVector data_parts;
@@ -2154,7 +2155,7 @@ void StorageMergeTree::backupData(BackupEntriesCollector & backup_entries_collec
     for (const auto & data_part : data_parts)
         min_data_version = std::min(min_data_version, data_part->info.getDataVersion() + 1);
 
-    backup_entries_collector.addBackupEntries(backupParts(data_parts, data_path_in_backup, local_context));
+    backup_entries_collector.addBackupEntries(backupParts(data_parts, data_path_in_backup, backup_settings, local_context));
     backup_entries_collector.addBackupEntries(backupMutations(min_data_version, data_path_in_backup));
 }
 
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index fba6e6cbafc..fcb7adbd69f 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -9252,6 +9252,8 @@ void StorageReplicatedMergeTree::backupData(
     /// First we generate backup entries in the same way as an ordinary MergeTree does.
     /// But then we don't add them to the BackupEntriesCollector right away,
     /// because we need to coordinate them with other replicas (other replicas can have better parts).
+
+    const auto & backup_settings = backup_entries_collector.getBackupSettings();
     auto local_context = backup_entries_collector.getContext();
 
     DataPartsVector data_parts;
@@ -9260,7 +9262,7 @@ void StorageReplicatedMergeTree::backupData(
     else
         data_parts = getVisibleDataPartsVector(local_context);
 
-    auto backup_entries = backupParts(data_parts, /* data_path_in_backup */ "", local_context);
+    auto backup_entries = backupParts(data_parts, /* data_path_in_backup */ "", backup_settings, local_context);
 
     auto coordination = backup_entries_collector.getBackupCoordination();
     String shared_id = getTableSharedID();
diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp
index 23bf88b9db5..b2e7c202800 100644
--- a/src/Storages/StorageStripeLog.cpp
+++ b/src/Storages/StorageStripeLog.cpp
@@ -545,6 +545,8 @@ void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collec
     fs::path temp_dir = temp_dir_owner->getPath();
     disk->createDirectories(temp_dir);
 
+    bool copy_encrypted = !backup_entries_collector.getBackupSettings().decrypt_files_from_encrypted_disks;
+
     /// data.bin
     {
         /// We make a copy of the data file because it can be changed later in write() or in truncate().
@@ -552,7 +554,7 @@ void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collec
         String hardlink_file_path = temp_dir / data_file_name;
         disk->createHardLink(data_file_path, hardlink_file_path);
         BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromAppendOnlyFile>(
-            disk, hardlink_file_path, file_checker.getFileSize(data_file_path));
+            disk, hardlink_file_path, copy_encrypted, file_checker.getFileSize(data_file_path));
         backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
         backup_entries_collector.addBackupEntry(data_path_in_backup_fs / data_file_name, std::move(backup_entry));
     }
@@ -564,7 +566,7 @@ void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collec
         String hardlink_file_path = temp_dir / index_file_name;
         disk->createHardLink(index_file_path, hardlink_file_path);
         BackupEntryPtr backup_entry = std::make_unique<BackupEntryFromAppendOnlyFile>(
-            disk, hardlink_file_path, file_checker.getFileSize(index_file_path));
+            disk, hardlink_file_path, copy_encrypted, file_checker.getFileSize(index_file_path));
         backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner);
         backup_entries_collector.addBackupEntry(data_path_in_backup_fs / index_file_name, std::move(backup_entry));
     }
@@ -572,7 +574,7 @@ void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collec
     /// sizes.json
     String files_info_path = file_checker.getPath();
     backup_entries_collector.addBackupEntry(
-        data_path_in_backup_fs / fileName(files_info_path), std::make_unique<BackupEntryFromSmallFile>(disk, files_info_path));
+        data_path_in_backup_fs / fileName(files_info_path), std::make_unique<BackupEntryFromSmallFile>(disk, files_info_path, copy_encrypted));
 
     /// columns.txt
     backup_entries_collector.addBackupEntry(
diff --git a/tests/integration/test_encrypted_disk/test.py b/tests/integration/test_encrypted_disk/test.py
index 90da5849c7f..66ff073f02b 100644
--- a/tests/integration/test_encrypted_disk/test.py
+++ b/tests/integration/test_encrypted_disk/test.py
@@ -309,17 +309,24 @@ def test_restart():
 
 
 @pytest.mark.parametrize(
-    "storage_policy,backup_type,storage_policy2",
+    "backup_type,old_storage_policy,new_storage_policy,decrypt_files_from_encrypted_disks",
     [
-        ("encrypted_policy", "S3", "encrypted_policy"),
-        ("encrypted_policy", "S3", "s3_encrypted_default_path"),
-        ("s3_encrypted_default_path", "S3", "s3_encrypted_default_path"),
-        ("s3_encrypted_default_path", "S3", "encrypted_policy"),
-        ("s3_encrypted_default_path", "File", "encrypted_policy"),
-        ("local_policy", "File", "encrypted_policy"),
+        ("S3", "encrypted_policy", "encrypted_policy", False),
+        ("S3", "encrypted_policy", "s3_encrypted_default_path", False),
+        ("S3", "s3_encrypted_default_path", "s3_encrypted_default_path", False),
+        ("S3", "s3_encrypted_default_path", "encrypted_policy", False),
+        ("File", "s3_encrypted_default_path", "encrypted_policy", False),
+        ("File", "local_policy", "encrypted_policy", False),
+        ("File", "encrypted_policy", "local_policy", False),
+        ("File", "encrypted_policy", "local_policy", True),
     ],
 )
-def test_backup_restore(storage_policy, backup_type, storage_policy2):
+def test_backup_restore(
+    backup_type,
+    old_storage_policy,
+    new_storage_policy,
+    decrypt_files_from_encrypted_disks,
+):
     node.query(
         f"""
         CREATE TABLE encrypted_test (
@@ -327,7 +334,7 @@ def test_backup_restore(storage_policy, backup_type, storage_policy2):
             data String
         ) ENGINE=MergeTree()
         ORDER BY id
-        SETTINGS storage_policy='{storage_policy}'
+        SETTINGS storage_policy='{old_storage_policy}'
         """
     )
 
@@ -343,22 +350,38 @@ def test_backup_restore(storage_policy, backup_type, storage_policy2):
     elif backup_type == "File":
         backup_destination = f"File('/backups/{backup_name}/')"
 
-    node.query(f"BACKUP TABLE encrypted_test TO {backup_destination}")
+    node.query(
+        f"BACKUP TABLE encrypted_test TO {backup_destination} SETTINGS decrypt_files_from_encrypted_disks={int(decrypt_files_from_encrypted_disks)}"
+    )
 
-    if backup_type == "File" and storage_policy.find("encrypted") != -1:
+    storage_policy_changed = old_storage_policy != new_storage_policy
+    old_disk_encrypted = old_storage_policy.find("encrypted") != -1
+    new_disk_encrypted = new_storage_policy.find("encrypted") != -1
+
+    if backup_type == "File":
         root_path = os.path.join(node.cluster.instances_dir, "backups", backup_name)
+        expect_encrypted_in_backup = (
+            old_disk_encrypted and not decrypt_files_from_encrypted_disks
+        )
+
+        with open(f"{root_path}/metadata/default/encrypted_test.sql") as file:
+            assert file.read().startswith("CREATE TABLE default.encrypted_test")
+
+        with open(f"{root_path}/.backup") as file:
+            found_encrypted_in_backup = (
+                file.read().find("<encrypted_by_disk>true</encrypted_by_disk>") != -1
+            )
+            assert found_encrypted_in_backup == expect_encrypted_in_backup
+
         with open(
             f"{root_path}/data/default/encrypted_test/all_1_1_0/data.bin", "rb"
         ) as file:
-            assert file.read().startswith(b"ENC")
-        with open(f"{root_path}/metadata/default/encrypted_test.sql") as file:
-            assert file.read().startswith("CREATE TABLE default.encrypted_test")
-        with open(f"{root_path}/.backup") as file:
-            assert file.read().find("<encrypted_by_disk>true</encrypted_by_disk>") != -1
+            found_encrypted_in_backup = file.read().startswith(b"ENC")
+            assert found_encrypted_in_backup == expect_encrypted_in_backup
 
     node.query(f"DROP TABLE encrypted_test SYNC")
 
-    if storage_policy != storage_policy2:
+    if storage_policy_changed:
         node.query(
             f"""
             CREATE TABLE encrypted_test (
@@ -366,56 +389,22 @@ def test_backup_restore(storage_policy, backup_type, storage_policy2):
                 data String
             ) ENGINE=MergeTree()
             ORDER BY id
-            SETTINGS storage_policy='{storage_policy2}'
+            SETTINGS storage_policy='{new_storage_policy}'
             """
         )
 
-    node.query(
-        f"RESTORE TABLE encrypted_test FROM {backup_destination} SETTINGS allow_different_table_def={int(storage_policy != storage_policy2)}"
-    )
+    restore_command = f"RESTORE TABLE encrypted_test FROM {backup_destination} SETTINGS allow_different_table_def={int(storage_policy_changed)}"
 
-    assert node.query(select_query) == "(0,'data'),(1,'data')"
+    expect_error = None
+    if (
+        old_disk_encrypted
+        and not new_disk_encrypted
+        and not decrypt_files_from_encrypted_disks
+    ):
+        expect_error = "can be restored only to an encrypted disk"
 
-
-def test_cannot_restore_encrypted_files_to_unencrypted_disk():
-    node.query(
-        """
-        CREATE TABLE encrypted_test (
-            id Int64,
-            data String
-        ) ENGINE=MergeTree()
-        ORDER BY id
-        SETTINGS storage_policy='encrypted_policy'
-        """
-    )
-
-    node.query("INSERT INTO encrypted_test VALUES (0,'data'),(1,'data')")
-    assert (
-        node.query("SELECT * FROM encrypted_test ORDER BY id FORMAT Values")
-        == "(0,'data'),(1,'data')"
-    )
-
-    backup_name = new_backup_name()
-    backup_destination = (
-        f"S3('http://minio1:9001/root/backups/{backup_name}', 'minio', 'minio123')"
-    )
-    node.query(f"BACKUP TABLE encrypted_test TO {backup_destination}")
-
-    node.query(f"DROP TABLE encrypted_test SYNC")
-
-    node.query(
-        f"""
-        CREATE TABLE encrypted_test (
-            id Int64,
-            data String
-        ) ENGINE=MergeTree()
-        ORDER BY id
-        SETTINGS storage_policy='local_policy'
-        """
-    )
-
-    expected_error = "can be restored only to an encrypted disk"
-    assert expected_error in node.query_and_get_error(
-        f"RESTORE TABLE encrypted_test FROM {backup_destination} SETTINGS allow_different_table_def=1"
-    )
->>>>>>> 9c08fb30995 (Add tests.)
+    if expect_error:
+        assert expect_error in node.query_and_get_error(restore_command)
+    else:
+        node.query(restore_command)
+        assert node.query(select_query) == "(0,'data'),(1,'data')"

From 2ec94a42b728a7daa6ab0c2c72fab1119e5f2a1d Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Fri, 5 May 2023 10:40:12 +0200
Subject: [PATCH 569/689] Remove default parameters from virtual functions.

---
 src/Disks/DiskEncrypted.h |  7 ++-----
 src/Disks/IDisk.cpp       |  2 +-
 src/Disks/IDisk.h         | 13 +++----------
 3 files changed, 6 insertions(+), 16 deletions(-)

diff --git a/src/Disks/DiskEncrypted.h b/src/Disks/DiskEncrypted.h
index c494dd6a216..530d9b2dc02 100644
--- a/src/Disks/DiskEncrypted.h
+++ b/src/Disks/DiskEncrypted.h
@@ -198,13 +198,10 @@ public:
         delegate->writeFileUsingBlobWritingFunction(wrapped_path, mode, std::move(write_blob_function));
     }
 
-    std::unique_ptr<ReadBufferFromFileBase> readEncryptedFile(
-        const String & path, const ReadSettings & settings,
-        std::optional<size_t> read_hint,
-        std::optional<size_t> file_size) const override
+    std::unique_ptr<ReadBufferFromFileBase> readEncryptedFile(const String & path, const ReadSettings & settings) const override
     {
         auto wrapped_path = wrappedPath(path);
-        return delegate->readFile(wrapped_path, settings, read_hint, file_size);
+        return delegate->readFile(wrapped_path, settings);
     }
 
     std::unique_ptr<WriteBufferFromFileBase> writeEncryptedFile(
diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp
index 9a5ae997b46..88dd65bfde7 100644
--- a/src/Disks/IDisk.cpp
+++ b/src/Disks/IDisk.cpp
@@ -52,7 +52,7 @@ void IDisk::removeSharedFiles(const RemoveBatchRequest & files, bool keep_all_ba
     }
 }
 
-std::unique_ptr<ReadBufferFromFileBase> IDisk::readEncryptedFile(const String &, const ReadSettings &, std::optional<size_t>, std::optional<size_t>) const
+std::unique_ptr<ReadBufferFromFileBase> IDisk::readEncryptedFile(const String &, const ReadSettings &) const
 {
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "File encryption is not implemented for disk of type {}", getDataSourceDescription().type);
 }
diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h
index 4d74fe8bbab..e3ac790d2b7 100644
--- a/src/Disks/IDisk.h
+++ b/src/Disks/IDisk.h
@@ -253,18 +253,11 @@ public:
     virtual void writeFileUsingBlobWritingFunction(const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) = 0;
 
     /// Reads a file from an encrypted disk without decrypting it (only for encrypted disks).
-    virtual std::unique_ptr<ReadBufferFromFileBase> readEncryptedFile( /// NOLINT
-        const String & path,
-        const ReadSettings & settings = ReadSettings{},
-        std::optional<size_t> read_hint = {},
-        std::optional<size_t> file_size = {}) const;
+    virtual std::unique_ptr<ReadBufferFromFileBase> readEncryptedFile(const String & path, const ReadSettings & settings) const;
 
     /// Writes an already encrypted file to the disk (only for encrypted disks).
-    virtual std::unique_ptr<WriteBufferFromFileBase> writeEncryptedFile( /// NOLINT
-        const String & path,
-        size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
-        WriteMode mode = WriteMode::Rewrite,
-        const WriteSettings & settings = {}) const;
+    virtual std::unique_ptr<WriteBufferFromFileBase> writeEncryptedFile(
+        const String & path, size_t buf_size, WriteMode mode, const WriteSettings & settings) const;
 
     /// Returns the size of an encrypted file (only for encrypted disks).
     virtual size_t getEncryptedFileSize(const String & path) const;

From b068f0b619ceb83023ba116c66a3fcafec7e8305 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Fri, 5 May 2023 11:40:33 +0200
Subject: [PATCH 570/689] Fix build.

---
 src/Backups/BackupEntryFromSmallFile.cpp               |  2 +-
 src/Backups/BackupIO_S3.cpp                            | 10 +++++-----
 src/Disks/IDisk.h                                      |  4 ++--
 src/Disks/ObjectStorages/DiskObjectStorage.cpp         |  4 ++--
 .../ObjectStorages/DiskObjectStorageTransaction.cpp    |  6 ++++--
 5 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/Backups/BackupEntryFromSmallFile.cpp b/src/Backups/BackupEntryFromSmallFile.cpp
index 22487767689..d0a99056b59 100644
--- a/src/Backups/BackupEntryFromSmallFile.cpp
+++ b/src/Backups/BackupEntryFromSmallFile.cpp
@@ -21,7 +21,7 @@ namespace
 
     String readFile(const DiskPtr & disk, const String & file_path, bool copy_encrypted)
     {
-        auto buf = copy_encrypted ? disk->readEncryptedFile(file_path) : disk->readFile(file_path);
+        auto buf = copy_encrypted ? disk->readEncryptedFile(file_path, {}) : disk->readFile(file_path);
         String s;
         readStringUntilEOF(s, *buf);
         return s;
diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp
index 69f56078f9d..40ecde71173 100644
--- a/src/Backups/BackupIO_S3.cpp
+++ b/src/Backups/BackupIO_S3.cpp
@@ -157,8 +157,8 @@ void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_s
                 fs::path(s3_uri.key) / path_in_backup,
                 0,
                 file_size,
-                /* dest_bucket= */ blob_path[0],
-                /* dest_key= */ blob_path[1],
+                /* dest_bucket= */ blob_path[1],
+                /* dest_key= */ blob_path[0],
                 request_settings,
                 object_attributes,
                 threadPoolCallbackRunner<void>(BackupsIOThreadPool::get(), "BackupReaderS3"),
@@ -196,7 +196,7 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src
     auto source_data_source_description = src_disk->getDataSourceDescription();
     if (source_data_source_description.sameKind(data_source_description) && (source_data_source_description.is_encrypted == copy_encrypted))
     {
-        /// getBlobPath() can return more than 2 elements if the file is stored as multiple objects in S3 bucket.
+        /// getBlobPath() can return more than 3 elements if the file is stored as multiple objects in S3 bucket.
         /// In this case we can't use the native copy.
         if (auto blob_path = src_disk->getBlobPath(src_path); blob_path.size() == 2)
         {
@@ -204,8 +204,8 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src
             LOG_TRACE(log, "Copying file {} from disk {} to S3 using native copy", src_path, src_disk->getName());
             copyS3File(
                 client,
-                /* src_bucket */ blob_path[0],
-                /* src_key= */ blob_path[1],
+                /* src_bucket */ blob_path[1],
+                /* src_key= */ blob_path[0],
                 start_pos,
                 length,
                 s3_uri.bucket,
diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h
index e3ac790d2b7..6bbd7c26bec 100644
--- a/src/Disks/IDisk.h
+++ b/src/Disks/IDisk.h
@@ -240,8 +240,8 @@ public:
 
     /// Returns the path to a blob representing a specified file.
     /// The meaning of the returned path depends on disk's type.
-    /// E.g. for DiskLocal it the absolute path to the file and for DiskObjectStorage it's the name of the objects' namespace
-    /// combined with StoredObject::absolute_path for each stored object representing a specified file.
+    /// E.g. for DiskLocal it's the absolute path to the file and for DiskObjectStorage it's
+    /// StoredObject::remote_path for each stored object combined with the name of the objects' namespace.
     virtual Strings getBlobPath(const String & path) const = 0;
 
     using WriteBlobFunction = std::function<size_t(const Strings & blob_path, WriteMode mode, const std::optional<ObjectAttributes> & object_attributes)>;
diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp
index bfa1ed1ab26..129f1ab1ef7 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp
@@ -584,11 +584,11 @@ Strings DiskObjectStorage::getBlobPath(const String & path) const
     auto objects = getStorageObjects(path);
     Strings res;
     res.reserve(objects.size() + 1);
+    for (const auto & object : objects)
+        res.emplace_back(object.remote_path);
     String objects_namespace = object_storage->getObjectsNamespace();
     if (!objects_namespace.empty())
         res.emplace_back(objects_namespace);
-    for (const auto & object : objects)
-        res.emplace_back(object.absolute_path);
     return res;
 }
 
diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
index f98ac55889b..2c22df64d90 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
+++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
@@ -694,8 +694,10 @@ void DiskObjectStorageTransaction::writeFileUsingBlobWritingFunction(
     /// See DiskObjectStorage::getBlobPath().
     Strings blob_path;
     blob_path.reserve(2);
-    blob_path.emplace_back(object_storage.getObjectsNamespace());
-    blob_path.emplace_back(object.absolute_path);
+    blob_path.emplace_back(object.remote_path);
+    String objects_namespace = object_storage.getObjectsNamespace();
+    if (!objects_namespace.empty())
+        blob_path.emplace_back(objects_namespace);
 
     /// We always use mode Rewrite because we simulate append using metadata and different files
     size_t object_size = std::move(write_blob_function)(blob_path, WriteMode::Rewrite, object_attributes);

From 68138395ebb4b7c129f9ef8c6c8fbd2a8dfc1c9b Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Fri, 5 May 2023 15:31:12 +0200
Subject: [PATCH 571/689] Fix parameterized views when query parameter used
 multiple times in the query

Example:

    CREATE VIEW view AS
    SELECT *
    FROM system.one
    WHERE dummy = {k1:Int}+1 OR dummy = {k1:Int}+2
                   ^^                    ^^

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Interpreters/ActionsVisitor.cpp               | 15 +++++++++++++--
 .../02428_parameterized_view.reference            |  3 ++-
 .../0_stateless/02428_parameterized_view.sh       |  5 +++--
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp
index b3bb6e9db90..28ba2e3e838 100644
--- a/src/Interpreters/ActionsVisitor.cpp
+++ b/src/Interpreters/ActionsVisitor.cpp
@@ -1216,11 +1216,22 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
             else if (data.is_create_parameterized_view && query_parameter)
             {
                 const auto data_type = DataTypeFactory::instance().get(query_parameter->type);
-                ColumnWithTypeAndName column(data_type,query_parameter->getColumnName());
+                /// Use getUniqueName() to allow multiple use of query parameter in the query:
+                ///
+                ///     CREATE VIEW view AS
+                ///     SELECT *
+                ///     FROM system.one
+                ///     WHERE dummy = {k1:Int}+1 OR dummy = {k1:Int}+2
+                ///                    ^^                    ^^
+                ///
+                /// NOTE: query in the VIEW will not be modified this is needed
+                /// only during analysis for CREATE VIEW to avoid duplicated
+                /// column names.
+                ColumnWithTypeAndName column(data_type, data.getUniqueName("__" + query_parameter->getColumnName()));
                 data.addColumn(column);
 
                 argument_types.push_back(data_type);
-                argument_names.push_back(query_parameter->name);
+                argument_names.push_back(column.name);
             }
             else
             {
diff --git a/tests/queries/0_stateless/02428_parameterized_view.reference b/tests/queries/0_stateless/02428_parameterized_view.reference
index 59a3f4d67dd..b73c52c478f 100644
--- a/tests/queries/0_stateless/02428_parameterized_view.reference
+++ b/tests/queries/0_stateless/02428_parameterized_view.reference
@@ -19,7 +19,8 @@ ERROR
 50
 ERROR
 10
-ERROR
+10
+20
 20
 ERROR
 30
diff --git a/tests/queries/0_stateless/02428_parameterized_view.sh b/tests/queries/0_stateless/02428_parameterized_view.sh
index 57a43ce08a0..3abfbfc22fc 100755
--- a/tests/queries/0_stateless/02428_parameterized_view.sh
+++ b/tests/queries/0_stateless/02428_parameterized_view.sh
@@ -62,7 +62,8 @@ $CLICKHOUSE_CLIENT -q "SELECT Price FROM test_02428_pv2(price=50)"  2>&1 |  grep
 $CLICKHOUSE_CLIENT -q "CREATE VIEW test_02428_pv3 AS SELECT * FROM test_02428_Catalog WHERE Price={price:UInt64} AND Quantity=3"
 $CLICKHOUSE_CLIENT -q "SELECT Price FROM test_02428_pv3(price=10)"
 
-$CLICKHOUSE_CLIENT -q "CREATE VIEW test_02428_pv4 AS SELECT * FROM test_02428_Catalog WHERE Price={price:UInt64} AND Quantity={price:UInt64}" 2>&1 |  grep -Fq "DUPLICATE_COLUMN" && echo 'ERROR' || echo 'OK'
+$CLICKHOUSE_CLIENT -q "CREATE VIEW test_02428_pv4 AS SELECT * FROM test_02428_Catalog WHERE Price={price:UInt64} OR Price={price:UInt64}*2"
+$CLICKHOUSE_CLIENT -q "SELECT Price FROM test_02428_pv4(price=10) ORDER BY Price"
 
 $CLICKHOUSE_CLIENT -q "CREATE DATABASE ${CLICKHOUSE_TEST_UNIQUE_NAME}"
 $CLICKHOUSE_CLIENT -q "CREATE TABLE ${CLICKHOUSE_TEST_UNIQUE_NAME}.Catalog (Name String, Price UInt64, Quantity UInt64) ENGINE = Memory"
@@ -130,4 +131,4 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE ${CLICKHOUSE_TEST_UNIQUE_NAME}.Catalog"
 $CLICKHOUSE_CLIENT -q "DROP DATABASE ${CLICKHOUSE_TEST_UNIQUE_NAME}"
 $CLICKHOUSE_CLIENT -q "DROP VIEW 02428_trace_view"
 $CLICKHOUSE_CLIENT -q "DROP TABLE 02428_otel_traces_trace_id_ts"
-$CLICKHOUSE_CLIENT -q "DROP TABLE 02428_otel_traces"
\ No newline at end of file
+$CLICKHOUSE_CLIENT -q "DROP TABLE 02428_otel_traces"

From ea5976180969fafb7dc7db8b5be5ff2d9c2976f0 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 16 May 2023 15:25:04 +0200
Subject: [PATCH 572/689] fix OptimizeRegularExpression

---
 src/Common/OptimizedRegularExpression.cpp | 33 +++++++++++++++--------
 src/Common/tests/gtest_optimize_re.cpp    | 32 ++++++++++++----------
 2 files changed, 40 insertions(+), 25 deletions(-)

diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp
index 68f5b86877e..92b3ad32ecd 100644
--- a/src/Common/OptimizedRegularExpression.cpp
+++ b/src/Common/OptimizedRegularExpression.cpp
@@ -63,12 +63,13 @@ const char * analyzeImpl(
     bool is_first_call = begin == regexp.data();
     int depth = 0;
     is_trivial = true;
+    bool is_prefix = true;
     required_substring.clear();
     bool has_alternative_on_depth_0 = false;
     bool has_case_insensitive_flag = false;
 
     /// Substring with a position.
-    using Substring = std::pair<std::string, size_t>;
+    using Substring = std::pair<std::string, bool>;
     using Substrings = std::vector<Substring>;
 
     Substrings trivial_substrings(1);
@@ -98,6 +99,9 @@ const char * analyzeImpl(
 
     auto finish_non_trivial_char = [&](bool create_new_substr = true)
     {
+        is_trivial = false;
+        if (create_new_substr)
+            is_prefix = false;
         if (depth != 0)
             return;
 
@@ -106,6 +110,7 @@ const char * analyzeImpl(
             if (alter.suffix)
             {
                 alter.literal += last_substring->first;
+                alter.suffix = false;
             }
         }
 
@@ -126,16 +131,24 @@ const char * analyzeImpl(
             if (alter.prefix)
             {
                 alter.literal = last_substring->first + alter.literal;
+                alter.prefix = is_prefix;
             }
         }
 
         if (group_required_string.prefix)
+        {
             last_substring->first += group_required_string.literal;
+            last_substring->second = is_prefix;
+        }
         else
         {
             finish_non_trivial_char();
             last_substring->first = group_required_string.literal;
+            last_substring->second = false;
         }
+
+        is_prefix = is_prefix && group_required_string.prefix && group_required_string.suffix;
+
         /// if we can still append, no need to finish it. e.g. abc(de)fg should capture abcdefg
         if (!last_substring->first.empty() && !group_required_string.suffix)
         {
@@ -185,7 +198,6 @@ const char * analyzeImpl(
                         goto ordinary;
                     default:
                         /// all other escape sequences are not supported
-                        is_trivial = false;
                         finish_non_trivial_char();
                         break;
                 }
@@ -196,6 +208,7 @@ const char * analyzeImpl(
 
             case '|':
                 is_trivial = false;
+                is_prefix = false;
                 ++pos;
                 if (depth == 0)
                 {
@@ -205,6 +218,7 @@ const char * analyzeImpl(
                 break;
 
             case '(':
+                /// bracket does not break is_prefix. for example abc(d) has a prefix 'abcd'
                 is_trivial = false;
                 if (!in_square_braces)
                 {
@@ -258,7 +272,6 @@ const char * analyzeImpl(
             case '[':
                 in_square_braces = true;
                 ++depth;
-                is_trivial = false;
                 finish_non_trivial_char();
                 ++pos;
                 break;
@@ -270,7 +283,6 @@ const char * analyzeImpl(
                 --depth;
                 if (depth == 0)
                     in_square_braces = false;
-                is_trivial = false;
                 finish_non_trivial_char();
                 ++pos;
                 break;
@@ -284,7 +296,6 @@ const char * analyzeImpl(
                 break;
 
             case '^': case '$': case '.': case '+':
-                is_trivial = false;
                 finish_non_trivial_char();
                 ++pos;
                 break;
@@ -296,7 +307,6 @@ const char * analyzeImpl(
             case '?':
                 [[fallthrough]];
             case '*':
-                is_trivial = false;
                 if (depth == 0 && !last_substring->first.empty() && !in_square_braces)
                 {
                     last_substring->first.resize(last_substring->first.size() - 1);
@@ -318,8 +328,9 @@ const char * analyzeImpl(
             default:
                 if (depth == 0 && !in_curly_braces && !in_square_braces)
                 {
+                    /// record the first position of last string.
                     if (last_substring->first.empty())
-                        last_substring->second = pos - begin;
+                        last_substring->second = is_prefix;
                     last_substring->first.push_back(*pos);
                 }
                 ++pos;
@@ -328,10 +339,9 @@ const char * analyzeImpl(
     }
 finish:
 
-    finish_non_trivial_char(false);
-
     if (!is_trivial)
     {
+        finish_non_trivial_char(false);
         /// we calculate required substring even though has_alternative_on_depth_0.
         /// we will clear the required substring after putting it to alternatives.
         if (!has_case_insensitive_flag)
@@ -357,7 +367,7 @@ finish:
             if (max_length >= MIN_LENGTH_FOR_STRSTR || (!is_first_call && max_length > 0))
             {
                 required_substring.literal = candidate_it->first;
-                required_substring.prefix = candidate_it->second == 0;
+                required_substring.prefix = candidate_it->second;
                 required_substring.suffix = candidate_it + 1 == trivial_substrings.end();
             }
         }
@@ -365,7 +375,8 @@ finish:
     else if (!trivial_substrings.empty())
     {
         required_substring.literal = trivial_substrings.front().first;
-        required_substring.prefix = trivial_substrings.front().second == 0;
+        /// trivial string means the whole regex is a simple string literal, so the prefix and suffix should be true.
+        required_substring.prefix = true;
         required_substring.suffix = true;
     }
 
diff --git a/src/Common/tests/gtest_optimize_re.cpp b/src/Common/tests/gtest_optimize_re.cpp
index 556700f1fcc..3710666d336 100644
--- a/src/Common/tests/gtest_optimize_re.cpp
+++ b/src/Common/tests/gtest_optimize_re.cpp
@@ -4,37 +4,40 @@
 
 TEST(OptimizeRE, analyze)
 {
-    auto test_f = [](const std::string & regexp, const std::string & answer, std::vector<std::string> expect_alternatives = {}, bool trival_expected = false)
+    auto test_f = [](const std::string & regexp, const std::string & required, std::vector<std::string> expect_alternatives = {}, bool trival_expected = false, bool prefix_expected = false)
     {
-        std::string required;
+        std::string answer;
         bool is_trivial;
         bool is_prefix;
         std::vector<std::string> alternatives;
-        OptimizedRegularExpression::analyze(regexp, required, is_trivial, is_prefix, alternatives);
+        OptimizedRegularExpression::analyze(regexp, answer, is_trivial, is_prefix, alternatives);
         std::cerr << regexp << std::endl;
         EXPECT_EQ(required, answer);
         EXPECT_EQ(alternatives, expect_alternatives);
         EXPECT_EQ(is_trivial, trival_expected);
+        EXPECT_EQ(is_prefix, prefix_expected);
     };
-    test_f("abc", "abc", {}, true);
+    test_f("abc", "abc", {}, true, true);
     test_f("c([^k]*)de", "");
-    test_f("abc(de)fg", "abcdefg");
-    test_f("abc(de|xyz)fg", "abc", {"abcdefg", "abcxyzfg"});
-    test_f("abc(de?f|xyz)fg", "abc", {"abcd", "abcxyzfg"});
+    test_f("abc(de)fg", "abcdefg", {}, false, true);
+    test_f("abc(de|xyz)fg", "abc", {"abcdefg", "abcxyzfg"}, false, true);
+    test_f("abc(de?f|xyz)fg", "abc", {"abcd", "abcxyzfg"}, false, true);
     test_f("abc|fgk|xyz", "", {"abc","fgk", "xyz"});
-    test_f("(abc)", "abc");
+    test_f("(abc)", "abc", {}, false, true);
     test_f("(abc|fgk)", "", {"abc","fgk"});
     test_f("(abc|fgk)(e|f|zkh|)", "", {"abc","fgk"});
     test_f("abc(abc|fg)xyzz", "xyzz", {"abcabcxyzz","abcfgxyzz"});
+    test_f("((abc|fg)kkk*)xyzz", "xyzz", {"abckk", "fgkk"});
+    test_f("abc(*(abc|fg)*)xyzz", "xyzz");
     test_f("abc[k]xyzz", "xyzz");
     test_f("(abc[k]xyzz)", "xyzz");
-    test_f("abc((de)fg(hi))jk", "abcdefghijk");
-    test_f("abc((?:de)fg(?:hi))jk", "abcdefghijk");
-    test_f("abc((de)fghi+zzz)jk", "abcdefghi");
-    test_f("abc((de)fg(hi))?jk", "abc");
-    test_f("abc((de)fghi?zzz)jk", "abcdefgh");
+    test_f("abc((de)fg(hi))jk", "abcdefghijk", {}, false, true);
+    test_f("abc((?:de)fg(?:hi))jk", "abcdefghijk", {}, false, true);
+    test_f("abc((de)fghi+zzz)jk", "abcdefghi", {}, false, true);
+    test_f("abc((de)fg(hi))?jk", "abc", {}, false, true);
+    test_f("abc((de)fghi?zzz)jk", "abcdefgh", {}, false, true);
     test_f("abc(*cd)jk", "cdjk");
-    test_f(R"(abc(de|xyz|(\{xx\}))fg)", "abc", {"abcdefg", "abcxyzfg", "abc{xx}fg"});
+    test_f(R"(abc(de|xyz|(\{xx\}))fg)", "abc", {"abcdefg", "abcxyzfg", "abc{xx}fg"}, false, true);
     test_f("abc(abc|fg)?xyzz", "xyzz");
     test_f("abc(abc|fg){0,1}xyzz", "xyzz");
     test_f("abc(abc|fg)xyzz|bcdd?k|bc(f|g|h?)z", "", {"abcabcxyzz", "abcfgxyzz", "bcd", "bc"});
@@ -43,4 +46,5 @@ TEST(OptimizeRE, analyze)
     test_f(R"([Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Daumoa(?:-feedfetcher|)|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|))", "", {"pider-", "bingbot", "Yeti-", "Yeti", "Catchpoint bot", "Catchpoint", "harlotte", "Daumoa-feedfetcher", "Daumoa", "-Googlebot", "Googlebot"});
     test_f("abc|(:?xx|yy|zz|x?)def", "", {"abc", "def"});
     test_f("abc|(:?xx|yy|zz|x?){1,2}def", "", {"abc", "def"});
+    test_f(R"(\\A(?:(?:[-0-9_a-z]+(?:\\.[-0-9_a-z]+)*)/k8s1)\\z)", "/k8s1");
 }

From 50a536bba80fac997c856939573532c2adfe38ed Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 16 May 2023 15:26:24 +0200
Subject: [PATCH 573/689] Remove unused code

---
 .../ReplicatedMergeTreePartCheckThread.h      | 24 -------------------
 1 file changed, 24 deletions(-)

diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h
index e7b0e224d9b..b86191dbf50 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h
+++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h
@@ -36,30 +36,6 @@ public:
     void start();
     void stop();
 
-    /// Don't create more than one instance of this object simultaneously.
-    struct TemporarilyStop : private boost::noncopyable
-    {
-        ReplicatedMergeTreePartCheckThread * parent;
-
-        explicit TemporarilyStop(ReplicatedMergeTreePartCheckThread * parent_) : parent(parent_)
-        {
-            parent->stop();
-        }
-
-        TemporarilyStop(TemporarilyStop && old) noexcept : parent(old.parent)
-        {
-            old.parent = nullptr;
-        }
-
-        ~TemporarilyStop()
-        {
-            if (parent)
-                parent->start();
-        }
-    };
-
-    TemporarilyStop temporarilyStop() { return TemporarilyStop(this); }
-
     /// Add a part (for which there are suspicions that it is missing, damaged or not needed) in the queue for check.
     /// delay_to_check_seconds - check no sooner than the specified number of seconds.
     void enqueuePart(const String & name, time_t delay_to_check_seconds = 0);

From 7df0e9d933267d7a2595e7d970c190650e09954c Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 16 May 2023 15:33:08 +0200
Subject: [PATCH 574/689] fix broken link

---
 docs/en/sql-reference/dictionaries/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index 66f661cce60..4abc41cdf42 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -2268,7 +2268,7 @@ In this case, we match the regular expression `\d+/tclwebkit(?:\d+[\.\d]*)` in t
 
 ### Use RegExpTree Dictionary on cloud
 
-We have shown how RegExpTree work in the local environment, but we cannot use `YAMLRegExpTree` in the cloud. If we have a local yaml file, we can use this file to create RegExpTree Dictionary in the local environment, then dump this dictionary to a csv file by the `dictionary` table function and [INTO OUTFILE](../../statements/select/into-outfile.md) clause.
+We have shown how RegExpTree work in the local environment, but we cannot use `YAMLRegExpTree` in the cloud. If we have a local yaml file, we can use this file to create RegExpTree Dictionary in the local environment, then dump this dictionary to a csv file by the `dictionary` table function and [INTO OUTFILE](../statements/select/into-outfile.md) clause.
 
 ```sql
 select * from dictionary(regexp_dict) into outfile('regexp_dict.csv')

From a7bb8f412fdc9e293cd07ffaa71aa29d0ee48a82 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 12 May 2023 11:51:39 +0000
Subject: [PATCH 575/689] Allow ASOF JOIN over nullable right column

---
 src/Interpreters/HashJoin.cpp                 | 37 +++++++-
 src/Interpreters/TableJoin.cpp                |  4 -
 tests/broken_tests.txt                        |  2 -
 .../01428_nullable_asof_join.reference        | 28 ++++++
 .../0_stateless/01428_nullable_asof_join.sql  | 38 ++++++--
 .../02735_asof_join_right_null.reference      | 95 +++++++++++++++++++
 .../02735_asof_join_right_null.sql            | 32 +++++++
 7 files changed, 217 insertions(+), 19 deletions(-)
 create mode 100644 tests/queries/0_stateless/02735_asof_join_right_null.reference
 create mode 100644 tests/queries/0_stateless/02735_asof_join_right_null.sql

diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp
index c9843dca825..c58120c3da9 100644
--- a/src/Interpreters/HashJoin.cpp
+++ b/src/Interpreters/HashJoin.cpp
@@ -710,15 +710,46 @@ Block HashJoin::prepareRightBlock(const Block & block) const
     return prepareRightBlock(block, savedBlockSample());
 }
 
-bool HashJoin::addJoinedBlock(const Block & source_block, bool check_limits)
+bool HashJoin::addJoinedBlock(const Block & source_block_, bool check_limits)
 {
     if (!data)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Join data was released");
 
     /// RowRef::SizeT is uint32_t (not size_t) for hash table Cell memory efficiency.
     /// It's possible to split bigger blocks and insert them by parts here. But it would be a dead code.
-    if (unlikely(source_block.rows() > std::numeric_limits<RowRef::SizeT>::max()))
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Too many rows in right table block for HashJoin: {}", source_block.rows());
+    if (unlikely(source_block_.rows() > std::numeric_limits<RowRef::SizeT>::max()))
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Too many rows in right table block for HashJoin: {}", source_block_.rows());
+
+    Block source_block = source_block_;
+    if (strictness == JoinStrictness::Asof)
+    {
+        chassert(kind == JoinKind::Left || kind == JoinKind::Inner);
+
+        // Filter out rows with NULLs in asof key
+        const auto & asof_key_name = table_join->getOnlyClause().key_names_right.back();
+        auto & asof_column = source_block.getByName(asof_key_name);
+
+        if (asof_column.type->isNullable())
+        {
+            /// filter rows with nulls in asof key
+            if (const auto * asof_const_column = typeid_cast<const ColumnConst *>(asof_column.column.get()))
+            {
+                if (asof_const_column->isNullAt(0))
+                    return false;
+            }
+            else
+            {
+                const auto & asof_column_nullable = assert_cast<const ColumnNullable &>(*asof_column.column).getNullMapData();
+
+                NullMap negative_null_map(asof_column_nullable.size());
+                for (size_t i = 0; i < asof_column_nullable.size(); ++i)
+                    negative_null_map[i] = !asof_column_nullable[i];
+
+                for (auto & column : source_block)
+                    column.column = column.column->filter(negative_null_map, -1);
+            }
+        }
+    }
 
     size_t rows = source_block.rows();
 
diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp
index 2d882083f3d..5a23fbd00ff 100644
--- a/src/Interpreters/TableJoin.cpp
+++ b/src/Interpreters/TableJoin.cpp
@@ -492,10 +492,6 @@ void TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const Rig
     {
         if (clauses.size() != 1)
             throw DB::Exception(ErrorCodes::NOT_IMPLEMENTED, "ASOF join over multiple keys is not supported");
-
-        auto asof_key_type = right_types.find(clauses.back().key_names_right.back());
-        if (asof_key_type != right_types.end() && asof_key_type->second->isNullable())
-            throw DB::Exception(ErrorCodes::NOT_IMPLEMENTED, "ASOF join over right table Nullable column is not implemented");
     }
 
     forAllKeys(clauses, [&](const auto & left_key_name, const auto & right_key_name)
diff --git a/tests/broken_tests.txt b/tests/broken_tests.txt
index 0b4efacba0b..7db123bf467 100644
--- a/tests/broken_tests.txt
+++ b/tests/broken_tests.txt
@@ -37,8 +37,6 @@
 01268_shard_avgweighted
 01270_optimize_skip_unused_shards_low_cardinality
 01319_optimize_skip_unused_shards_nesting
-01353_low_cardinality_join_types
-01428_nullable_asof_join
 01455_shard_leaf_max_rows_bytes_to_read
 01476_right_full_join_switch
 01477_lc_in_merge_join_left_key
diff --git a/tests/queries/0_stateless/01428_nullable_asof_join.reference b/tests/queries/0_stateless/01428_nullable_asof_join.reference
index f04655fefaa..73825dce725 100644
--- a/tests/queries/0_stateless/01428_nullable_asof_join.reference
+++ b/tests/queries/0_stateless/01428_nullable_asof_join.reference
@@ -5,6 +5,15 @@ left asof using
 0	\N	0	\N	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
 1	\N	1	\N	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
 1	1	2	2	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
+0	\N	0	\N	UInt8	Nullable(UInt8)	UInt8	Nullable(UInt8)
+1	1	1	0	UInt8	Nullable(UInt8)	UInt8	Nullable(UInt8)
+1	1	2	0	UInt8	Nullable(UInt8)	UInt8	Nullable(UInt8)
+0	\N	0	\N	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
+1	1	1	0	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
+1	1	2	0	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
+0	\N	0	\N	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
+1	1	1	0	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
+1	1	2	0	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
 left asof on
 0	\N	0	\N	UInt8	Nullable(UInt8)	UInt8	Nullable(UInt8)
 1	\N	1	\N	UInt8	Nullable(UInt8)	UInt8	Nullable(UInt8)
@@ -12,9 +21,28 @@ left asof on
 0	\N	0	\N	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
 1	\N	1	\N	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
 1	1	2	2	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
+0	\N	0	\N	UInt8	Nullable(UInt8)	UInt8	Nullable(UInt8)
+1	1	1	0	UInt8	Nullable(UInt8)	UInt8	Nullable(UInt8)
+1	1	2	0	UInt8	Nullable(UInt8)	UInt8	Nullable(UInt8)
+0	\N	0	\N	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
+1	1	1	0	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
+1	1	2	0	UInt8	Nullable(UInt8)	Nullable(UInt8)	Nullable(UInt8)
 asof using
 1	1	2	2	UInt8	UInt8	UInt8	UInt8
 1	1	2	2	UInt8	UInt8	Nullable(UInt8)	UInt8
+1	1	2	2	UInt8	UInt8	Nullable(UInt8)	Nullable(UInt8)
+1	1	1	0	UInt8	UInt8	UInt8	Nullable(UInt8)
+1	1	2	0	UInt8	UInt8	UInt8	Nullable(UInt8)
+1	1	1	0	UInt8	UInt8	Nullable(UInt8)	Nullable(UInt8)
+1	1	2	0	UInt8	UInt8	Nullable(UInt8)	Nullable(UInt8)
+1	1	1	0	UInt8	UInt8	Nullable(UInt8)	Nullable(UInt8)
+1	1	2	0	UInt8	UInt8	Nullable(UInt8)	Nullable(UInt8)
 asof on
 1	1	2	2	UInt8	UInt8	UInt8	UInt8
 1	1	2	2	UInt8	UInt8	Nullable(UInt8)	UInt8
+1	1	1	0	UInt8	UInt8	UInt8	Nullable(UInt8)
+1	1	2	0	UInt8	UInt8	UInt8	Nullable(UInt8)
+1	1	1	0	UInt8	UInt8	Nullable(UInt8)	Nullable(UInt8)
+1	1	2	0	UInt8	UInt8	Nullable(UInt8)	Nullable(UInt8)
+1	1	1	0	UInt8	UInt8	Nullable(UInt8)	Nullable(UInt8)
+1	1	2	0	UInt8	UInt8	Nullable(UInt8)	Nullable(UInt8)
diff --git a/tests/queries/0_stateless/01428_nullable_asof_join.sql b/tests/queries/0_stateless/01428_nullable_asof_join.sql
index e1b00158d68..f07a26edd97 100644
--- a/tests/queries/0_stateless/01428_nullable_asof_join.sql
+++ b/tests/queries/0_stateless/01428_nullable_asof_join.sql
@@ -18,13 +18,19 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma
 FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
 ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b
 USING(pk, dt)
-ORDER BY a.dt; -- { serverError 48 }
+ORDER BY a.dt SETTINGS allow_experimental_analyzer = 0;
+
+SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
+FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
+ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b
+USING(pk, dt)
+ORDER BY a.dt SETTINGS allow_experimental_analyzer = 1;
 
 SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
 FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
 ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b
 USING(pk, dt)
-ORDER BY a.dt; -- { serverError 48 }
+ORDER BY a.dt;
 
 select 'left asof on';
 
@@ -44,13 +50,13 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma
 FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
 ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b
 ON a.pk = b.pk AND a.dt >= b.dt
-ORDER BY a.dt; -- { serverError 48 }
+ORDER BY a.dt;
 
 SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
 FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
 ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b
 ON a.dt >= b.dt AND a.pk = b.pk
-ORDER BY a.dt; -- { serverError 48 }
+ORDER BY a.dt;
 
 select 'asof using';
 
@@ -64,19 +70,31 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma
 FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
 ASOF JOIN (SELECT 1 as pk, 2 as dt) b
 USING(pk, dt)
-ORDER BY a.dt;
+ORDER BY a.dt SETTINGS allow_experimental_analyzer = 0;
+
+SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
+FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
+ASOF JOIN (SELECT 1 as pk, 2 as dt) b
+USING(pk, dt)
+ORDER BY a.dt SETTINGS allow_experimental_analyzer = 1;
 
 SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
 FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
 ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b
 USING(pk, dt)
-ORDER BY a.dt; -- { serverError 48 }
+ORDER BY a.dt SETTINGS allow_experimental_analyzer = 0;
+
+SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
+FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
+ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b
+USING(pk, dt)
+ORDER BY a.dt SETTINGS allow_experimental_analyzer = 1;
 
 SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
 FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
 ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b
 USING(pk, dt)
-ORDER BY a.dt; -- { serverError 48 }
+ORDER BY a.dt;
 
 select 'asof on';
 
@@ -96,19 +114,19 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma
 FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
 ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b
 ON a.pk = b.pk AND a.dt >= b.dt
-ORDER BY a.dt; -- { serverError 48 }
+ORDER BY a.dt;
 
 SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
 FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
 ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b
 ON a.pk = b.pk AND a.dt >= b.dt
-ORDER BY a.dt; -- { serverError 48 }
+ORDER BY a.dt;
 
 SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
 FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
 ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b
 ON a.dt >= b.dt AND a.pk = b.pk
-ORDER BY a.dt; -- { serverError 48 }
+ORDER BY a.dt;
 
 SELECT *
 FROM (SELECT NULL AS y, 1 AS x, '2020-01-01 10:10:10' :: DateTime64 AS t) AS t1
diff --git a/tests/queries/0_stateless/02735_asof_join_right_null.reference b/tests/queries/0_stateless/02735_asof_join_right_null.reference
new file mode 100644
index 00000000000..d4332556cb5
--- /dev/null
+++ b/tests/queries/0_stateless/02735_asof_join_right_null.reference
@@ -0,0 +1,95 @@
+-- { echoOn }
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b < t2.b ORDER BY t1.b;
+1	-1	1	1
+1	0	1	1
+1	1	1	2
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b <= t2.b ORDER BY t1.b;
+1	-1	1	1
+1	0	1	1
+1	1	1	1
+1	2	1	2
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b > t2.b ORDER BY t1.b;
+1	2	1	1
+1	3	1	2
+1	4	1	2
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b >= t2.b ORDER BY t1.b;
+1	1	1	1
+1	2	1	2
+1	3	1	2
+1	4	1	2
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b < t2.b ORDER BY t1.b;
+1	-1	1	1
+1	0	1	1
+1	1	1	2
+1	2	0	\N
+1	3	0	\N
+1	4	0	\N
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b <= t2.b ORDER BY t1.b;
+1	-1	1	1
+1	0	1	1
+1	1	1	1
+1	2	1	2
+1	3	0	\N
+1	4	0	\N
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b > t2.b ORDER BY t1.b;
+1	-1	0	\N
+1	0	0	\N
+1	1	0	\N
+1	2	1	1
+1	3	1	2
+1	4	1	2
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b >= t2.b ORDER BY t1.b;
+1	-1	0	\N
+1	0	0	\N
+1	1	1	1
+1	2	1	2
+1	3	1	2
+1	4	1	2
+SET join_use_nulls = 1;
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b < t2.b ORDER BY t1.b;
+1	-1	1	1
+1	0	1	1
+1	1	1	2
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b <= t2.b ORDER BY t1.b;
+1	-1	1	1
+1	0	1	1
+1	1	1	1
+1	2	1	2
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b > t2.b ORDER BY t1.b;
+1	2	1	1
+1	3	1	2
+1	4	1	2
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b >= t2.b ORDER BY t1.b;
+1	1	1	1
+1	2	1	2
+1	3	1	2
+1	4	1	2
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b < t2.b ORDER BY t1.b;
+1	-1	1	1
+1	0	1	1
+1	1	1	2
+1	2	\N	\N
+1	3	\N	\N
+1	4	\N	\N
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b <= t2.b ORDER BY t1.b;
+1	-1	1	1
+1	0	1	1
+1	1	1	1
+1	2	1	2
+1	3	\N	\N
+1	4	\N	\N
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b > t2.b ORDER BY t1.b;
+1	-1	\N	\N
+1	0	\N	\N
+1	1	\N	\N
+1	2	1	1
+1	3	1	2
+1	4	1	2
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b >= t2.b ORDER BY t1.b;
+1	-1	\N	\N
+1	0	\N	\N
+1	1	1	1
+1	2	1	2
+1	3	1	2
+1	4	1	2
+DROP TABLE t1;
diff --git a/tests/queries/0_stateless/02735_asof_join_right_null.sql b/tests/queries/0_stateless/02735_asof_join_right_null.sql
new file mode 100644
index 00000000000..997d33a0570
--- /dev/null
+++ b/tests/queries/0_stateless/02735_asof_join_right_null.sql
@@ -0,0 +1,32 @@
+
+CREATE TABLE t1 (a Int, b Int) ENGINE = Memory;
+INSERT INTO t1 VALUES (1, -1), (1, 0), (1, 1), (1, 2), (1, 3), (1, 4);
+
+CREATE TABLE t2 (a Int, b Nullable(Int)) ENGINE = Memory;
+INSERT INTO t2 VALUES (1, 1), (1, NULL), (1, 2);
+
+-- { echoOn }
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b < t2.b ORDER BY t1.b;
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b <= t2.b ORDER BY t1.b;
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b > t2.b ORDER BY t1.b;
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b >= t2.b ORDER BY t1.b;
+
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b < t2.b ORDER BY t1.b;
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b <= t2.b ORDER BY t1.b;
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b > t2.b ORDER BY t1.b;
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b >= t2.b ORDER BY t1.b;
+
+SET join_use_nulls = 1;
+
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b < t2.b ORDER BY t1.b;
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b <= t2.b ORDER BY t1.b;
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b > t2.b ORDER BY t1.b;
+SELECT * FROM t1 ASOF JOIN t2 ON t1.a = t2.a AND t1.b >= t2.b ORDER BY t1.b;
+
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b < t2.b ORDER BY t1.b;
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b <= t2.b ORDER BY t1.b;
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b > t2.b ORDER BY t1.b;
+SELECT * FROM t1 ASOF LEFT JOIN t2 ON t1.a = t2.a AND t1.b >= t2.b ORDER BY t1.b;
+
+DROP TABLE t1;
+

From ca005ecea14635126f0d2d4d835fcaeaab25081c Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Mon, 15 May 2023 16:57:27 +0000
Subject: [PATCH 576/689] Update comment about filtering nulls in asof join

---
 src/Interpreters/HashJoin.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp
index c58120c3da9..0af33a8bd20 100644
--- a/src/Interpreters/HashJoin.cpp
+++ b/src/Interpreters/HashJoin.cpp
@@ -725,7 +725,9 @@ bool HashJoin::addJoinedBlock(const Block & source_block_, bool check_limits)
     {
         chassert(kind == JoinKind::Left || kind == JoinKind::Inner);
 
-        // Filter out rows with NULLs in asof key
+        /// Filter out rows with NULLs in ASOF key, nulls are not joined with anything since they are not comparable
+        /// We support only INNER/LEFT ASOF join, so rows with NULLs never return from the right joined table.
+        /// So filter them out here not to handle in implementation.
         const auto & asof_key_name = table_join->getOnlyClause().key_names_right.back();
         auto & asof_column = source_block.getByName(asof_key_name);
 

From 1f55c320b42267efd2dc94ef024b03bba5b1ba0b Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Mon, 15 May 2023 21:41:45 +0200
Subject: [PATCH 577/689] Fix style

---
 src/Interpreters/HashJoin.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h
index 79fac60d8bc..50eda4482bd 100644
--- a/src/Interpreters/HashJoin.h
+++ b/src/Interpreters/HashJoin.h
@@ -155,7 +155,7 @@ public:
     /** Add block of data from right hand of JOIN to the map.
       * Returns false, if some limit was exceeded and you should not insert more data.
       */
-    bool addJoinedBlock(const Block & block, bool check_limits) override;
+    bool addJoinedBlock(const Block & source_block_, bool check_limits) override;
 
     void checkTypesOfKeys(const Block & block) const override;
 

From 1308c29747be863b951dbe17bb7cc2e4f3e0aaa8 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 16 May 2023 11:42:19 +0200
Subject: [PATCH 578/689] Update tests/broken_tests.txt

---
 tests/broken_tests.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/broken_tests.txt b/tests/broken_tests.txt
index 7db123bf467..d408b45f188 100644
--- a/tests/broken_tests.txt
+++ b/tests/broken_tests.txt
@@ -37,6 +37,7 @@
 01268_shard_avgweighted
 01270_optimize_skip_unused_shards_low_cardinality
 01319_optimize_skip_unused_shards_nesting
+01353_low_cardinality_join_types
 01455_shard_leaf_max_rows_bytes_to_read
 01476_right_full_join_switch
 01477_lc_in_merge_join_left_key

From dea5cbcf4e01af9793ba1beac777c88a7567cc8c Mon Sep 17 00:00:00 2001
From: Igor Nikonov <954088+devcrafter@users.noreply.github.com>
Date: Tue, 16 May 2023 16:39:00 +0200
Subject: [PATCH 579/689] Slightly update comment

---
 src/Common/HashTable/HashTableKeyHolder.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/HashTable/HashTableKeyHolder.h b/src/Common/HashTable/HashTableKeyHolder.h
index d1b38744255..e2f9d80db53 100644
--- a/src/Common/HashTable/HashTableKeyHolder.h
+++ b/src/Common/HashTable/HashTableKeyHolder.h
@@ -88,7 +88,7 @@ inline StringRef & ALWAYS_INLINE keyHolderGetKey(DB::ArenaKeyHolder & holder)
 
 inline void ALWAYS_INLINE keyHolderPersistKey(DB::ArenaKeyHolder & holder)
 {
-    // Normally, our hash table shouldn't ask us to persist a zero key,
+    // Normally, our hash table shouldn't ask to persist a zero key,
     // but it can happened in the case of clearable hash table (ClearableHashSet, for example).
     // The clearable hash table doesn't use zero storage and
     // distinguishes empty keys by using cell version, not the value itself.

From 77adb7c8bc4c680503d2937a8448f097fef1af3d Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Tue, 16 May 2023 17:00:05 +0200
Subject: [PATCH 580/689] Update 02535_analyzer_group_by_use_nulls reference

---
 .../02535_analyzer_group_by_use_nulls.reference        | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference
index 50755627996..63610604ddd 100644
--- a/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference
+++ b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference
@@ -5,15 +5,25 @@ GROUP BY ROLLUP(number, number % 2)
 ORDER BY (number, number % 2, val)
 SETTINGS group_by_use_nulls=1;
 0	0	0
+0	\N	0
 1	1	1
+1	\N	1
 2	0	2
+2	\N	2
 3	1	3
+3	\N	3
 4	0	4
+4	\N	4
 5	1	5
+5	\N	5
 6	0	6
+6	\N	6
 7	1	7
+7	\N	7
 8	0	8
+8	\N	8
 9	1	9
+9	\N	9
 \N	\N	45
 set optimize_group_by_function_keys = 0;
 SELECT number, number % 2, sum(number) AS val

From 4bc5a76fa787e67be8b46fa6dba23f014d783ee8 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio2368@users.noreply.github.com>
Date: Tue, 16 May 2023 17:20:06 +0200
Subject: [PATCH 581/689] Add Compose request for GCS (#49693)

* Add compose request

* Check if outcome is successful

---------

Co-authored-by: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
---
 src/IO/S3/Client.cpp     |  57 +++++++++++++++++++++-
 src/IO/S3/Client.h       |   3 ++
 src/IO/S3/Requests.cpp   | 101 +++++++++++++++++++++++++++++++++++++++
 src/IO/S3/Requests.h     |  36 ++++++++++++++
 src/IO/S3/copyS3File.cpp |  46 ++++++------------
 5 files changed, 211 insertions(+), 32 deletions(-)

diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp
index 3c39893b44e..93ef30a927e 100644
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@@ -2,6 +2,7 @@
 
 #if USE_AWS_S3
 
+#include <aws/core/client/CoreErrors.h>
 #include <aws/core/client/DefaultRetryStrategy.h>
 #include <aws/s3/model/HeadBucketRequest.h>
 #include <aws/s3/model/GetObjectRequest.h>
@@ -10,6 +11,7 @@
 #include <aws/core/client/AWSErrorMarshaller.h>
 #include <aws/core/endpoint/EndpointParameter.h>
 #include <aws/core/utils/HashingUtils.h>
+#include <aws/core/utils/logging/ErrorMacros.h>
 
 #include <IO/S3Common.h>
 #include <IO/S3/Requests.h>
@@ -349,8 +351,32 @@ Model::CreateMultipartUploadOutcome Client::CreateMultipartUpload(const CreateMu
 
 Model::CompleteMultipartUploadOutcome Client::CompleteMultipartUpload(const CompleteMultipartUploadRequest & request) const
 {
-    return doRequest(
+    auto outcome = doRequest(
         request, [this](const Model::CompleteMultipartUploadRequest & req) { return CompleteMultipartUpload(req); });
+
+    if (!outcome.IsSuccess() || provider_type != ProviderType::GCS)
+        return outcome;
+
+    const auto & key = request.GetKey();
+    const auto & bucket = request.GetBucket();
+
+    /// For GCS we will try to compose object at the end, otherwise we cannot do a native copy
+    /// for the object (e.g. for backups)
+    /// We don't care if the compose fails, because the upload was still successful, only the
+    /// performance for copying the object will be affected
+    S3::ComposeObjectRequest compose_req;
+    compose_req.SetBucket(bucket);
+    compose_req.SetKey(key);
+    compose_req.SetComponentNames({key});
+    compose_req.SetContentType("binary/octet-stream");
+    auto compose_outcome = ComposeObject(compose_req);
+
+    if (compose_outcome.IsSuccess())
+        LOG_TRACE(log, "Composing object was successful");
+    else
+        LOG_INFO(log, "Failed to compose object. Message: {}, Key: {}, Bucket: {}", compose_outcome.GetError().GetMessage(), key, bucket);
+
+    return outcome;
 }
 
 Model::CopyObjectOutcome Client::CopyObject(const CopyObjectRequest & request) const
@@ -383,6 +409,35 @@ Model::DeleteObjectsOutcome Client::DeleteObjects(const DeleteObjectsRequest & r
     return doRequest(request, [this](const Model::DeleteObjectsRequest & req) { return DeleteObjects(req); });
 }
 
+Client::ComposeObjectOutcome Client::ComposeObject(const ComposeObjectRequest & request) const
+{
+    auto request_fn = [this](const ComposeObjectRequest & req)
+    {
+        auto & endpoint_provider = const_cast<Client &>(*this).accessEndpointProvider();
+        AWS_OPERATION_CHECK_PTR(endpoint_provider, ComposeObject, Aws::Client::CoreErrors, Aws::Client::CoreErrors::ENDPOINT_RESOLUTION_FAILURE);
+
+        if (!req.BucketHasBeenSet())
+        {
+            AWS_LOGSTREAM_ERROR("ComposeObject", "Required field: Bucket, is not set")
+            return ComposeObjectOutcome(Aws::Client::AWSError<Aws::S3::S3Errors>(Aws::S3::S3Errors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Bucket]", false));
+        }
+
+        if (!req.KeyHasBeenSet())
+        {
+            AWS_LOGSTREAM_ERROR("ComposeObject", "Required field: Key, is not set")
+            return ComposeObjectOutcome(Aws::Client::AWSError<Aws::S3::S3Errors>(Aws::S3::S3Errors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Key]", false));
+        }
+
+        auto endpointResolutionOutcome = endpoint_provider->ResolveEndpoint(req.GetEndpointContextParams());
+        AWS_OPERATION_CHECK_SUCCESS(endpointResolutionOutcome, ComposeObject, Aws::Client::CoreErrors, Aws::Client::CoreErrors::ENDPOINT_RESOLUTION_FAILURE, endpointResolutionOutcome.GetError().GetMessage());
+        endpointResolutionOutcome.GetResult().AddPathSegments(req.GetKey());
+        endpointResolutionOutcome.GetResult().SetQueryString("?compose");
+        return ComposeObjectOutcome(MakeRequest(req, endpointResolutionOutcome.GetResult(), Aws::Http::HttpMethod::HTTP_PUT));
+    };
+
+    return doRequest(request, request_fn);
+}
+
 template <typename RequestType, typename RequestFn>
 std::invoke_result_t<RequestFn, RequestType>
 Client::doRequest(const RequestType & request, RequestFn request_fn) const
diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h
index 330c85c418a..36edb443681 100644
--- a/src/IO/S3/Client.h
+++ b/src/IO/S3/Client.h
@@ -192,6 +192,9 @@ public:
     Model::DeleteObjectOutcome DeleteObject(const DeleteObjectRequest & request) const;
     Model::DeleteObjectsOutcome DeleteObjects(const DeleteObjectsRequest & request) const;
 
+    using ComposeObjectOutcome = Aws::Utils::Outcome<Aws::NoResult, Aws::S3::S3Error>;
+    ComposeObjectOutcome ComposeObject(const ComposeObjectRequest & request) const;
+
     using Aws::S3::S3Client::EnableRequestProcessing;
     using Aws::S3::S3Client::DisableRequestProcessing;
 
diff --git a/src/IO/S3/Requests.cpp b/src/IO/S3/Requests.cpp
index 1972b39d2af..56d2e44a2c4 100644
--- a/src/IO/S3/Requests.cpp
+++ b/src/IO/S3/Requests.cpp
@@ -3,6 +3,8 @@
 #if USE_AWS_S3
 
 #include <Common/logger_useful.h>
+#include <aws/core/endpoint/EndpointParameter.h>
+#include <aws/core/utils/xml/XmlSerializer.h>
 
 namespace DB::S3
 {
@@ -50,6 +52,105 @@ Aws::Http::HeaderValueCollection CopyObjectRequest::GetRequestSpecificHeaders()
     return headers;
 }
 
+Aws::String ComposeObjectRequest::SerializePayload() const
+{
+    if (component_names.empty())
+        return {};
+
+    Aws::Utils::Xml::XmlDocument payload_doc = Aws::Utils::Xml::XmlDocument::CreateWithRootNode("ComposeRequest");
+    auto root_node = payload_doc.GetRootElement();
+
+    for (const auto & name : component_names)
+    {
+        auto component_node = root_node.CreateChildElement("Component");
+        auto name_node = component_node.CreateChildElement("Name");
+        name_node.SetText(name);
+    }
+
+    return payload_doc.ConvertToString();
+}
+
+void ComposeObjectRequest::AddQueryStringParameters(Aws::Http::URI & /*uri*/) const
+{
+}
+
+Aws::Http::HeaderValueCollection ComposeObjectRequest::GetRequestSpecificHeaders() const
+{
+    if (content_type.empty())
+        return {};
+
+    return {Aws::Http::HeaderValuePair(Aws::Http::CONTENT_TYPE_HEADER, content_type)};
+}
+
+Aws::Endpoint::EndpointParameters ComposeObjectRequest::GetEndpointContextParams() const
+{
+    EndpointParameters parameters;
+    if (BucketHasBeenSet())
+        parameters.emplace_back("Bucket", GetBucket(), Aws::Endpoint::EndpointParameter::ParameterOrigin::OPERATION_CONTEXT);
+
+    return parameters;
+}
+
+const Aws::String & ComposeObjectRequest::GetBucket() const
+{
+    return bucket;
+}
+
+bool ComposeObjectRequest::BucketHasBeenSet() const
+{
+    return !bucket.empty();
+}
+
+void ComposeObjectRequest::SetBucket(const Aws::String & value)
+{
+    bucket = value;
+}
+
+void ComposeObjectRequest::SetBucket(Aws::String && value)
+{
+    bucket = std::move(value);
+}
+
+void ComposeObjectRequest::SetBucket(const char * value)
+{
+    bucket.assign(value);
+}
+
+const Aws::String & ComposeObjectRequest::GetKey() const
+{
+    return key;
+}
+
+bool ComposeObjectRequest::KeyHasBeenSet() const
+{
+    return !key.empty();
+}
+
+void ComposeObjectRequest::SetKey(const Aws::String & value)
+{
+    key = value;
+}
+
+void ComposeObjectRequest::SetKey(Aws::String && value)
+{
+    key = std::move(value);
+}
+
+void ComposeObjectRequest::SetKey(const char * value)
+{
+    key.assign(value);
+}
+
+void ComposeObjectRequest::SetComponentNames(std::vector<Aws::String> component_names_)
+{
+    component_names = std::move(component_names_);
+}
+
+void ComposeObjectRequest::SetContentType(Aws::String value)
+{
+    content_type = std::move(value);
+}
+
 }
 
 #endif
diff --git a/src/IO/S3/Requests.h b/src/IO/S3/Requests.h
index 7b6e911c104..560ba9b2775 100644
--- a/src/IO/S3/Requests.h
+++ b/src/IO/S3/Requests.h
@@ -94,6 +94,42 @@ using PutObjectRequest = ExtendedRequest<Model::PutObjectRequest>;
 using DeleteObjectRequest = ExtendedRequest<Model::DeleteObjectRequest>;
 using DeleteObjectsRequest = ExtendedRequest<Model::DeleteObjectsRequest>;
 
+
+class ComposeObjectRequest : public ExtendedRequest<Aws::S3::S3Request>
+{
+public:
+    inline const char * GetServiceRequestName() const override { return "ComposeObject"; }
+
+    AWS_S3_API Aws::String SerializePayload() const override;
+
+    AWS_S3_API void AddQueryStringParameters(Aws::Http::URI & uri) const override;
+
+    AWS_S3_API Aws::Http::HeaderValueCollection GetRequestSpecificHeaders() const override;
+
+    AWS_S3_API EndpointParameters GetEndpointContextParams() const override;
+
+    const Aws::String & GetBucket() const;
+    bool BucketHasBeenSet() const;
+    void SetBucket(const Aws::String & value);
+    void SetBucket(Aws::String && value);
+    void SetBucket(const char* value);
+
+    const Aws::String & GetKey() const;
+    bool KeyHasBeenSet() const;
+    void SetKey(const Aws::String & value);
+    void SetKey(Aws::String && value);
+    void SetKey(const char * value);
+
+    void SetComponentNames(std::vector<Aws::String> component_names_);
+
+    void SetContentType(Aws::String value);
+private:
+    Aws::String bucket;
+    Aws::String key;
+    std::vector<Aws::String> component_names;
+    Aws::String content_type;
+};
+
 }
 
 #endif
diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp
index 3d99a584933..3a2fd513392 100644
--- a/src/IO/S3/copyS3File.cpp
+++ b/src/IO/S3/copyS3File.cpp
@@ -601,29 +601,10 @@ namespace
 
         void performCopy()
         {
-            if (size <= upload_settings.max_single_operation_copy_size)
-            {
+            if (!supports_multipart_copy || size <= upload_settings.max_single_operation_copy_size)
                 performSingleOperationCopy();
-            }
-            else if (!supports_multipart_copy)
-            {
-                LOG_INFO(&Poco::Logger::get("copyS3File"), "Multipart upload using copy is not supported, will use regular upload");
-                copyDataToS3File(
-                    getSourceObjectReadBuffer(),
-                    offset,
-                    size,
-                    client_ptr,
-                    dest_bucket,
-                    dest_key,
-                    request_settings,
-                    object_metadata,
-                    schedule,
-                    for_disk_s3);
-            }
             else
-            {
                 performMultipartUploadCopy();
-            }
 
             if (request_settings.check_objects_after_upload)
                 checkObjectAfterUpload();
@@ -696,19 +677,12 @@ namespace
 
                 if (outcome.GetError().GetExceptionName() == "EntityTooLarge" || outcome.GetError().GetExceptionName() == "InvalidRequest" || outcome.GetError().GetExceptionName() == "InvalidArgument")
                 {
-                    // Can't come here with MinIO, MinIO allows single part upload for large objects.
-                    LOG_INFO(
-                        log,
-                        "Single operation copy failed with error {} for Bucket: {}, Key: {}, Object size: {}, will retry with multipart "
-                        "upload copy",
-                        outcome.GetError().GetExceptionName(),
-                        dest_bucket,
-                        dest_key,
-                        size);
-
                     if (!supports_multipart_copy)
                     {
-                        LOG_INFO(log, "Multipart upload using copy is not supported, will try regular upload");
+                        LOG_INFO(log, "Multipart upload using copy is not supported, will try regular upload for Bucket: {}, Key: {}, Object size: {}",
+                                dest_bucket,
+                                dest_key,
+                                size);
                         copyDataToS3File(
                             getSourceObjectReadBuffer(),
                             offset,
@@ -724,6 +698,16 @@ namespace
                     }
                     else
                     {
+                        // Can't come here with MinIO, MinIO allows single part upload for large objects.
+                        LOG_INFO(
+                            log,
+                            "Single operation copy failed with error {} for Bucket: {}, Key: {}, Object size: {}, will retry with multipart "
+                            "upload copy",
+                            outcome.GetError().GetExceptionName(),
+                            dest_bucket,
+                            dest_key,
+                            size);
+
                         performMultipartUploadCopy();
                         break;
                     }

From 353791b2dd9e7f309f2a4b300c701cd32c1bf9df Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Tue, 16 May 2023 17:26:22 +0200
Subject: [PATCH 582/689] Try to fix flaky test_distributed_load_balancing
 tests (#49912)

* Try to fix flaky test_distributed_load_balancing tests

* Automatic style fix

---------

Co-authored-by: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Co-authored-by: Alexander Tokmakov <tavplubix@clickhouse.com>
---
 .../test_distributed_load_balancing/test.py    | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/tests/integration/test_distributed_load_balancing/test.py b/tests/integration/test_distributed_load_balancing/test.py
index 90771c027dc..1dba6a30bc4 100644
--- a/tests/integration/test_distributed_load_balancing/test.py
+++ b/tests/integration/test_distributed_load_balancing/test.py
@@ -9,9 +9,21 @@ from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
 
-n1 = cluster.add_instance("n1", main_configs=["configs/remote_servers.xml"])
-n2 = cluster.add_instance("n2", main_configs=["configs/remote_servers.xml"])
-n3 = cluster.add_instance("n3", main_configs=["configs/remote_servers.xml"])
+n1 = cluster.add_instance(
+    "n1",
+    main_configs=["configs/remote_servers.xml"],
+    user_configs=["configs/users.xml"],
+)
+n2 = cluster.add_instance(
+    "n2",
+    main_configs=["configs/remote_servers.xml"],
+    user_configs=["configs/users.xml"],
+)
+n3 = cluster.add_instance(
+    "n3",
+    main_configs=["configs/remote_servers.xml"],
+    user_configs=["configs/users.xml"],
+)
 
 nodes = len(cluster.instances)
 queries = nodes * 10

From 724949927b0da0d1150f271f51e0fe11e214e472 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Tue, 16 May 2023 17:36:48 +0200
Subject: [PATCH 583/689] Add logging

---
 src/Interpreters/Cache/FileCache.cpp            | 15 ++++++++++++---
 src/Interpreters/Cache/LRUFileCachePriority.cpp |  7 ++++++-
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index 2d5744b630e..9ab7943e263 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -573,8 +573,6 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size)
     else
         queue_size += 1;
 
-    size_t removed_size = 0;
-
     class EvictionCandidates final : public std::vector<FileSegmentMetadataPtr>
     {
     public:
@@ -600,6 +598,7 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size)
 
     std::unordered_map<Key, EvictionCandidates> to_delete;
 
+    size_t removed_size = 0;
     auto iterate_func = [&](LockedKey & locked_key, FileSegmentMetadataPtr segment_metadata)
     {
         chassert(segment_metadata->file_segment->assertCorrectness());
@@ -655,8 +654,18 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size)
     {
         /// max_size == 0 means unlimited cache size,
         /// max_element_size means unlimited number of cache elements.
-        return (main_priority->getSizeLimit() != 0 && main_priority->getSize(cache_lock) + size - removed_size > main_priority->getSizeLimit())
+        const bool is_overflow = (main_priority->getSizeLimit() != 0
+                                  && main_priority->getSize(cache_lock) + size - removed_size > main_priority->getSizeLimit())
             || (main_priority->getElementsLimit() != 0 && queue_size > main_priority->getElementsLimit());
+
+        LOG_TEST(
+            log, "Overflow: {}, size: {}, ready to remove: {}, current cache size: {}/{}, elements: {}/{}, while reserving for {}:{}",
+            is_overflow, size, removed_size,
+            main_priority->getSize(cache_lock), main_priority->getSizeLimit(),
+            main_priority->getElementsCount(cache_lock), main_priority->getElementsLimit(),
+            file_segment.key(), file_segment.offset());
+
+        return is_overflow;
     };
 
     main_priority->iterate(
diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp
index 5e05acd9680..3c36962a0e5 100644
--- a/src/Interpreters/Cache/LRUFileCachePriority.cpp
+++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp
@@ -44,7 +44,7 @@ IFileCachePriority::Iterator LRUFileCachePriority::add(
         throw Exception(
             ErrorCodes::LOGICAL_ERROR,
             "Not enough space to add {}:{} with size {}: current size: {}/{}",
-            key, offset, size, current_size, getSizeLimit());
+            key, offset, size, current_size, size_limit);
     }
 
     auto iter = queue.insert(queue.end(), Entry(key, offset, size, key_metadata));
@@ -161,6 +161,11 @@ void LRUFileCachePriority::LRUFileCacheIterator::annul()
 
 void LRUFileCachePriority::LRUFileCacheIterator::updateSize(int64_t size)
 {
+    LOG_TEST(
+        cache_priority->log,
+        "Update size with {} in LRU queue for key: {}, offset: {}, previous size: {}",
+        size, queue_iter->key, queue_iter->offset, queue_iter->size);
+
     cache_priority->current_size += size;
     queue_iter->size += size;
 

From 0da82945ac2df6c79d7a47ae628346b3d1fda6e0 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Tue, 16 May 2023 18:18:48 +0200
Subject: [PATCH 584/689] fix

---
 src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp       | 3 ++-
 .../0_stateless/02440_mutations_finalization.reference    | 2 +-
 .../queries/0_stateless/02440_mutations_finalization.sql  | 5 +++--
 .../0_stateless/02441_alter_delete_and_drop_column.sql    | 8 +++++++-
 4 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
index 9f7ae3222a4..e3c9a54023c 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
@@ -1841,7 +1841,8 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands(
     MutationCommands commands;
     for (auto it = begin; it != end; ++it)
     {
-        chassert(mutation_pointer < it->second->entry->znode_name);
+        /// FIXME uncomment this assertion after relesing 23.5 (currently it fails in Upgrade check)
+        /// chassert(mutation_pointer < it->second->entry->znode_name);
         mutation_ids.push_back(it->second->entry->znode_name);
         const auto & commands_from_entry = it->second->entry->commands;
         commands.insert(commands.end(), commands_from_entry.begin(), commands_from_entry.end());
diff --git a/tests/queries/0_stateless/02440_mutations_finalization.reference b/tests/queries/0_stateless/02440_mutations_finalization.reference
index a8b9c2acdce..c4bad0a3806 100644
--- a/tests/queries/0_stateless/02440_mutations_finalization.reference
+++ b/tests/queries/0_stateless/02440_mutations_finalization.reference
@@ -2,4 +2,4 @@
 1
 0000000000	UPDATE n = 2 WHERE n = 1	['all_0_0_0']	0
 2
-0000000000	UPDATE n = 2 WHERE n = 1	[]	1
+0000000000	UPDATE n = 2 WHERE n = 1	[]
diff --git a/tests/queries/0_stateless/02440_mutations_finalization.sql b/tests/queries/0_stateless/02440_mutations_finalization.sql
index 796dcde8e4e..c522d8ab9df 100644
--- a/tests/queries/0_stateless/02440_mutations_finalization.sql
+++ b/tests/queries/0_stateless/02440_mutations_finalization.sql
@@ -6,6 +6,7 @@ system stop merges mut;
 alter table mut update n = 2 where n = 1;
 -- it will create MUTATE_PART entry, but will not execute it
 
+system sync replica mut pull;
 select mutation_id, command, parts_to_do_names, is_done from system.mutations where database=currentDatabase() and table='mut';
 
 -- merges (and mutations) will start again after detach/attach, we need to avoid this somehow...
@@ -26,8 +27,8 @@ select mutation_id, command, parts_to_do_names, is_done from system.mutations wh
 alter table mut modify setting max_number_of_mutations_for_replica=100;
 system sync replica mut;
 
--- and now it should
+-- and now it should (is_done may be 0, but it's okay)
 select * from mut;
-select mutation_id, command, parts_to_do_names, is_done from system.mutations where database=currentDatabase() and table='mut';
+select mutation_id, command, parts_to_do_names from system.mutations where database=currentDatabase() and table='mut';
 
 drop table tmp; -- btw, it will check that mutation can be cancelled between blocks on shutdown
diff --git a/tests/queries/0_stateless/02441_alter_delete_and_drop_column.sql b/tests/queries/0_stateless/02441_alter_delete_and_drop_column.sql
index d274fae1a4f..b9b1b645e8e 100644
--- a/tests/queries/0_stateless/02441_alter_delete_and_drop_column.sql
+++ b/tests/queries/0_stateless/02441_alter_delete_and_drop_column.sql
@@ -5,11 +5,17 @@ insert into mut values (1, 2, 3), (10, 20, 30);
 
 system stop merges mut;
 alter table mut delete where n = 10;
+
+-- a funny way to wait for a MUTATE_PART to be assigned
+select sleepEachRow(2) from url('http://localhost:8123/?param_tries={1..10}&query=' || encodeURLComponent(
+            'select 1 where ''MUTATE_PART'' not in (select type from system.replication_queue where database=''' || currentDatabase() || ''' and table=''mut'')'
+    ), 'LineAsString', 's String') settings max_threads=1 format Null;
+
 alter table mut drop column k settings alter_sync=0;
 system sync replica mut pull;
 
 -- a funny way to wait for ALTER_METADATA to disappear from the replication queue
-select sleepEachRow(1) from url('http://localhost:8123/?param_tries={1..30}&query=' || encodeURLComponent(
+select sleepEachRow(2) from url('http://localhost:8123/?param_tries={1..10}&query=' || encodeURLComponent(
     'select * from system.replication_queue where database=''' || currentDatabase() || ''' and table=''mut'' and type=''ALTER_METADATA'''
     ), 'LineAsString', 's String') settings max_threads=1 format Null;
 

From b12eefc69444c7171ea3c0f2ba4c96bc51deece8 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Tue, 16 May 2023 18:57:04 +0000
Subject: [PATCH 585/689] fix timeout units and log message

---
 src/Common/PoolBase.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/PoolBase.h b/src/Common/PoolBase.h
index a9c595c440c..ba19c3e2150 100644
--- a/src/Common/PoolBase.h
+++ b/src/Common/PoolBase.h
@@ -146,12 +146,12 @@ public:
 
             if (timeout < 0)
             {
-                LOG_INFO(log, "No free connections in pool. Waiting undefinitelly.");
+                LOG_INFO(log, "No free connections in pool. Waiting indefinitely.");
                 available.wait(lock);
             }
             else
             {
-                auto timeout_ms = std::chrono::microseconds(timeout);
+                auto timeout_ms = std::chrono::milliseconds(timeout);
                 LOG_INFO(log, "No free connections in pool. Waiting {} ms.", timeout_ms.count());
                 available.wait_for(lock, timeout_ms);
             }

From 64bf477ff29c6541d5c96c6ea333fb371d40f1c4 Mon Sep 17 00:00:00 2001
From: DanRoscigno <dan@roscigno.com>
Date: Tue, 16 May 2023 14:58:04 -0400
Subject: [PATCH 586/689] add DDL

---
 docs/en/operations/named-collections.md | 152 ++++++++++++++++++++----
 1 file changed, 127 insertions(+), 25 deletions(-)

diff --git a/docs/en/operations/named-collections.md b/docs/en/operations/named-collections.md
index 0189cc0a8e2..b65438b97f3 100644
--- a/docs/en/operations/named-collections.md
+++ b/docs/en/operations/named-collections.md
@@ -2,34 +2,89 @@
 slug: /en/operations/named-collections
 sidebar_position: 69
 sidebar_label: "Named collections"
+title: "Named collections"
 ---
 
-# Storing details for connecting to external sources in configuration files
+Named collections provide a way to store collections of key-value pairs to be 
+used to configure integrations with external sources. You can use named collections with
+dictionaries, tables, table functions, and object storage. 
+
+Named collections can be configured with DDL or in configuration files and are applied
+when ClickHouse starts. They simplify the creation of objects and the hiding of credentials
+from users without administrative access.
+
+## Storing details for connecting to external sources in the system database
+
+### Permissions to create named collections with DDL
+
+To manage named collections with DDL a user must have the `named_control_collection` privilege.  This can be assigned by adding a file to `/etc/clickhouse-server/users.d/`.  The example gives the user `default` both the `access_management` and `named_collection_control` privileges:
+
+```xml title='/etc/clickhouse-server/users.d/user_default.xml'
+<clickhouse>
+  <users>
+    <default>
+      <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex replace=true>
+      <access_management>1</access_management>
+      <!-- highlight-start -->
+      <named_collection_control>1</named_collection_control>
+      <!-- highlight-end -->
+    </default>
+  </users>
+</clickhouse>
+```
+
+:::tip
+In the above example the `passowrd_sha256_hex` value is the hexadecimal representation of the SHA256 hash of the password.  This configuration for the user `default` has the attribute `replace=true` as in the default configuration has a plain text `password` set, and it is not possible to have both plaintext and sha256 hex passwords set for a user. 
+:::
+
+## Storing details for connecting to external sources in configuration files
 
-Details for connecting to external sources (dictionaries, tables, table functions) can be saved
-in configuration files and thus simplify the creation of objects and hide credentials
-from users with only SQL access.
 
 Parameters can be set in XML `<format>CSV</format>` and overridden in SQL `, format = 'TSV'`.
 The parameters in SQL can be overridden using format `key` = `value`: `compression_method = 'gzip'`.
 
-Named collections are stored in the `config.xml` file of the ClickHouse server in the `<named_collections>` section and are applied when ClickHouse starts.
+Named collections are stored in the system table `system.named_collections` or in `config.xml` file of the ClickHouse server in the `<named_collections>` section and are applied when ClickHouse starts.
+
+### DDL example
+
+```sql
+CREATE NAMED COLLECTION name AS
+key_1 = 'value',
+key_2 = 'value2',
+url = 'https://connection.url/'
+```
+
+### XML example
 
-Example of configuration:
 ```xml
 $ cat /etc/clickhouse-server/config.d/named_collections.xml
 <clickhouse>
      <named_collections>
-     ...
+        <name>
+            <key_1>value</key_1>
+            <key_2>value_2</key_2>
+            <url>https://connection.url/</url>
+        </name>
      </named_collections>
 </clickhouse>
 ```
 
-## Named collections for accessing S3.
+## Named collections for accessing S3
 
 The description of parameters see [s3 Table Function](../sql-reference/table-functions/s3.md).
 
-Example of configuration:
+### DDL example
+
+```sql
+CREATE NAMED COLLECTION s3_mydata AS
+access_key_id = 'AKIAIOSFODNN7EXAMPLE',
+secret_access_key = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
+format = 'CSV',
+url = 'https://s3.us-east-1.amazonaws.com/yourbucket/mydata/'
+```
+
+### XML example
+
 ```xml
 <clickhouse>
     <named_collections>
@@ -43,23 +98,23 @@ Example of configuration:
 </clickhouse>
 ```
 
-### Example of using named collections with the s3 function
+### s3() function and S3 Table Named collection examples
+
+Both of the following examples use the same named collection `s3_mydata`:
+
+#### s3() function
 
 ```sql
 INSERT INTO FUNCTION s3(s3_mydata, filename = 'test_file.tsv.gz',
    format = 'TSV', structure = 'number UInt64', compression_method = 'gzip')
 SELECT * FROM numbers(10000);
-
-SELECT count()
-FROM s3(s3_mydata, filename = 'test_file.tsv.gz')
-
-┌─count()─┐
-│   10000 │
-└─────────┘
-1 rows in set. Elapsed: 0.279 sec. Processed 10.00 thousand rows, 90.00 KB (35.78 thousand rows/s., 322.02 KB/s.)
 ```
 
-### Example of using named collections with an S3 table
+:::tip
+The first argument to the `s3()` function above is the name of the collection, `s3_mydata`.  Without named collections, the access key ID, secret, format, and URL would all be passed in every call to the `s3()` function.
+:::
+
+#### S3 table
 
 ```sql
 CREATE TABLE s3_engine_table (number Int64)
@@ -78,7 +133,22 @@ SELECT * FROM s3_engine_table LIMIT 3;
 
 The description of parameters see [mysql](../sql-reference/table-functions/mysql.md).
 
-Example of configuration:
+### DDL example
+
+```sql
+CREATE NAMED COLLECTION mymysql AS
+user = 'myuser',
+password = 'mypass',
+host = '127.0.0.1',
+port = 3306,
+database = 'test'
+connection_pool_size = 8
+on_duplicate_clause = 1
+replace_query = 1
+```
+
+### XML example
+
 ```xml
 <clickhouse>
     <named_collections>
@@ -96,7 +166,11 @@ Example of configuration:
 </clickhouse>
 ```
 
-### Example of using named collections with the mysql function
+### mysql() function, MySQL table, MySQL database, and Dictionary named collection examples
+
+The four following examples use the same named collection `mymysql`:
+
+#### mysql() function
 
 ```sql
 SELECT count() FROM mysql(mymysql, table = 'test');
@@ -105,8 +179,11 @@ SELECT count() FROM mysql(mymysql, table = 'test');
 │       3 │
 └─────────┘
 ```
+:::note
+The named collection does not specify the `table` parameter, so it is specified in the function call as `table = 'test'`.
+:::
 
-### Example of using named collections with an MySQL table
+#### MySQL table
 
 ```sql
 CREATE TABLE mytable(A Int64) ENGINE = MySQL(mymysql, table = 'test', connection_pool_size=3, replace_query=0);
@@ -117,7 +194,11 @@ SELECT count() FROM mytable;
 └─────────┘
 ```
 
-### Example of using named collections with database with engine MySQL
+:::note
+The DDL overrides the named collection setting for connection_pool_size.
+:::
+
+#### MySQL database
 
 ```sql
 CREATE DATABASE mydatabase ENGINE = MySQL(mymysql);
@@ -130,7 +211,7 @@ SHOW TABLES FROM mydatabase;
 └────────┘
 ```
 
-### Example of using named collections with a dictionary with source MySQL
+#### MySQL Dictionary
 
 ```sql
 CREATE DICTIONARY dict (A Int64, B String)
@@ -150,6 +231,17 @@ SELECT dictGet('dict', 'B', 2);
 
 The description of parameters see [postgresql](../sql-reference/table-functions/postgresql.md).
 
+```sql
+CREATE NAMED COLLECTION mypg AS
+user = 'pguser',
+password = 'jw8s0F4',
+host = '127.0.0.1',
+port = 5432,
+database = 'test',
+schema = 'test_schema',
+connection_pool_size = 8
+```
+
 Example of configuration:
 ```xml
 <clickhouse>
@@ -229,12 +321,22 @@ SELECT dictGet('dict', 'b', 2);
 └─────────────────────────┘
 ```
 
-## Named collections for accessing remote ClickHouse database
+## Named collections for accessing a remote ClickHouse database
 
 The description of parameters see [remote](../sql-reference/table-functions/remote.md/#parameters).
 
 Example of configuration:
 
+```sql
+CREATE NAMED COLLECTION remote1 AS
+host = 'remote_host',
+port = 9000,
+database = 'system',
+user = 'foo',
+password = 'secret',
+secure = 1
+```
+
 ```xml
 <clickhouse>
     <named_collections>

From abacf1f99088ee52b46173537a65aab6b0e6306c Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Tue, 16 May 2023 19:14:54 +0000
Subject: [PATCH 587/689] add missing `quota_key` in operator== for connections

---
 src/Client/ConnectionPool.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Client/ConnectionPool.h b/src/Client/ConnectionPool.h
index c3d0955019e..9b1f7dc5b9d 100644
--- a/src/Client/ConnectionPool.h
+++ b/src/Client/ConnectionPool.h
@@ -192,6 +192,7 @@ inline bool operator==(const ConnectionPoolFactory::Key & lhs, const ConnectionP
 {
     return lhs.max_connections == rhs.max_connections && lhs.host == rhs.host && lhs.port == rhs.port
         && lhs.default_database == rhs.default_database && lhs.user == rhs.user && lhs.password == rhs.password
+        && lhs.quota_key == rhs.quota_key
         && lhs.cluster == rhs.cluster && lhs.cluster_secret == rhs.cluster_secret && lhs.client_name == rhs.client_name
         && lhs.compression == rhs.compression && lhs.secure == rhs.secure && lhs.priority == rhs.priority;
 }

From 35f00f72b3bad1025453181f785c20c9945dda36 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 16 May 2023 22:42:43 +0200
Subject: [PATCH 588/689] add functional test

---
 tests/queries/0_stateless/02751_match_constant_needle.reference | 1 +
 tests/queries/0_stateless/02751_match_constant_needle.sql       | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 tests/queries/0_stateless/02751_match_constant_needle.reference
 create mode 100644 tests/queries/0_stateless/02751_match_constant_needle.sql

diff --git a/tests/queries/0_stateless/02751_match_constant_needle.reference b/tests/queries/0_stateless/02751_match_constant_needle.reference
new file mode 100644
index 00000000000..d00491fd7e5
--- /dev/null
+++ b/tests/queries/0_stateless/02751_match_constant_needle.reference
@@ -0,0 +1 @@
+1
diff --git a/tests/queries/0_stateless/02751_match_constant_needle.sql b/tests/queries/0_stateless/02751_match_constant_needle.sql
new file mode 100644
index 00000000000..71bdcc7cb0a
--- /dev/null
+++ b/tests/queries/0_stateless/02751_match_constant_needle.sql
@@ -0,0 +1 @@
+select match('default/k8s1', '\\A(?:(?:[-0-9_a-z]+(?:\\.[-0-9_a-z]+)*)/k8s1)\\z');

From 15cb6276883288ce57966e067bd1efdae428ed54 Mon Sep 17 00:00:00 2001
From: Thom O'Connor <thom@clickhouse.com>
Date: Tue, 16 May 2023 16:35:41 -0600
Subject: [PATCH 589/689] Update postgresql.md

The type cannot be optional, so we need to be clear in the docs that [type1] [type2] cannot be in square brackets, so removed them
---
 docs/en/engines/table-engines/integrations/postgresql.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md
index fbd6d944363..f27d4d48f75 100644
--- a/docs/en/engines/table-engines/integrations/postgresql.md
+++ b/docs/en/engines/table-engines/integrations/postgresql.md
@@ -13,8 +13,8 @@ The PostgreSQL engine allows to perform `SELECT` and `INSERT` queries on data th
 ``` sql
 CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
 (
-    name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1],
-    name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2],
+    name1 type1 [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1],
+    name2 type2 [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2],
     ...
 ) ENGINE = PostgreSQL('host:port', 'database', 'table', 'user', 'password'[, `schema`]);
 ```

From 282297b677d0650b5d268f895d1eaa5c233e3eb1 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Tue, 16 May 2023 23:46:01 +0000
Subject: [PATCH 590/689] binary encoding of IPv6 in protobuf

---
 src/Formats/ProtobufSerializer.cpp | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp
index 8bdef0e7d3f..ee33d2d9991 100644
--- a/src/Formats/ProtobufSerializer.cpp
+++ b/src/Formats/ProtobufSerializer.cpp
@@ -1,4 +1,5 @@
 #include <Formats/ProtobufSerializer.h>
+#include "Common/formatIPv6.h"
 
 #if USE_PROTOBUF
 #   include <Columns/ColumnAggregateFunction.h>
@@ -1852,25 +1853,26 @@ namespace
 
             write_function = [this](IPv6 value)
             {
-                ipToString(value, text_buffer);
+                text_buffer = String(IPV6_BINARY_LENGTH, '\0');
+                memcpy(text_buffer.data(), &value.toUnderType(), IPV6_BINARY_LENGTH);
                 writeStr(text_buffer);
             };
 
             read_function = [this]() -> IPv6
             {
                 readStr(text_buffer);
-                return parse<IPv6>(text_buffer);
+                if (text_buffer.size() != IPV6_BINARY_LENGTH)
+                    throw Exception(ErrorCodes::PROTOBUF_BAD_CAST,
+                                    "Could not convert bytes field {} to IPv6 for inserting into column {} - field size {} is not equal to IPv6 size {}",
+                                    field_descriptor.full_name(), column_name, text_buffer.size(), IPV6_BINARY_LENGTH);
+                IPv6 value;
+                memcpy(&value.toUnderType(), text_buffer.data(), IPV6_BINARY_LENGTH);
+                return value;
             };
 
             default_function = [this]() -> IPv6 { return parse<IPv6>(field_descriptor.default_value_string()); };
         }
 
-        static void ipToString(const IPv6 & ip, String & str)
-        {
-            WriteBufferFromString buf{str};
-            writeText(ip, buf);
-        }
-
         std::function<void(IPv6)> write_function;
         std::function<IPv6()> read_function;
         std::function<IPv6()> default_function;

From 0a44a69dc82bac8f490399b8197c6220d673e1d3 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Wed, 17 May 2023 00:22:13 +0000
Subject: [PATCH 591/689] remove unnecessary header

---
 src/Formats/ProtobufSerializer.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp
index ee33d2d9991..f690800d145 100644
--- a/src/Formats/ProtobufSerializer.cpp
+++ b/src/Formats/ProtobufSerializer.cpp
@@ -1,5 +1,4 @@
 #include <Formats/ProtobufSerializer.h>
-#include "Common/formatIPv6.h"
 
 #if USE_PROTOBUF
 #   include <Columns/ColumnAggregateFunction.h>

From f4ac4c3f9d783a83e86185373d90cdf4bbe6e1ba Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Wed, 17 May 2023 03:14:30 +0200
Subject: [PATCH 592/689] Corrections after review.

---
 src/Backups/BackupFileInfo.cpp | 12 +++++++-----
 src/Backups/BackupIO_Disk.cpp  |  2 ++
 src/Backups/BackupIO_File.cpp  |  4 ++--
 src/Storages/StorageMemory.cpp | 28 ++++++++++++++++------------
 4 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/src/Backups/BackupFileInfo.cpp b/src/Backups/BackupFileInfo.cpp
index 42546d1b1b8..d539ada55c4 100644
--- a/src/Backups/BackupFileInfo.cpp
+++ b/src/Backups/BackupFileInfo.cpp
@@ -36,7 +36,7 @@ namespace
     {
         /// We cannot reuse base backup because our file is smaller
         /// than file stored in previous backup
-        if (new_entry_info.size < base_backup_info.first)
+        if ((new_entry_info.size < base_backup_info.first) || !base_backup_info.first)
             return CheckBackupResult::HasNothing;
 
         if (base_backup_info.first == new_entry_info.size)
@@ -48,7 +48,10 @@ namespace
 
     struct ChecksumsForNewEntry
     {
-        UInt128 full_checksum;
+        /// 0 is the valid checksum of empty data.
+        UInt128 full_checksum = 0;
+
+        /// std::nullopt here means that it's too difficult to calculate a partial checksum so it shouldn't be used.
         std::optional<UInt128> prefix_checksum;
     };
 
@@ -58,8 +61,7 @@ namespace
     {
         ChecksumsForNewEntry res;
         /// The partial checksum should be calculated before the full checksum to enable optimization in BackupEntryWithChecksumCalculation.
-        if (prefix_size > 0)
-            res.prefix_checksum = entry->getPartialChecksum(prefix_size);
+        res.prefix_checksum = entry->getPartialChecksum(prefix_size);
         res.full_checksum = entry->getChecksum();
         return res;
     }
@@ -116,7 +118,7 @@ BackupFileInfo buildFileInfoForBackupEntry(const String & file_name, const Backu
 
     /// We have info about this file in base backup
     /// If file has no checksum -- calculate and fill it.
-    if (base_backup_file_info.has_value())
+    if (base_backup_file_info)
     {
         LOG_TRACE(log, "File {} found in base backup, checking for equality", adjusted_path);
         CheckBackupResult check_base = checkBaseBackupForFile(*base_backup_file_info, info);
diff --git a/src/Backups/BackupIO_Disk.cpp b/src/Backups/BackupIO_Disk.cpp
index 3b1651bb223..1514b4c24c7 100644
--- a/src/Backups/BackupIO_Disk.cpp
+++ b/src/Backups/BackupIO_Disk.cpp
@@ -37,6 +37,7 @@ void BackupReaderDisk::copyFileToDisk(const String & path_in_backup, size_t file
                                       DiskPtr destination_disk, const String & destination_path, WriteMode write_mode)
 {
     /// Use IDisk::copyFile() as a more optimal way to copy a file if it's possible.
+    /// However IDisk::copyFile() can't use throttling for reading, and can't copy an encrypted file or do appending.
     bool has_throttling = disk->isRemote() ? static_cast<bool>(read_settings.remote_throttler) : static_cast<bool>(read_settings.local_throttler);
     if (!has_throttling && (write_mode == WriteMode::Rewrite) && !encrypted_in_backup)
     {
@@ -106,6 +107,7 @@ void BackupWriterDisk::copyFileFromDisk(const String & path_in_backup, DiskPtr s
                                         bool copy_encrypted, UInt64 start_pos, UInt64 length)
 {
     /// Use IDisk::copyFile() as a more optimal way to copy a file if it's possible.
+    /// However IDisk::copyFile() can't use throttling for reading, and can't copy an encrypted file or copy a part of the file.
     bool has_throttling = src_disk->isRemote() ? static_cast<bool>(read_settings.remote_throttler) : static_cast<bool>(read_settings.local_throttler);
     if (!has_throttling && !start_pos && !copy_encrypted)
     {
diff --git a/src/Backups/BackupIO_File.cpp b/src/Backups/BackupIO_File.cpp
index 7ffae26d16f..e1a3f336521 100644
--- a/src/Backups/BackupIO_File.cpp
+++ b/src/Backups/BackupIO_File.cpp
@@ -41,7 +41,7 @@ std::unique_ptr<SeekableReadBuffer> BackupReaderFile::readFile(const String & fi
 void BackupReaderFile::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
                                       DiskPtr destination_disk, const String & destination_path, WriteMode write_mode)
 {
-    /// std::filesystem::copy() can copy from the filesystem only, and it can't do the throttling or appending.
+    /// std::filesystem::copy() can copy from the filesystem only, and can't do throttling or appending.
     bool has_throttling = static_cast<bool>(read_settings.local_throttler);
     if (!has_throttling && (write_mode == WriteMode::Rewrite))
     {
@@ -121,7 +121,7 @@ void BackupWriterFile::removeFiles(const Strings & file_names)
 void BackupWriterFile::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
                                         bool copy_encrypted, UInt64 start_pos, UInt64 length)
 {
-    /// std::filesystem::copy() can copy from the filesystem only, and it can't do the throttling or copy a part of the file.
+    /// std::filesystem::copy() can copy from the filesystem only, and can't do throttling or copy a part of the file.
     bool has_throttling = static_cast<bool>(read_settings.local_throttler);
     if (!has_throttling)
     {
diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp
index c9654cfd105..31e45db55cb 100644
--- a/src/Storages/StorageMemory.cpp
+++ b/src/Storages/StorageMemory.cpp
@@ -319,24 +319,28 @@ namespace
             IndexForNativeFormat index;
             {
                 auto data_file_path = temp_dir / fs::path{file_paths[data_bin_pos]}.filename();
-                {
-                    auto data_out_compressed = temp_disk->writeFile(data_file_path);
-                    CompressedWriteBuffer data_out{*data_out_compressed, CompressionCodecFactory::instance().getDefaultCodec(), max_compress_block_size};
-                    NativeWriter block_out{data_out, 0, metadata_snapshot->getSampleBlock(), false, &index};
-                    for (const auto & block : *blocks)
-                        block_out.write(block);
-                }
+                auto data_out_compressed = temp_disk->writeFile(data_file_path);
+                auto data_out = std::make_unique<CompressedWriteBuffer>(*data_out_compressed, CompressionCodecFactory::instance().getDefaultCodec(), max_compress_block_size);
+                NativeWriter block_out{*data_out, 0, metadata_snapshot->getSampleBlock(), false, &index};
+                for (const auto & block : *blocks)
+                    block_out.write(block);
+                data_out->finalize();
+                data_out.reset();
+                data_out_compressed->finalize();
+                data_out_compressed.reset();
                 backup_entries[data_bin_pos] = {file_paths[data_bin_pos], std::make_shared<BackupEntryFromAppendOnlyFile>(temp_disk, data_file_path)};
             }
 
             /// Writing index.mrk
             {
                 auto index_mrk_path = temp_dir / fs::path{file_paths[index_mrk_pos]}.filename();
-                {
-                    auto index_mrk_out_compressed = temp_disk->writeFile(index_mrk_path);
-                    CompressedWriteBuffer index_mrk_out{*index_mrk_out_compressed};
-                    index.write(index_mrk_out);
-                }
+                auto index_mrk_out_compressed = temp_disk->writeFile(index_mrk_path);
+                auto index_mrk_out = std::make_unique<CompressedWriteBuffer>(*index_mrk_out_compressed);
+                index.write(*index_mrk_out);
+                index_mrk_out->finalize();
+                index_mrk_out.reset();
+                index_mrk_out_compressed->finalize();
+                index_mrk_out_compressed.reset();
                 backup_entries[index_mrk_pos] = {file_paths[index_mrk_pos], std::make_shared<BackupEntryFromAppendOnlyFile>(temp_disk, index_mrk_path)};
             }
 

From c10435489487626448b8e861074ef9b41ecb9f4e Mon Sep 17 00:00:00 2001
From: fhbai <fhbai@tencent.com>
Date: Wed, 17 May 2023 14:39:30 +0800
Subject: [PATCH 593/689] fix

---
 src/Functions/array/arrayDotProduct.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp
index e3c80775f1b..d17c223cc2f 100644
--- a/src/Functions/array/arrayDotProduct.cpp
+++ b/src/Functions/array/arrayDotProduct.cpp
@@ -57,7 +57,7 @@ public:
     }
 
     template <typename ResultType, typename T, typename U>
-    static ResultType apply(
+    static inline NO_SANITIZE_UNDEFINED ResultType apply(
         const T * left,
         const U * right,
         size_t size)

From 3c80e30f02f40acd372942506d8d57ec803ef2b6 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 11 May 2023 16:59:19 +0200
Subject: [PATCH 594/689] Fix per-query IO/BACKUPs throttling settings (when
 default profile has them)

When some of this settings was set for default profile (in
users.xml/users.yml), then it will be always used regardless of what
user passed.

Fix this by not inherit per-query throttlers, for this they should be
reset before making query context and they should not be initialized as
before in Context::makeQueryContext(), since makeQueryContext() called
too early, when user settings was not read yet.

But there we had also initialization of per-server throttling, move this
into the ContextSharedPart::configureServerWideThrottling(), and call it
once we have ServerSettings set.

Also note, that this patch makes the following settings - server
settings:
- max_replicated_fetches_network_bandwidth_for_server
- max_replicated_sends_network_bandwidth_for_server
But this change should not affect anybody, since it is done with
compatiblity (i.e. if this setting is set in users profile it will be
read from it as well as a fallback).

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Core/ServerSettings.h                     |   2 +
 src/Core/Settings.h                           |   4 +-
 src/Interpreters/Context.cpp                  | 150 +++++++-----------
 .../configs/limit_replication_config.xml      |   8 +-
 .../test_replicated_fetches_bandwidth/test.py |   2 +-
 5 files changed, 65 insertions(+), 101 deletions(-)

diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h
index ee3482414af..2a73930836a 100644
--- a/src/Core/ServerSettings.h
+++ b/src/Core/ServerSettings.h
@@ -23,6 +23,8 @@ namespace DB
     M(UInt64, io_thread_pool_queue_size, 10000, "Queue size for IO thread pool.", 0) \
     M(UInt64, max_outdated_parts_loading_thread_pool_size, 32, "The maximum number of threads that would be used for loading outdated data parts on startup", 0) \
     M(UInt64, outdated_part_loading_thread_pool_queue_size, 10000, "Queue size for parts loading thread pool.", 0) \
+    M(UInt64, max_replicated_fetches_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated fetches. Zero means unlimited.", 0) \
+    M(UInt64, max_replicated_sends_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated sends. Zero means unlimited.", 0) \
     M(UInt64, max_remote_read_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for read. Zero means unlimited.", 0) \
     M(UInt64, max_remote_write_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for write. Zero means unlimited.", 0) \
     M(UInt64, max_local_read_bandwidth_for_server, 0, "The maximum speed of local reads in bytes per second. Zero means unlimited.", 0) \
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 610c7135a75..874e31b8d37 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -101,8 +101,6 @@ class IColumn;
     M(Bool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \
     M(Bool, use_uncompressed_cache, false, "Whether to use the cache of uncompressed blocks.", 0) \
     M(Bool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \
-    M(UInt64, max_replicated_fetches_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated fetches. Zero means unlimited. Only has meaning at server startup.", 0) \
-    M(UInt64, max_replicated_sends_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated sends. Zero means unlimited. Only has meaning at server startup.", 0) \
     M(UInt64, max_remote_read_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for read.", 0) \
     M(UInt64, max_remote_write_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for write.", 0) \
     M(UInt64, max_local_read_bandwidth, 0, "The maximum speed of local reads in bytes per second.", 0) \
@@ -791,6 +789,8 @@ class IColumn;
     MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_distributed_schedule_pool_size, 16) \
     MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_remote_read_network_bandwidth_for_server, 0) \
     MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_remote_write_network_bandwidth_for_server, 0) \
+    MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_replicated_fetches_network_bandwidth_for_server, 0) \
+    MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_replicated_sends_network_bandwidth_for_server, 0) \
     /* ---- */ \
     MAKE_OBSOLETE(M, DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Atomic) \
     MAKE_OBSOLETE(M, UInt64, max_pipeline_depth, 0)                                                                                 \
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index b4bdb7cf233..0ef9ea53ee8 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -631,6 +631,30 @@ struct ContextSharedPart : boost::noncopyable
         log->warning(message);
         warnings.push_back(message);
     }
+
+    void configureServerWideThrottling()
+    {
+        if (auto bandwidth = server_settings.max_replicated_fetches_network_bandwidth_for_server)
+            replicated_fetches_throttler = std::make_shared<Throttler>(bandwidth);
+
+        if (auto bandwidth = server_settings.max_replicated_sends_network_bandwidth_for_server)
+            replicated_sends_throttler = std::make_shared<Throttler>(bandwidth);
+
+        if (auto bandwidth = server_settings.max_remote_read_network_bandwidth_for_server)
+            remote_read_throttler = std::make_shared<Throttler>(bandwidth);
+
+        if (auto bandwidth = server_settings.max_remote_write_network_bandwidth_for_server)
+            remote_write_throttler = std::make_shared<Throttler>(bandwidth);
+
+        if (auto bandwidth = server_settings.max_local_read_bandwidth_for_server)
+            local_read_throttler = std::make_shared<Throttler>(bandwidth);
+
+        if (auto bandwidth = server_settings.max_local_write_bandwidth_for_server)
+            local_write_throttler = std::make_shared<Throttler>(bandwidth);
+
+        if (auto bandwidth = server_settings.max_backup_bandwidth_for_server)
+            backups_server_throttler = std::make_shared<Throttler>(bandwidth);
+    }
 };
 
 
@@ -1897,16 +1921,22 @@ void Context::makeQueryContext()
 {
     query_context = shared_from_this();
 
-    /// Create throttlers, to inherit the ThrottlePtr in the context copies.
-    {
-        getRemoteReadThrottler();
-        getRemoteWriteThrottler();
-
-        getLocalReadThrottler();
-        getLocalWriteThrottler();
-
-        getBackupsThrottler();
-    }
+    /// Throttling should not be inherited, otherwise if you will set
+    /// throttling for default profile you will not able to overwrite it
+    /// per-user/query.
+    ///
+    /// Note, that if you need to set it server-wide, you should use
+    /// per-server settings, i.e.:
+    /// - max_backup_bandwidth_for_server
+    /// - max_remote_read_network_bandwidth_for_server
+    /// - max_remote_write_network_bandwidth_for_server
+    /// - max_local_read_bandwidth_for_server
+    /// - max_local_write_bandwidth_for_server
+    remote_read_query_throttler.reset();
+    remote_write_query_throttler.reset();
+    local_read_query_throttler.reset();
+    local_write_query_throttler.reset();
+    backups_query_throttler.reset();
 }
 
 void Context::makeSessionContext()
@@ -2438,143 +2468,76 @@ BackgroundSchedulePool & Context::getMessageBrokerSchedulePool() const
 
 ThrottlerPtr Context::getReplicatedFetchesThrottler() const
 {
-    auto lock = getLock();
-    if (!shared->replicated_fetches_throttler)
-        shared->replicated_fetches_throttler = std::make_shared<Throttler>(
-            settings.max_replicated_fetches_network_bandwidth_for_server);
-
     return shared->replicated_fetches_throttler;
 }
 
 ThrottlerPtr Context::getReplicatedSendsThrottler() const
 {
-    auto lock = getLock();
-    if (!shared->replicated_sends_throttler)
-        shared->replicated_sends_throttler = std::make_shared<Throttler>(
-            settings.max_replicated_sends_network_bandwidth_for_server);
-
     return shared->replicated_sends_throttler;
 }
 
 ThrottlerPtr Context::getRemoteReadThrottler() const
 {
-    ThrottlerPtr throttler;
-
-    const auto & query_settings = getSettingsRef();
-    UInt64 bandwidth_for_server = shared->server_settings.max_remote_read_network_bandwidth_for_server;
-    if (bandwidth_for_server)
-    {
-        auto lock = getLock();
-        if (!shared->remote_read_throttler)
-            shared->remote_read_throttler = std::make_shared<Throttler>(bandwidth_for_server);
-        throttler = shared->remote_read_throttler;
-    }
-
-    if (query_settings.max_remote_read_network_bandwidth)
+    ThrottlerPtr throttler = shared->remote_read_throttler;
+    if (auto bandwidth = getSettingsRef().max_remote_read_network_bandwidth)
     {
         auto lock = getLock();
         if (!remote_read_query_throttler)
-            remote_read_query_throttler = std::make_shared<Throttler>(query_settings.max_remote_read_network_bandwidth, throttler);
+            remote_read_query_throttler = std::make_shared<Throttler>(bandwidth, throttler);
         throttler = remote_read_query_throttler;
     }
-
     return throttler;
 }
 
 ThrottlerPtr Context::getRemoteWriteThrottler() const
 {
-    ThrottlerPtr throttler;
-
-    const auto & query_settings = getSettingsRef();
-    UInt64 bandwidth_for_server = shared->server_settings.max_remote_write_network_bandwidth_for_server;
-    if (bandwidth_for_server)
-    {
-        auto lock = getLock();
-        if (!shared->remote_write_throttler)
-            shared->remote_write_throttler = std::make_shared<Throttler>(bandwidth_for_server);
-        throttler = shared->remote_write_throttler;
-    }
-
-    if (query_settings.max_remote_write_network_bandwidth)
+    ThrottlerPtr throttler = shared->remote_write_throttler;
+    if (auto bandwidth = getSettingsRef().max_remote_write_network_bandwidth)
     {
         auto lock = getLock();
         if (!remote_write_query_throttler)
-            remote_write_query_throttler = std::make_shared<Throttler>(query_settings.max_remote_write_network_bandwidth, throttler);
+            remote_write_query_throttler = std::make_shared<Throttler>(bandwidth, throttler);
         throttler = remote_write_query_throttler;
     }
-
     return throttler;
 }
 
 ThrottlerPtr Context::getLocalReadThrottler() const
 {
-    ThrottlerPtr throttler;
-
-    if (shared->server_settings.max_local_read_bandwidth_for_server)
-    {
-        auto lock = getLock();
-        if (!shared->local_read_throttler)
-            shared->local_read_throttler = std::make_shared<Throttler>(shared->server_settings.max_local_read_bandwidth_for_server);
-        throttler = shared->local_read_throttler;
-    }
-
-    const auto & query_settings = getSettingsRef();
-    if (query_settings.max_local_read_bandwidth)
+    ThrottlerPtr throttler = shared->local_read_throttler;
+    if (auto bandwidth = getSettingsRef().max_local_read_bandwidth)
     {
         auto lock = getLock();
         if (!local_read_query_throttler)
-            local_read_query_throttler = std::make_shared<Throttler>(query_settings.max_local_read_bandwidth, throttler);
+            local_read_query_throttler = std::make_shared<Throttler>(bandwidth, throttler);
         throttler = local_read_query_throttler;
     }
-
     return throttler;
 }
 
 ThrottlerPtr Context::getLocalWriteThrottler() const
 {
-    ThrottlerPtr throttler;
-
-    if (shared->server_settings.max_local_write_bandwidth_for_server)
-    {
-        auto lock = getLock();
-        if (!shared->local_write_throttler)
-            shared->local_write_throttler = std::make_shared<Throttler>(shared->server_settings.max_local_write_bandwidth_for_server);
-        throttler = shared->local_write_throttler;
-    }
-
-    const auto & query_settings = getSettingsRef();
-    if (query_settings.max_local_write_bandwidth)
+    ThrottlerPtr throttler = shared->local_write_throttler;
+    if (auto bandwidth = getSettingsRef().max_local_write_bandwidth)
     {
         auto lock = getLock();
         if (!local_write_query_throttler)
-            local_write_query_throttler = std::make_shared<Throttler>(query_settings.max_local_write_bandwidth, throttler);
+            local_write_query_throttler = std::make_shared<Throttler>(bandwidth, throttler);
         throttler = local_write_query_throttler;
     }
-
     return throttler;
 }
 
 ThrottlerPtr Context::getBackupsThrottler() const
 {
-    ThrottlerPtr throttler;
-
-    if (shared->server_settings.max_backup_bandwidth_for_server)
-    {
-        auto lock = getLock();
-        if (!shared->backups_server_throttler)
-            shared->backups_server_throttler = std::make_shared<Throttler>(shared->server_settings.max_backup_bandwidth_for_server);
-        throttler = shared->backups_server_throttler;
-    }
-
-    const auto & query_settings = getSettingsRef();
-    if (query_settings.max_backup_bandwidth)
+    ThrottlerPtr throttler = shared->backups_server_throttler;
+    if (auto bandwidth = getSettingsRef().max_backup_bandwidth)
     {
         auto lock = getLock();
         if (!backups_query_throttler)
-            backups_query_throttler = std::make_shared<Throttler>(query_settings.max_backup_bandwidth, throttler);
+            backups_query_throttler = std::make_shared<Throttler>(bandwidth, throttler);
         throttler = backups_query_throttler;
     }
-
     return throttler;
 }
 
@@ -3633,7 +3596,10 @@ void Context::setApplicationType(ApplicationType type)
     shared->application_type = type;
 
     if (type == ApplicationType::SERVER)
+    {
         shared->server_settings.loadSettingsFromConfig(Poco::Util::Application::instance().config());
+        shared->configureServerWideThrottling();
+    }
 }
 
 void Context::setDefaultProfiles(const Poco::Util::AbstractConfiguration & config)
diff --git a/tests/integration/test_replicated_fetches_bandwidth/configs/limit_replication_config.xml b/tests/integration/test_replicated_fetches_bandwidth/configs/limit_replication_config.xml
index b18f0fbc93f..2a7b47e3560 100644
--- a/tests/integration/test_replicated_fetches_bandwidth/configs/limit_replication_config.xml
+++ b/tests/integration/test_replicated_fetches_bandwidth/configs/limit_replication_config.xml
@@ -1,8 +1,4 @@
 <clickhouse>
-    <profiles>
-        <default>
-          <max_replicated_fetches_network_bandwidth_for_server>5242880</max_replicated_fetches_network_bandwidth_for_server>
-          <max_replicated_sends_network_bandwidth_for_server>10485760</max_replicated_sends_network_bandwidth_for_server>
-        </default>
-    </profiles>
+    <max_replicated_fetches_network_bandwidth_for_server>5242880</max_replicated_fetches_network_bandwidth_for_server>
+    <max_replicated_sends_network_bandwidth_for_server>10485760</max_replicated_sends_network_bandwidth_for_server>
 </clickhouse>
diff --git a/tests/integration/test_replicated_fetches_bandwidth/test.py b/tests/integration/test_replicated_fetches_bandwidth/test.py
index 059102f8683..cd969746c31 100644
--- a/tests/integration/test_replicated_fetches_bandwidth/test.py
+++ b/tests/integration/test_replicated_fetches_bandwidth/test.py
@@ -12,7 +12,7 @@ cluster = ClickHouseCluster(__file__)
 node1 = cluster.add_instance("node1", with_zookeeper=True)
 node2 = cluster.add_instance("node2", with_zookeeper=True)
 node3 = cluster.add_instance(
-    "node3", user_configs=["configs/limit_replication_config.xml"], with_zookeeper=True
+    "node3", main_configs=["configs/limit_replication_config.xml"], with_zookeeper=True
 )
 
 

From 7383da0c526399963ff3496c22b6a59f062ff98f Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 11 May 2023 21:43:43 +0200
Subject: [PATCH 595/689] Fix per-query remote throttler

remote throttler by some reason had been overwritten by the global one
during reloads, likely this is for graceful reload of this option, but
it breaks per-query throttling, remove this logic.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../AzureBlobStorage/AzureObjectStorage.cpp           |  1 -
 src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp   |  5 -----
 src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h     |  5 -----
 src/Disks/ObjectStorages/IObjectStorage.cpp           | 11 -----------
 src/Disks/ObjectStorages/IObjectStorage.h             | 11 ++++-------
 src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp       |  1 -
 6 files changed, 4 insertions(+), 30 deletions(-)

diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
index 9db5d13a7f8..62c3216ad3f 100644
--- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
@@ -282,7 +282,6 @@ void AzureObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguratio
 {
     auto new_settings = getAzureBlobStorageSettings(config, config_prefix, context);
     settings.set(std::move(new_settings));
-    applyRemoteThrottlingSettings(context);
     /// We don't update client
 }
 
diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
index e50e410823d..38c088ab213 100644
--- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
@@ -156,11 +156,6 @@ void HDFSObjectStorage::copyObject( /// NOLINT
 }
 
 
-void HDFSObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr context)
-{
-    applyRemoteThrottlingSettings(context);
-}
-
 std::unique_ptr<IObjectStorage> HDFSObjectStorage::cloneObjectStorage(const std::string &, const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr)
 {
     throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "HDFS object storage doesn't support cloning");
diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h
index 4064a5c5b7f..fdc47ad16a6 100644
--- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h
+++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h
@@ -105,11 +105,6 @@ public:
 
     void startup() override;
 
-    void applyNewSettings(
-        const Poco::Util::AbstractConfiguration & config,
-        const std::string & config_prefix,
-        ContextPtr context) override;
-
     String getObjectsNamespace() const override { return ""; }
 
     std::unique_ptr<IObjectStorage> cloneObjectStorage(
diff --git a/src/Disks/ObjectStorages/IObjectStorage.cpp b/src/Disks/ObjectStorages/IObjectStorage.cpp
index 52e8b1a465d..1ee55a7b342 100644
--- a/src/Disks/ObjectStorages/IObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/IObjectStorage.cpp
@@ -55,27 +55,16 @@ const std::string & IObjectStorage::getCacheName() const
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getCacheName() is not implemented for object storage");
 }
 
-void IObjectStorage::applyRemoteThrottlingSettings(ContextPtr context)
-{
-    std::unique_lock lock{throttlers_mutex};
-    remote_read_throttler = context->getRemoteReadThrottler();
-    remote_write_throttler = context->getRemoteWriteThrottler();
-}
-
 ReadSettings IObjectStorage::patchSettings(const ReadSettings & read_settings) const
 {
-    std::unique_lock lock{throttlers_mutex};
     ReadSettings settings{read_settings};
-    settings.remote_throttler = remote_read_throttler;
     settings.for_object_storage = true;
     return settings;
 }
 
 WriteSettings IObjectStorage::patchSettings(const WriteSettings & write_settings) const
 {
-    std::unique_lock lock{throttlers_mutex};
     WriteSettings settings{write_settings};
-    settings.remote_throttler = remote_write_throttler;
     settings.for_object_storage = true;
     return settings;
 }
diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h
index f4f1b063ade..58a31fdc8c3 100644
--- a/src/Disks/ObjectStorages/IObjectStorage.h
+++ b/src/Disks/ObjectStorages/IObjectStorage.h
@@ -165,9 +165,10 @@ public:
 
     /// Apply new settings, in most cases reiniatilize client and some other staff
     virtual void applyNewSettings(
-        const Poco::Util::AbstractConfiguration & config,
-        const std::string & config_prefix,
-        ContextPtr context) = 0;
+        const Poco::Util::AbstractConfiguration &,
+        const std::string & /*config_prefix*/,
+        ContextPtr)
+    {}
 
     /// Sometimes object storages have something similar to chroot or namespace, for example
     /// buckets in S3. If object storage doesn't have any namepaces return empty string.
@@ -205,10 +206,6 @@ public:
 
     virtual WriteSettings patchSettings(const WriteSettings & write_settings) const;
 
-protected:
-    /// Should be called from implementation of applyNewSettings()
-    void applyRemoteThrottlingSettings(ContextPtr context);
-
 private:
     mutable std::mutex throttlers_mutex;
     ThrottlerPtr remote_read_throttler;
diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
index 79b3d3a2b8b..9faae3a6c62 100644
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
@@ -443,7 +443,6 @@ void S3ObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &
     auto new_client = getClient(config, config_prefix, context, *new_s3_settings);
     s3_settings.set(std::move(new_s3_settings));
     client.set(std::move(new_client));
-    applyRemoteThrottlingSettings(context);
 }
 
 std::unique_ptr<IObjectStorage> S3ObjectStorage::cloneObjectStorage(

From fdfb1eda55b0c2ee95dc7284629f7ded6f4c9196 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 11 May 2023 22:15:05 +0200
Subject: [PATCH 596/689] Fix {Local,Remote}ReadThrottlerSleepMicroseconds
 metric values

And also update the test, since now you could have slightly less sleep
intervals, if query spend some time in other places.

But what is important is that query_duration_ms does not exceeded
calculated delay.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Common/Throttler.cpp                         | 16 ++++++++--------
 src/Common/Throttler.h                           |  8 ++++----
 .../02703_max_local_read_bandwidth.sh            |  2 +-
 .../02703_max_local_write_bandwidth.sh           |  2 +-
 4 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/Common/Throttler.cpp b/src/Common/Throttler.cpp
index 4f99c24cc8d..4c1320db27a 100644
--- a/src/Common/Throttler.cpp
+++ b/src/Common/Throttler.cpp
@@ -61,20 +61,20 @@ UInt64 Throttler::add(size_t amount)
         throw Exception::createDeprecated(limit_exceeded_exception_message + std::string(" Maximum: ") + toString(limit), ErrorCodes::LIMIT_EXCEEDED);
 
     /// Wait unless there is positive amount of tokens - throttling
-    Int64 sleep_time = 0;
+    Int64 sleep_time_ns = 0;
     if (max_speed && tokens_value < 0)
     {
-        sleep_time = static_cast<Int64>(-tokens_value / max_speed * NS);
-        accumulated_sleep += sleep_time;
-        sleepForNanoseconds(sleep_time);
-        accumulated_sleep -= sleep_time;
-        ProfileEvents::increment(ProfileEvents::ThrottlerSleepMicroseconds, sleep_time / 1000UL);
+        sleep_time_ns = static_cast<Int64>(-tokens_value / max_speed * NS);
+        accumulated_sleep += sleep_time_ns;
+        sleepForNanoseconds(sleep_time_ns);
+        accumulated_sleep -= sleep_time_ns;
+        ProfileEvents::increment(ProfileEvents::ThrottlerSleepMicroseconds, sleep_time_ns / 1000UL);
     }
 
     if (parent)
-        sleep_time += parent->add(amount);
+        sleep_time_ns += parent->add(amount);
 
-    return static_cast<UInt64>(sleep_time);
+    return static_cast<UInt64>(sleep_time_ns);
 }
 
 void Throttler::reset()
diff --git a/src/Common/Throttler.h b/src/Common/Throttler.h
index 4b117ae7637..7508065096b 100644
--- a/src/Common/Throttler.h
+++ b/src/Common/Throttler.h
@@ -34,15 +34,15 @@ public:
               const std::shared_ptr<Throttler> & parent_ = nullptr);
 
     /// Use `amount` tokens, sleeps if required or throws exception on limit overflow.
-    /// Returns duration of sleep in microseconds (to distinguish sleeping on different kinds of throttlers for metrics)
+    /// Returns duration of sleep in nanoseconds (to distinguish sleeping on different kinds of throttlers for metrics)
     UInt64 add(size_t amount);
 
     UInt64 add(size_t amount, ProfileEvents::Event event_amount, ProfileEvents::Event event_sleep_us)
     {
-        UInt64 sleep_us = add(amount);
+        UInt64 sleep_ns = add(amount);
         ProfileEvents::increment(event_amount, amount);
-        ProfileEvents::increment(event_sleep_us, sleep_us);
-        return sleep_us;
+        ProfileEvents::increment(event_sleep_us, sleep_ns / 1000UL);
+        return sleep_ns;
     }
 
     /// Not thread safe
diff --git a/tests/queries/0_stateless/02703_max_local_read_bandwidth.sh b/tests/queries/0_stateless/02703_max_local_read_bandwidth.sh
index 130f3a29ade..d47e2f363bd 100755
--- a/tests/queries/0_stateless/02703_max_local_read_bandwidth.sh
+++ b/tests/queries/0_stateless/02703_max_local_read_bandwidth.sh
@@ -32,7 +32,7 @@ for read_method in "${read_methods[@]}"; do
             query_duration_ms >= 7e3,
             ProfileEvents['ReadBufferFromFileDescriptorReadBytes'] > 8e6,
             ProfileEvents['LocalReadThrottlerBytes'] > 8e6,
-            ProfileEvents['LocalReadThrottlerSleepMicroseconds'] > 7e6
+            ProfileEvents['LocalReadThrottlerSleepMicroseconds'] > 7e6*0.9
         FROM system.query_log
         WHERE current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' AND type != 'QueryStart'
     "
diff --git a/tests/queries/0_stateless/02703_max_local_write_bandwidth.sh b/tests/queries/0_stateless/02703_max_local_write_bandwidth.sh
index 80713e90169..41165d35d37 100755
--- a/tests/queries/0_stateless/02703_max_local_write_bandwidth.sh
+++ b/tests/queries/0_stateless/02703_max_local_write_bandwidth.sh
@@ -19,7 +19,7 @@ $CLICKHOUSE_CLIENT -nm -q "
         query_duration_ms >= 7e3,
         ProfileEvents['WriteBufferFromFileDescriptorWriteBytes'] > 8e6,
         ProfileEvents['LocalWriteThrottlerBytes'] > 8e6,
-        ProfileEvents['LocalWriteThrottlerSleepMicroseconds'] > 7e6
+        ProfileEvents['LocalWriteThrottlerSleepMicroseconds'] > 7e6*0.9
     FROM system.query_log
     WHERE current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' AND type != 'QueryStart'
 "

From 9fe4f1a934c86d5d42fef6bbd7c3d19db4e7f97b Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 11 May 2023 21:58:07 +0200
Subject: [PATCH 597/689] Extensive coverage for bandwidth limiting settings

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/integration/test_throttling/__init__.py |   0
 .../configs/server_backups.xml                |  34 ++
 .../configs/server_overrides.xml              |   3 +
 .../configs/users_overrides.xml               |   3 +
 tests/integration/test_throttling/test.py     | 413 ++++++++++++++++++
 5 files changed, 453 insertions(+)
 create mode 100644 tests/integration/test_throttling/__init__.py
 create mode 100644 tests/integration/test_throttling/configs/server_backups.xml
 create mode 100644 tests/integration/test_throttling/configs/server_overrides.xml
 create mode 100644 tests/integration/test_throttling/configs/users_overrides.xml
 create mode 100644 tests/integration/test_throttling/test.py

diff --git a/tests/integration/test_throttling/__init__.py b/tests/integration/test_throttling/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_throttling/configs/server_backups.xml b/tests/integration/test_throttling/configs/server_backups.xml
new file mode 100644
index 00000000000..d25c67a779c
--- /dev/null
+++ b/tests/integration/test_throttling/configs/server_backups.xml
@@ -0,0 +1,34 @@
+<clickhouse>
+    <storage_configuration>
+        <disks>
+            <s3>
+                <type>s3</type>
+                <endpoint>http://minio1:9001/root/data/</endpoint>
+                <access_key_id>minio</access_key_id>
+                <secret_access_key>minio123</secret_access_key>
+            </s3>
+        </disks>
+        <policies>
+            <s3>
+                <volumes>
+                    <main>
+                        <disk>s3</disk>
+                    </main>
+                </volumes>
+            </s3>
+        </policies>
+    </storage_configuration>
+
+    <named_collections>
+        <s3>
+            <url>http://minio1:9001/root/data/</url>
+            <access_key_id>minio</access_key_id>
+            <secret_access_key>minio123</secret_access_key>
+        </s3>
+    </named_collections>
+
+    <backups>
+        <allowed_disk>default</allowed_disk>
+        <allowed_path>/backups/</allowed_path>
+    </backups>
+</clickhouse>
diff --git a/tests/integration/test_throttling/configs/server_overrides.xml b/tests/integration/test_throttling/configs/server_overrides.xml
new file mode 100644
index 00000000000..197bf660500
--- /dev/null
+++ b/tests/integration/test_throttling/configs/server_overrides.xml
@@ -0,0 +1,3 @@
+<clickhouse>
+    <!-- Will be overwritten by the test -->
+</clickhouse>
diff --git a/tests/integration/test_throttling/configs/users_overrides.xml b/tests/integration/test_throttling/configs/users_overrides.xml
new file mode 100644
index 00000000000..197bf660500
--- /dev/null
+++ b/tests/integration/test_throttling/configs/users_overrides.xml
@@ -0,0 +1,3 @@
+<clickhouse>
+    <!-- Will be overwritten by the test -->
+</clickhouse>
diff --git a/tests/integration/test_throttling/test.py b/tests/integration/test_throttling/test.py
new file mode 100644
index 00000000000..ff8e7154d0d
--- /dev/null
+++ b/tests/integration/test_throttling/test.py
@@ -0,0 +1,413 @@
+# pylint: disable=unused-argument
+# pylint: disable=redefined-outer-name
+# pylint: disable=line-too-long
+
+# This test covers the following options:
+# - max_backup_bandwidth
+# - max_backup_bandwidth_for_server
+# - max_local_read_bandwidth
+# - max_local_read_bandwidth_for_server
+# - max_local_write_bandwidth
+# - max_local_write_bandwidth_for_server
+# - max_remote_read_network_bandwidth
+# - max_remote_read_network_bandwidth_for_server
+# - max_remote_write_network_bandwidth
+# - max_remote_write_network_bandwidth_for_server
+# - and that max_backup_bandwidth from the query will override setting from the user profile
+
+import time
+import pytest
+
+from helpers.cluster import ClickHouseCluster
+
+cluster = ClickHouseCluster(__file__)
+
+
+def elapsed(func, *args, **kwargs):
+    start = time.time()
+    ret = func(*args, **kwargs)
+    end = time.time()
+    return ret, end - start
+
+
+node = cluster.add_instance(
+    "node",
+    stay_alive=True,
+    main_configs=["configs/server_backups.xml", "configs/server_overrides.xml"],
+    user_configs=["configs/users_overrides.xml"],
+    with_minio=True,
+)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def start_cluster():
+    try:
+        cluster.start()
+        yield
+    finally:
+        cluster.shutdown()
+
+
+@pytest.fixture(scope="function", autouse=True)
+def revert_config():
+    # Revert configs after the test, not before
+    yield
+    node.exec_in_container(
+        [
+            "bash",
+            "-c",
+            f"echo '<clickhouse></clickhouse>' > /etc/clickhouse-server/config.d/server_overrides.xml",
+        ]
+    )
+    node.exec_in_container(
+        [
+            "bash",
+            "-c",
+            f"echo '<clickhouse></clickhouse>' > /etc/clickhouse-server/users.d/users_overrides.xml",
+        ]
+    )
+    node.restart_clickhouse()
+
+
+backup_id_counter = 0
+
+
+def next_backup_name(storage):
+    global backup_id_counter
+    if storage == "local":
+        backup_id_counter += 1
+        return f"Disk('default', '{backup_id_counter}/')"
+    elif storage == "remote":
+        backup_id_counter += 1
+        return f"S3(s3, '{backup_id_counter}/')"
+    else:
+        raise Exception(storage)
+
+
+def node_update_config(mode, setting, value=None):
+    if mode is None:
+        return
+    if mode == "server":
+        config_path = "/etc/clickhouse-server/config.d/server_overrides.xml"
+        config_content = f"""
+        <clickhouse><{setting}>{value}</{setting}></clickhouse>
+        """
+    else:
+        config_path = "/etc/clickhouse-server/users.d/users_overrides.xml"
+        config_content = f"""
+        <clickhouse>
+            <profiles>
+                <default>
+                    <{setting}>{value}</{setting}>
+                </default>
+            </profiles>
+        </clickhouse>
+        """
+    node.exec_in_container(
+        [
+            "bash",
+            "-c",
+            f"echo '{config_content}' > {config_path}",
+        ]
+    )
+    node.restart_clickhouse()
+
+
+def assert_took(took, should_took):
+    assert took >= should_took[0] * 0.9 and took < should_took[1]
+
+
+@pytest.mark.parametrize(
+    "policy,backup_name,mode,setting,value,should_took",
+    [
+        #
+        # Local -> Local
+        #
+        pytest.param(
+            "default",
+            next_backup_name("local"),
+            None,
+            None,
+            None,
+            (0, 3),
+            id="no_local_throttling",
+        ),
+        # reading 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
+        pytest.param(
+            "default",
+            next_backup_name("local"),
+            "user",
+            "max_backup_bandwidth",
+            "1M",
+            (7, 14),
+            id="user_local_throttling",
+        ),
+        # reading 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds
+        pytest.param(
+            "default",
+            next_backup_name("local"),
+            "server",
+            "max_backup_bandwidth_for_server",
+            "2M",
+            (3, 7),
+            id="server_local_throttling",
+        ),
+        #
+        # Remote -> Local
+        #
+        pytest.param(
+            "s3",
+            next_backup_name("local"),
+            None,
+            None,
+            None,
+            (0, 3),
+            id="no_remote_to_local_throttling",
+        ),
+        # reading 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
+        pytest.param(
+            "s3",
+            next_backup_name("local"),
+            "user",
+            "max_backup_bandwidth",
+            "1M",
+            (7, 14),
+            id="user_remote_to_local_throttling",
+        ),
+        # reading 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds
+        pytest.param(
+            "s3",
+            next_backup_name("local"),
+            "server",
+            "max_backup_bandwidth_for_server",
+            "2M",
+            (3, 7),
+            id="server_remote_to_local_throttling",
+        ),
+        #
+        # Remote -> Remote
+        #
+        pytest.param(
+            "s3",
+            next_backup_name("remote"),
+            None,
+            None,
+            None,
+            (0, 3),
+            id="no_remote_to_remote_throttling",
+        ),
+        # No throttling for S3-to-S3, uses native copy
+        pytest.param(
+            "s3",
+            next_backup_name("remote"),
+            "user",
+            "max_backup_bandwidth",
+            "1M",
+            (0, 3),
+            id="user_remote_to_remote_throttling",
+        ),
+        # No throttling for S3-to-S3, uses native copy
+        pytest.param(
+            "s3",
+            next_backup_name("remote"),
+            "server",
+            "max_backup_bandwidth_for_server",
+            "2M",
+            (0, 3),
+            id="server_remote_to_remote_throttling",
+        ),
+        #
+        # Local -> Remote
+        #
+        # NOTE: S3 is complex, it will read file 3 times:
+        # - first for calculating the checksum
+        # - second for calculating the signature
+        # - and finally to write the payload to S3
+        # Hence the value should be multipled by 3.
+        pytest.param(
+            "default",
+            next_backup_name("remote"),
+            None,
+            None,
+            None,
+            (0, 3),
+            id="no_local_to_remote_throttling",
+        ),
+        # reading 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds, but for S3Client it is 2x more
+        pytest.param(
+            "default",
+            next_backup_name("remote"),
+            "user",
+            "max_backup_bandwidth",
+            "1M",
+            (7 * 3, 7 * 4 - 1),
+            id="user_local_to_remote_throttling",
+        ),
+        # reading 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds, but for S3Client it is 2x more
+        pytest.param(
+            "default",
+            next_backup_name("remote"),
+            "server",
+            "max_backup_bandwidth_for_server",
+            "2M",
+            (3 * 3, 3 * 5),
+            id="server_local_to_remote_throttling",
+        ),
+    ],
+)
+def test_backup_throttling(policy, backup_name, mode, setting, value, should_took):
+    node_update_config(mode, setting, value)
+    node.query(
+        f"""
+        drop table if exists data;
+        create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9, storage_policy='{policy}';
+        insert into data select * from numbers(1e6);
+    """
+    )
+    _, took = elapsed(node.query, f"backup table data to {backup_name}")
+    assert_took(took, should_took)
+
+
+def test_backup_throttling_override():
+    node_update_config("user", "max_backup_bandwidth", "1M")
+    node.query(
+        """
+        drop table if exists data;
+        create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9;
+        insert into data select * from numbers(1e6);
+    """
+    )
+
+    backup_name = next_backup_name("local")
+    _, took = elapsed(
+        node.query,
+        f"backup table data to {backup_name}",
+        settings={
+            "max_backup_bandwidth": "500K",
+        },
+    )
+    # reading 1e6*8 bytes with 500Ki default bandwith should take (8-0.5)/0.5=15 seconds
+    assert_took(took, (15, 20))
+
+
+@pytest.mark.parametrize(
+    "policy,mode,setting,value,should_took",
+    [
+        #
+        # Local
+        #
+        pytest.param("default", None, None, None, (0, 3), id="no_local_throttling"),
+        # reading 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
+        pytest.param(
+            "default",
+            "user",
+            "max_local_read_bandwidth",
+            "1M",
+            (7, 14),
+            id="user_local_throttling",
+        ),
+        # reading 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds
+        pytest.param(
+            "default",
+            "server",
+            "max_local_read_bandwidth_for_server",
+            "2M",
+            (3, 7),
+            id="server_local_throttling",
+        ),
+        #
+        # Remote
+        #
+        pytest.param("s3", None, None, None, (0, 3), id="no_remote_throttling"),
+        # reading 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
+        pytest.param(
+            "s3",
+            "user",
+            "max_remote_read_network_bandwidth",
+            "1M",
+            (7, 14),
+            id="user_remote_throttling",
+        ),
+        # reading 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds
+        pytest.param(
+            "s3",
+            "server",
+            "max_remote_read_network_bandwidth_for_server",
+            "2M",
+            (3, 7),
+            id="server_remote_throttling",
+        ),
+    ],
+)
+def test_read_throttling(policy, mode, setting, value, should_took):
+    node_update_config(mode, setting, value)
+    node.query(
+        f"""
+        drop table if exists data;
+        create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9, storage_policy='{policy}';
+        insert into data select * from numbers(1e6);
+    """
+    )
+    _, took = elapsed(node.query, f"select * from data")
+    assert_took(took, should_took)
+
+
+@pytest.mark.parametrize(
+    "policy,mode,setting,value,should_took",
+    [
+        #
+        # Local
+        #
+        pytest.param("default", None, None, None, (0, 3), id="no_local_throttling"),
+        # reading 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
+        pytest.param(
+            "default",
+            "user",
+            "max_local_write_bandwidth",
+            "1M",
+            (7, 14),
+            id="local_user_throttling",
+        ),
+        # reading 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds
+        pytest.param(
+            "default",
+            "server",
+            "max_local_write_bandwidth_for_server",
+            "2M",
+            (3, 7),
+            id="local_server_throttling",
+        ),
+        #
+        # Remote
+        #
+        pytest.param("s3", None, None, None, (0, 3), id="no_remote_throttling"),
+        # writeing 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
+        pytest.param(
+            "s3",
+            "user",
+            "max_remote_write_network_bandwidth",
+            "1M",
+            (7, 14),
+            id="user_remote_throttling",
+        ),
+        # writeing 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds
+        pytest.param(
+            "s3",
+            "server",
+            "max_remote_write_network_bandwidth_for_server",
+            "2M",
+            (3, 7),
+            id="server_remote_throttling",
+        ),
+    ],
+)
+def test_write_throttling(policy, mode, setting, value, should_took):
+    node_update_config(mode, setting, value)
+    node.query(
+        f"""
+        drop table if exists data;
+        create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9, storage_policy='{policy}';
+    """
+    )
+    _, took = elapsed(node.query, f"insert into data select * from numbers(1e6)")
+    assert_took(took, should_took)

From 3787b7f127750b573a98920f8e7243e67fe15d68 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Wed, 17 May 2023 12:16:18 +0200
Subject: [PATCH 598/689] Update Metadata.cpp

---
 src/Interpreters/Cache/Metadata.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp
index 01cdc7f1d1b..c87eaabbbf2 100644
--- a/src/Interpreters/Cache/Metadata.cpp
+++ b/src/Interpreters/Cache/Metadata.cpp
@@ -1,7 +1,6 @@
 #include <Interpreters/Cache/Metadata.h>
 #include <Interpreters/Cache/FileCache.h>
 #include <Interpreters/Cache/FileSegment.h>
-#include "Common/Exception.h"
 #include <Common/logger_useful.h>
 #include <filesystem>
 

From 3d8b6b708fd655c4012afd1702092a36a3fb2e86 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Wed, 17 May 2023 10:20:29 +0000
Subject: [PATCH 599/689] Fix for analyzer: 02377_
 ptimize_sorting_by_input_stream_properties_explain

---
 ..._input_stream_properties_explain.reference | 74 +++++++++++++++++++
 ...ting_by_input_stream_properties_explain.sh |  5 +-
 2 files changed, 78 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.reference b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.reference
index 0b34e878cee..69c325c21a9 100644
--- a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.reference
+++ b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.reference
@@ -5,6 +5,12 @@ Sorting (Sorting for ORDER BY)
 Sorting (Global): a ASC
 Sorting (None)
 Sorting (None)
+-- QUERY (analyzer): set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a FROM optimize_sorting ORDER BY a
+Sorting (Global): a ASC
+Sorting (Sorting for ORDER BY)
+Sorting (Global): default.optimize_sorting.a_0 ASC
+Sorting (None)
+Sorting (None)
 -- disable optimization -> sorting order is NOT propagated from subquery -> full sort
 -- QUERY: set optimize_sorting_by_input_stream_properties=0;set query_plan_read_in_order=0;set max_threads=3;EXPLAIN PIPELINE SELECT a FROM (SELECT a FROM optimize_sorting) ORDER BY a
 MergingSortedTransform 3 → 1
@@ -27,12 +33,24 @@ Sorting (Sorting for ORDER BY)
 Sorting (Global): a ASC
 Sorting (Stream): a ASC
 Sorting (Stream): a ASC
+-- QUERY (analyzer): set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a FROM optimize_sorting ORDER BY a
+Sorting (Global): a ASC
+Sorting (Sorting for ORDER BY)
+Sorting (Global): default.optimize_sorting.a_0 ASC
+Sorting (Stream): default.optimize_sorting.a_0 ASC
+Sorting (Stream): a ASC
 -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a FROM optimize_sorting ORDER BY a+1
 Sorting (None)
 Sorting (Sorting for ORDER BY)
 Sorting (Global): plus(a, 1) ASC
 Sorting (Chunk): a ASC
 Sorting (Chunk): a ASC
+-- QUERY (analyzer): set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a FROM optimize_sorting ORDER BY a+1
+Sorting (None)
+Sorting (Sorting for ORDER BY)
+Sorting (Global): plus(default.optimize_sorting.a_0, 1_UInt8) ASC
+Sorting (Chunk): default.optimize_sorting.a_0 ASC
+Sorting (Chunk): a ASC
 -- ExpressionStep breaks sort mode
 -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a+1 FROM optimize_sorting ORDER BY a+1
 Sorting (Global): plus(a, 1) ASC
@@ -40,28 +58,54 @@ Sorting (Sorting for ORDER BY)
 Sorting (Global): plus(a, 1) ASC
 Sorting (None)
 Sorting (Chunk): a ASC
+-- QUERY (analyzer): set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a+1 FROM optimize_sorting ORDER BY a+1
+Sorting (Global): plus(a, 1) ASC
+Sorting (Sorting for ORDER BY)
+Sorting (Global): plus(default.optimize_sorting.a_0, 1_UInt8) ASC
+Sorting (None)
+Sorting (Chunk): a ASC
 -- FilterStep preserves sort mode
 -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a FROM optimize_sorting WHERE a > 0
 Sorting (Chunk): a ASC
 Sorting (Chunk): a ASC
 Sorting (Chunk): a ASC
+-- QUERY (analyzer): set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a FROM optimize_sorting WHERE a > 0
+Sorting (Chunk): a ASC
+Sorting (Chunk): default.optimize_sorting.a_0 ASC
+Sorting (Chunk): a ASC
 -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a FROM optimize_sorting WHERE a+1 > 0
 Sorting (Chunk): a ASC
 Sorting (Chunk): a ASC
 Sorting (Chunk): a ASC
+-- QUERY (analyzer): set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a FROM optimize_sorting WHERE a+1 > 0
+Sorting (Chunk): a ASC
+Sorting (Chunk): default.optimize_sorting.a_0 ASC
+Sorting (Chunk): a ASC
 -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a, a+1 FROM optimize_sorting WHERE a+1 > 0
 Sorting (Chunk): a ASC
 Sorting (Chunk): a ASC
 Sorting (Chunk): a ASC
+-- QUERY (analyzer): set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a, a+1 FROM optimize_sorting WHERE a+1 > 0
+Sorting (Chunk): a ASC
+Sorting (Chunk): default.optimize_sorting.a_0 ASC
+Sorting (Chunk): a ASC
 -- FilterStep breaks sort mode
 -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a > 0 FROM optimize_sorting WHERE a > 0
 Sorting (None)
 Sorting (None)
 Sorting (Chunk): a ASC
+-- QUERY (analyzer): set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a > 0 FROM optimize_sorting WHERE a > 0
+Sorting (None)
+Sorting (None)
+Sorting (Chunk): a ASC
 -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a+1 FROM optimize_sorting WHERE a+1 > 0
 Sorting (None)
 Sorting (None)
 Sorting (Chunk): a ASC
+-- QUERY (analyzer): set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a+1 FROM optimize_sorting WHERE a+1 > 0
+Sorting (None)
+Sorting (None)
+Sorting (Chunk): a ASC
 -- aliases break sorting order
 -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a FROM (SELECT sipHash64(a) AS a FROM (SELECT a FROM optimize_sorting ORDER BY a)) ORDER BY a
 Sorting (Global): a ASC
@@ -72,6 +116,15 @@ Sorting (Sorting for ORDER BY)
 Sorting (Global): a ASC
 Sorting (Stream): a ASC
 Sorting (Stream): a ASC
+-- QUERY (analyzer): set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a FROM (SELECT sipHash64(a) AS a FROM (SELECT a FROM optimize_sorting ORDER BY a)) ORDER BY a
+Sorting (Global): a ASC
+Sorting (Sorting for ORDER BY)
+Sorting (Global): a_0 ASC
+Sorting (None)
+Sorting (Sorting for ORDER BY)
+Sorting (Global): default.optimize_sorting.a_2 ASC
+Sorting (Stream): default.optimize_sorting.a_2 ASC
+Sorting (Stream): a ASC
 -- aliases DONT break sorting order
 -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a, b FROM (SELECT x AS a, y AS b FROM (SELECT a AS x, b AS y FROM optimize_sorting) ORDER BY x, y)
 Sorting (Global): a ASC, b ASC
@@ -79,6 +132,12 @@ Sorting (Sorting for ORDER BY)
 Sorting (Global): x ASC, y ASC
 Sorting (Stream): x ASC, y ASC
 Sorting (Stream): a ASC, b ASC
+-- QUERY (analyzer): set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a, b FROM (SELECT x AS a, y AS b FROM (SELECT a AS x, b AS y FROM optimize_sorting) ORDER BY x, y)
+Sorting (Global): a ASC, b ASC
+Sorting (Sorting for ORDER BY)
+Sorting (Global): x_2 ASC, y_3 ASC
+Sorting (Stream): x_2 ASC, y_3 ASC
+Sorting (Stream): a ASC, b ASC
 -- actions chain breaks sorting order: input(column a)->sipHash64(column a)->alias(sipHash64(column a), a)->plus(alias a, 1)
 -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a, z FROM (SELECT sipHash64(a) AS a, a + 1 AS z FROM (SELECT a FROM optimize_sorting ORDER BY a + 1)) ORDER BY a + 1
 Sorting (None)
@@ -89,6 +148,15 @@ Sorting (Sorting for ORDER BY)
 Sorting (Global): plus(a, 1) ASC
 Sorting (Chunk): a ASC
 Sorting (Chunk): a ASC
+-- QUERY (analyzer): set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a, z FROM (SELECT sipHash64(a) AS a, a + 1 AS z FROM (SELECT a FROM optimize_sorting ORDER BY a + 1)) ORDER BY a + 1
+Sorting (None)
+Sorting (Sorting for ORDER BY)
+Sorting (Global): plus(a_0, 1_UInt8) ASC
+Sorting (Global): plus(default.optimize_sorting.a_3, 1_UInt8) ASC
+Sorting (Sorting for ORDER BY)
+Sorting (Global): plus(default.optimize_sorting.a_3, 1_UInt8) ASC
+Sorting (Chunk): default.optimize_sorting.a_3 ASC
+Sorting (Chunk): a ASC
 -- check that correct sorting info is provided in case of only prefix of sorting key is in ORDER BY clause but all sorting key columns returned by query
 -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN sorting=1 SELECT a, b FROM optimize_sorting ORDER BY a
 Sorting (Global): a ASC
@@ -96,3 +164,9 @@ Sorting (Sorting for ORDER BY)
 Sorting (Global): a ASC
 Sorting (Stream): a ASC
 Sorting (Stream): a ASC
+-- QUERY (analyzer): set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN sorting=1 SELECT a, b FROM optimize_sorting ORDER BY a
+Sorting (Global): a ASC
+Sorting (Sorting for ORDER BY)
+Sorting (Global): default.optimize_sorting.a_0 ASC
+Sorting (Stream): default.optimize_sorting.a_0 ASC
+Sorting (Stream): a ASC
diff --git a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh
index e4cab987d35..7e937ac42b6 100755
--- a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh
+++ b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh
@@ -17,9 +17,12 @@ function explain_sorting {
     echo "-- QUERY: "$1
     $CLICKHOUSE_CLIENT -nq "$1" | eval $FIND_SORTING
 }
+
 function explain_sortmode {
     echo "-- QUERY: "$1
-    $CLICKHOUSE_CLIENT -nq "$1" | eval $FIND_SORTMODE
+    $CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 -nq "$1" | eval $FIND_SORTMODE
+    echo "-- QUERY (analyzer): "$1
+    $CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 -nq "$1" | eval $FIND_SORTMODE
 }
 
 $CLICKHOUSE_CLIENT -q "drop table if exists optimize_sorting sync"

From 36c31e1d795412b2d3107f58409d4c8806505a1a Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Wed, 17 May 2023 14:07:34 +0300
Subject: [PATCH 600/689] Improve concurrent parts removal with zero copy
 replication (#49630)

* improve concurrent parts removal

* fix

* fix
---
 src/Storages/MergeTree/MergeTreeData.cpp   | 154 ++++++++++++++++++---
 src/Storages/MergeTree/MergeTreeSettings.h |   2 +
 2 files changed, 133 insertions(+), 23 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index b8208052f19..6ddfc3b806e 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -19,6 +19,7 @@
 #include <Common/typeid_cast.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/ThreadFuzzer.h>
+#include <Common/getNumberOfPhysicalCPUCores.h>
 #include <Compression/CompressedReadBuffer.h>
 #include <Core/QueryProcessingStage.h>
 #include <DataTypes/DataTypeEnum.h>
@@ -2429,9 +2430,13 @@ void MergeTreeData::clearPartsFromFilesystemImpl(const DataPartsVector & parts_t
     }
 
     /// Parallel parts removal.
-    size_t num_threads = std::min<size_t>(settings->max_part_removal_threads, parts_to_remove.size());
+    size_t num_threads = settings->max_part_removal_threads;
+    if (!num_threads)
+        num_threads = getNumberOfPhysicalCPUCores() * 2;
+    num_threads = std::min<size_t>(num_threads, parts_to_remove.size());
     std::mutex part_names_mutex;
-    ThreadPool pool(CurrentMetrics::MergeTreePartsCleanerThreads, CurrentMetrics::MergeTreePartsCleanerThreadsActive, num_threads);
+    ThreadPool pool(CurrentMetrics::MergeTreePartsCleanerThreads, CurrentMetrics::MergeTreePartsCleanerThreadsActive,
+                    num_threads, num_threads, /* unlimited queue size */ 0);
 
     /// This flag disallow straightforward concurrent parts removal. It's required only in case
     /// when we have parts on zero-copy disk + at least some of them were mutated.
@@ -2490,29 +2495,62 @@ void MergeTreeData::clearPartsFromFilesystemImpl(const DataPartsVector & parts_t
     /// We remove disjoint subsets of parts in parallel.
     /// The problem is that it's not trivial to divide Outdated parts into disjoint subsets,
     /// because Outdated parts legally can be intersecting (but intersecting parts must be separated by a DROP_RANGE).
-    /// So we ignore level and version and use block numbers only.
-    ActiveDataPartSet independent_ranges_set(format_version);
-    for (const auto & part : parts_to_remove)
+    /// So we ignore level and version and use block numbers only (they cannot intersect by block numbers unless we have a bug).
+
+    struct RemovalRanges
     {
-        MergeTreePartInfo range_info = part->info;
-        range_info.level = static_cast<UInt32>(range_info.max_block - range_info.min_block);
-        range_info.mutation = 0;
-        independent_ranges_set.add(range_info, range_info.getPartNameV1());
-    }
+        std::vector<MergeTreePartInfo> infos;
+        std::vector<DataPartsVector> parts;
+        std::vector<UInt64> split_times;
+    };
 
-    auto independent_ranges_infos = independent_ranges_set.getPartInfos();
-    size_t sum_of_ranges = 0;
-    for (auto range : independent_ranges_infos)
+    auto split_into_independent_ranges = [this](const DataPartsVector & parts_to_remove_, size_t split_times) -> RemovalRanges
     {
-        range.level = MergeTreePartInfo::MAX_LEVEL;
-        range.mutation = MergeTreePartInfo::MAX_BLOCK_NUMBER;
+        if (parts_to_remove_.empty())
+            return {};
 
-        DataPartsVector parts_in_range;
-        for (const auto & part : parts_to_remove)
-            if (range.contains(part->info))
-                parts_in_range.push_back(part);
-        sum_of_ranges += parts_in_range.size();
+        ActiveDataPartSet independent_ranges_set(format_version);
+        for (const auto & part : parts_to_remove_)
+        {
+            MergeTreePartInfo range_info = part->info;
+            range_info.level = static_cast<UInt32>(range_info.max_block - range_info.min_block);
+            range_info.mutation = 0;
+            independent_ranges_set.add(range_info, range_info.getPartNameV1());
+        }
 
+        RemovalRanges independent_ranges;
+        independent_ranges.infos = independent_ranges_set.getPartInfos();
+        size_t num_ranges = independent_ranges.infos.size();
+        independent_ranges.parts.resize(num_ranges);
+        independent_ranges.split_times.resize(num_ranges, split_times);
+        size_t avg_range_size = parts_to_remove_.size() / num_ranges;
+
+        size_t sum_of_ranges = 0;
+        for (size_t i = 0; i < num_ranges; ++i)
+        {
+            MergeTreePartInfo & range = independent_ranges.infos[i];
+            DataPartsVector & parts_in_range = independent_ranges.parts[i];
+            range.level = MergeTreePartInfo::MAX_LEVEL;
+            range.mutation = MergeTreePartInfo::MAX_BLOCK_NUMBER;
+
+            parts_in_range.reserve(avg_range_size * 2);
+            for (const auto & part : parts_to_remove_)
+                if (range.contains(part->info))
+                    parts_in_range.push_back(part);
+            sum_of_ranges += parts_in_range.size();
+        }
+
+        if (parts_to_remove_.size() != sum_of_ranges)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Number of removed parts is not equal to number of parts in independent ranges "
+                                                       "({} != {}), it's a bug", parts_to_remove_.size(), sum_of_ranges);
+
+        return independent_ranges;
+    };
+
+    auto schedule_parts_removal = [this, &pool, &part_names_mutex, part_names_succeed](
+        const MergeTreePartInfo & range, DataPartsVector && parts_in_range)
+    {
+        /// Below, range should be captured by copy to avoid use-after-scope on exception from pool
         pool.scheduleOrThrowOnError(
             [this, range, &part_names_mutex, part_names_succeed, thread_group = CurrentThread::getGroup(), batch = std::move(parts_in_range)]
         {
@@ -2535,13 +2573,83 @@ void MergeTreeData::clearPartsFromFilesystemImpl(const DataPartsVector & parts_t
                 }
             }
         });
+    };
+
+    RemovalRanges independent_ranges = split_into_independent_ranges(parts_to_remove, /* split_times */ 0);
+    DataPartsVector excluded_parts;
+    size_t num_ranges = independent_ranges.infos.size();
+    size_t sum_of_ranges = 0;
+    for (size_t i = 0; i < num_ranges; ++i)
+    {
+        MergeTreePartInfo & range = independent_ranges.infos[i];
+        DataPartsVector & parts_in_range = independent_ranges.parts[i];
+        UInt64 split_times = independent_ranges.split_times[i];
+
+        /// It may happen that we have a huge part covering thousands small parts.
+        /// In this case, we will get a huge range that will be process by only one thread causing really long tail latency.
+        /// Let's try to exclude such parts in order to get smaller tasks for thread pool and more uniform distribution.
+        if (settings->concurrent_part_removal_threshold < parts_in_range.size() &&
+            split_times < settings->zero_copy_concurrent_part_removal_max_split_times)
+        {
+            auto smaller_parts_pred = [&range](const DataPartPtr & part)
+            {
+                return !(part->info.min_block == range.min_block && part->info.max_block == range.max_block);
+            };
+
+            size_t covered_parts_count = std::count_if(parts_in_range.begin(), parts_in_range.end(), smaller_parts_pred);
+            size_t top_level_count = parts_in_range.size() - covered_parts_count;
+            chassert(top_level_count);
+            Float32 parts_to_exclude_ratio = static_cast<Float32>(top_level_count) / parts_in_range.size();
+            if (settings->zero_copy_concurrent_part_removal_max_postpone_ratio < parts_to_exclude_ratio)
+            {
+                /// Most likely we have a long mutations chain here
+                LOG_DEBUG(log, "Block range {} contains {} parts including {} top-level parts, will not try to split it",
+                          range.getPartNameForLogs(), parts_in_range.size(), top_level_count);
+            }
+            else
+            {
+                auto new_end_it = std::partition(parts_in_range.begin(), parts_in_range.end(), smaller_parts_pred);
+                std::move(new_end_it, parts_in_range.end(), std::back_inserter(excluded_parts));
+                parts_in_range.erase(new_end_it, parts_in_range.end());
+
+                RemovalRanges subranges = split_into_independent_ranges(parts_in_range, split_times + 1);
+
+                LOG_DEBUG(log, "Block range {} contained {} parts, it was split into {} independent subranges after excluding {} top-level parts",
+                          range.getPartNameForLogs(), parts_in_range.size() + top_level_count, subranges.infos.size(), top_level_count);
+
+                std::move(subranges.infos.begin(), subranges.infos.end(), std::back_inserter(independent_ranges.infos));
+                std::move(subranges.parts.begin(), subranges.parts.end(), std::back_inserter(independent_ranges.parts));
+                std::move(subranges.split_times.begin(), subranges.split_times.end(), std::back_inserter(independent_ranges.split_times));
+                num_ranges += subranges.infos.size();
+                continue;
+            }
+        }
+
+        sum_of_ranges += parts_in_range.size();
+
+        schedule_parts_removal(range, std::move(parts_in_range));
+    }
+
+    /// Remove excluded parts as well. They were reordered, so sort them again
+    std::sort(excluded_parts.begin(), excluded_parts.end(), [](const auto & x, const auto & y) { return x->info < y->info; });
+    LOG_TRACE(log, "Will remove {} big parts separately: {}", excluded_parts.size(), fmt::join(excluded_parts, ", "));
+
+    independent_ranges = split_into_independent_ranges(excluded_parts, /* split_times */ 0);
+    pool.wait();
+
+    for (size_t i = 0; i < independent_ranges.infos.size(); ++i)
+    {
+        MergeTreePartInfo & range = independent_ranges.infos[i];
+        DataPartsVector & parts_in_range = independent_ranges.parts[i];
+        schedule_parts_removal(range, std::move(parts_in_range));
     }
 
     pool.wait();
 
-    if (parts_to_remove.size() != sum_of_ranges)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Number of removed parts is not equal to number of parts in independent ranges "
-                                                   "({} != {}), it's a bug", parts_to_remove.size(), sum_of_ranges);
+    if (parts_to_remove.size() != sum_of_ranges + excluded_parts.size())
+        throw Exception(ErrorCodes::LOGICAL_ERROR,
+                        "Number of parts to remove was not equal to number of parts in independent ranges and excluded parts"
+                        "({} != {} + {}), it's a bug", parts_to_remove.size(), sum_of_ranges, excluded_parts.size());
 }
 
 size_t MergeTreeData::clearOldBrokenPartsFromDetachedDirectory()
diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h
index 9c7488cb6a6..c9e81ce9103 100644
--- a/src/Storages/MergeTree/MergeTreeSettings.h
+++ b/src/Storages/MergeTree/MergeTreeSettings.h
@@ -146,6 +146,8 @@ struct Settings;
     M(MaxThreads, max_part_loading_threads, 0, "The number of threads to load data parts at startup.", 0) \
     M(MaxThreads, max_part_removal_threads, 0, "The number of threads for concurrent removal of inactive data parts. One is usually enough, but in 'Google Compute Environment SSD Persistent Disks' file removal (unlink) operation is extraordinarily slow and you probably have to increase this number (recommended is up to 16).", 0) \
     M(UInt64, concurrent_part_removal_threshold, 100, "Activate concurrent part removal (see 'max_part_removal_threads') only if the number of inactive data parts is at least this.", 0) \
+    M(UInt64, zero_copy_concurrent_part_removal_max_split_times, 5, "Max recursion depth for splitting independent Outdated parts ranges into smaller subranges (highly not recommended to change)", 0) \
+    M(Float, zero_copy_concurrent_part_removal_max_postpone_ratio, static_cast<Float32>(0.05), "Max percentage of top level parts to postpone removal in order to get smaller independent ranges (highly not recommended to change)", 0) \
     M(String, storage_policy, "default", "Name of storage disk policy", 0) \
     M(String, disk, "", "Name of storage disk. Can be specified instead of storage policy.", 0) \
     M(Bool, allow_nullable_key, false, "Allow Nullable types as primary keys.", 0) \

From f2dbcb514673e439b0b11efa16575dd4fb8be164 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Wed, 17 May 2023 16:25:31 +0200
Subject: [PATCH 601/689] Better fix

---
 src/Interpreters/Cache/Metadata.cpp | 38 +++++++++++++++++++----------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp
index c87eaabbbf2..e0b82763a08 100644
--- a/src/Interpreters/Cache/Metadata.cpp
+++ b/src/Interpreters/Cache/Metadata.cpp
@@ -257,30 +257,42 @@ void CacheMetadata::doCleanup()
         }
 
         locked_metadata->markAsRemoved();
+        erase(it);
+        LOG_DEBUG(log, "Key {} is removed from metadata", cleanup_key);
+
+        const fs::path key_directory = getPathInLocalCache(cleanup_key);
+        const fs::path key_prefix_directory = key_directory.parent_path();
 
         try
         {
-            const fs::path key_directory = getPathInLocalCache(cleanup_key);
             if (fs::exists(key_directory))
                 fs::remove_all(key_directory);
-
-            const fs::path key_prefix_directory = key_directory.parent_path();
-            if (fs::exists(key_prefix_directory) && fs::is_empty(key_prefix_directory))
-                fs::remove_all(key_prefix_directory);
         }
         catch (...)
         {
-            LOG_ERROR(log, "Error while removing key {}: {}", cleanup_key, getCurrentExceptionMessage(false));
+            LOG_ERROR(log, "Error while removing key {}: {}", cleanup_key, getCurrentExceptionMessage(true));
             chassert(false);
+            continue;
         }
 
-        /// Remove key from metadata AFTER deleting key directory, because otherwise key lock is
-        /// released before we delete directory from fs and there might be a race:
-        /// a key, which we just removed, can be added back to cache before we start removing key directory,
-        /// which makes key directory either non-empty (and we get exception in try catch above)
-        /// or we removed directory while another thread thinks it exists.
-        erase(it);
-        LOG_DEBUG(log, "Key {} is removed from metadata", cleanup_key);
+        try
+        {
+            if (fs::exists(key_prefix_directory) && fs::is_empty(key_prefix_directory))
+                fs::remove_all(key_prefix_directory);
+        }
+        catch (const fs::filesystem_error & e)
+        {
+            /// Key prefix directory can become non-empty just now, it is expected.
+            if (e.code() == std::errc::directory_not_empty)
+                return;
+            LOG_ERROR(log, "Error while removing key {}: {}", cleanup_key, getCurrentExceptionMessage(true));
+            chassert(false);
+        }
+        catch (...)
+        {
+            LOG_ERROR(log, "Error while removing key {}: {}", cleanup_key, getCurrentExceptionMessage(true));
+            chassert(false);
+        }
     }
 }
 

From 1c04085e8fcbb916292635a6297ae45fde7da49d Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Wed, 17 May 2023 18:15:51 +0200
Subject: [PATCH 602/689] Update MergeTreeWriteAheadLog.h

---
 src/Storages/MergeTree/MergeTreeWriteAheadLog.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.h b/src/Storages/MergeTree/MergeTreeWriteAheadLog.h
index eba7698b9f9..f5398a24e7d 100644
--- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.h
+++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.h
@@ -60,7 +60,6 @@ public:
 
     ~MergeTreeWriteAheadLog();
 
-    void addPart(DataPartInMemoryPtr & part);
     void dropPart(const String & part_name);
     std::vector<MergeTreeMutableDataPartPtr> restore(
         const StorageMetadataPtr & metadata_snapshot,
@@ -77,7 +76,6 @@ public:
 private:
     void init();
     void rotate(const std::unique_lock<std::mutex> & lock);
-    void sync(std::unique_lock<std::mutex> & lock);
 
     const MergeTreeData & storage;
     DiskPtr disk;

From c7ab59302f3ae1850619a79f138dee865a92bbb6 Mon Sep 17 00:00:00 2001
From: Timur Solodovnikov <tsolodov@gmail.com>
Date: Wed, 17 May 2023 11:03:42 -0700
Subject: [PATCH 603/689] Set allow_experimental_query_cache setting as
 obsolete (#49934)

* set allow_experimental_query_cache as obsolete

* add tsolodov to trusted contributors

* CI linter

---------

Co-authored-by: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
---
 src/Core/Settings.h                                             | 1 +
 tests/ci/workflow_approve_rerun_lambda/app.py                   | 1 +
 ...726_set_allow_experimental_query_cache_as_obsolete.reference | 0
 .../02726_set_allow_experimental_query_cache_as_obsolete.sql    | 2 ++
 4 files changed, 4 insertions(+)
 create mode 100644 tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.reference
 create mode 100644 tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.sql

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 610c7135a75..c65958b86b7 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -779,6 +779,7 @@ class IColumn;
     MAKE_OBSOLETE(M, UInt64, partial_merge_join_optimizations, 0) \
     MAKE_OBSOLETE(M, MaxThreads, max_alter_threads, 0) \
     MAKE_OBSOLETE(M, Bool, allow_experimental_projection_optimization, true) \
+    MAKE_OBSOLETE(M, Bool, allow_experimental_query_cache, true) \
     /* moved to config.xml: see also src/Core/ServerSettings.h */ \
     MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_buffer_flush_schedule_pool_size, 16) \
     MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_pool_size, 16) \
diff --git a/tests/ci/workflow_approve_rerun_lambda/app.py b/tests/ci/workflow_approve_rerun_lambda/app.py
index 7d234bdcfc4..2cdbdecca6e 100644
--- a/tests/ci/workflow_approve_rerun_lambda/app.py
+++ b/tests/ci/workflow_approve_rerun_lambda/app.py
@@ -125,6 +125,7 @@ TRUSTED_CONTRIBUTORS = {
         "thevar1able",  # ClickHouse Employee
         "aalexfvk",
         "MikhailBurdukov",
+        "tsolodov",  # ClickHouse Employee
     ]
 }
 
diff --git a/tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.reference b/tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.sql b/tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.sql
new file mode 100644
index 00000000000..244ba4e959a
--- /dev/null
+++ b/tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.sql
@@ -0,0 +1,2 @@
+SET allow_experimental_query_cache = 0;
+SET allow_experimental_query_cache = 1;

From a1fc96953f41818a1e661af49f14d4cad8d7226f Mon Sep 17 00:00:00 2001
From: DanRoscigno <dan@roscigno.com>
Date: Wed, 17 May 2023 14:48:16 -0400
Subject: [PATCH 604/689] reorder

---
 docs/en/operations/named-collections.md | 37 +++++++++++++------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/docs/en/operations/named-collections.md b/docs/en/operations/named-collections.md
index b65438b97f3..1866315529e 100644
--- a/docs/en/operations/named-collections.md
+++ b/docs/en/operations/named-collections.md
@@ -13,7 +13,23 @@ Named collections can be configured with DDL or in configuration files and are a
 when ClickHouse starts. They simplify the creation of objects and the hiding of credentials
 from users without administrative access.
 
-## Storing details for connecting to external sources in the system database
+The keys in a named collection must match the parameter names of the corresponding
+function, table engine, database, etc. In the examples below the parameter list is
+linked to for each type.
+
+Parameters set in a named collection can be overridden in SQL, this is shown in the examples
+below.
+
+## Storing named collections in the system database
+
+### DDL example
+
+```sql
+CREATE NAMED COLLECTION name AS
+key_1 = 'value',
+key_2 = 'value2',
+url = 'https://connection.url/'
+```
 
 ### Permissions to create named collections with DDL
 
@@ -37,22 +53,7 @@ To manage named collections with DDL a user must have the `named_control_collect
 In the above example the `passowrd_sha256_hex` value is the hexadecimal representation of the SHA256 hash of the password.  This configuration for the user `default` has the attribute `replace=true` as in the default configuration has a plain text `password` set, and it is not possible to have both plaintext and sha256 hex passwords set for a user. 
 :::
 
-## Storing details for connecting to external sources in configuration files
-
-
-Parameters can be set in XML `<format>CSV</format>` and overridden in SQL `, format = 'TSV'`.
-The parameters in SQL can be overridden using format `key` = `value`: `compression_method = 'gzip'`.
-
-Named collections are stored in the system table `system.named_collections` or in `config.xml` file of the ClickHouse server in the `<named_collections>` section and are applied when ClickHouse starts.
-
-### DDL example
-
-```sql
-CREATE NAMED COLLECTION name AS
-key_1 = 'value',
-key_2 = 'value2',
-url = 'https://connection.url/'
-```
+## Storing named collections in configuration files
 
 ### XML example
 
@@ -98,7 +99,7 @@ url = 'https://s3.us-east-1.amazonaws.com/yourbucket/mydata/'
 </clickhouse>
 ```
 
-### s3() function and S3 Table Named collection examples
+### s3() function and S3 Table named collection examples
 
 Both of the following examples use the same named collection `s3_mydata`:
 

From 549af4d35112f598f3bbaaf196d0d994ff9076dc Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 17 May 2023 21:23:32 +0200
Subject: [PATCH 605/689] address comments

---
 docs/en/sql-reference/dictionaries/index.md   | 50 +++++++++----------
 ...4_regexp_dictionary_table_source.reference | 40 +++++++--------
 .../02504_regexp_dictionary_table_source.sql  | 14 +++---
 3 files changed, 50 insertions(+), 54 deletions(-)

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index 4abc41cdf42..eb45247e74a 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -2197,13 +2197,13 @@ Result:
 └─────────────────────────────────┴───────┘
 ```
 
-## RegExpTree Dictionary {#regexp-tree-dictionary}
+## Regular Expression Tree Dictionary {#regexp-tree-dictionary}
 
-RegExpTree Dictionary is designed to store multiple regular expressions in a dictionary, and query if a string could match one or multiple regular expressions. In some scenarios, for example, User Agent Parser, this data structure is very useful. We can use it in both local and cloud environments.
+Regular expression tree dictionaries are a special type of dictionary which represent the mapping from key to attributes using a tree of regular expressions. There are some use cases, e.g. parsing of (user agent)[https://en.wikipedia.org/wiki/User_agent] strings, which can be expressed elegantly with regexp tree dictionaries.
 
-### Use RegExpTree Dictionary in local environment
+### Use Regular Expression Tree Dictionary in ClickHouse Open-Source Environment
 
-In local environment, we create RegexpTree dictionary by a yaml file:
+Regular expression tree dictionaries are defined in ClickHouse open-source using the YAMLRegExpTree source which is provided the path to a YAML file containing the regular expression tree.
 
 ```sql
 create dictionary regexp_dict
@@ -2218,7 +2218,7 @@ LAYOUT(regexp_tree)
 ...
 ```
 
-The dictionary source `YAMLRegExpTree` represents the structure of a Regexp Tree. For example:
+The dictionary source `YAMLRegExpTree` represents the structure of a regexp tree. For example:
 
 ```yaml
 - regexp: 'Linux/(\d+[\.\d]*).+tlinux'
@@ -2226,7 +2226,7 @@ The dictionary source `YAMLRegExpTree` represents the structure of a Regexp Tree
   version: '\1'
 
 - regexp: '\d+/tclwebkit(?:\d+[\.\d]*)'
-  name: 'Andriod'
+  name: 'Android'
   versions:
     - regexp: '33/tclwebkit'
       version: '13'
@@ -2238,15 +2238,15 @@ The dictionary source `YAMLRegExpTree` represents the structure of a Regexp Tree
       version: '10'
 ```
 
-This config consists of a list of RegExpTree nodes. Each node has following structure:
+This config consists of a list of Regular Expression Tree nodes. Each node has following structure:
 
 - **regexp** means the regular expression of this node.
-- **user defined attributions** is a list of dictionary attributions defined in the dictionary structure. In this case, we have two attributions: `name` and `version`. The first nodes have both attributions. The second node only has `name` attribution, because the `version` is defined in the children nodes.
-  - The value of an attribution could contain a **back reference** which refers to a capture group of the matched regular expression. Reference number ranges from 1 to 9 and writes as `$1` or `\1`. During the query execution, the back reference in the value will be replaced by the matched capture group.
-- **children nodes** is the secondary layer of the RegExpTree nodes, which also contains a list of RegExpTree nodes. If a string matches a regexp node in the first layer, the dictionary will check if the string matches the children nodes of it. If it matches, we assign the attributions of the matching nodes. If two or more nodes define the same attribution, children nodes have more priority.
-  - the name of **children nodes** in yaml files can be arbitrary.
+- **user defined attributes** is a list of dictionary attributes defined in the dictionary structure. In this case, we have two attributes: `name` and `version`. The first node has both attributes. The second node only has `name` attribute, because the `version` is defined in the children nodes.
+  - The value of an attribute could contain a **back reference** which refers to a capture group of the matched regular expression. Reference number ranges from 1 to 9 and writes as `$1` or `\1`. During the query execution, the back reference in the value will be replaced by the matched capture group.
+- **children nodes** is the children of a regexp tree node, which has their own attributes and children nodes. String matching preceeds in a depth-first fasion. If a string matches any regexp node in the top layer, the dictionary checks if the string matches the children nodes of it. If it matches, we assign the attributes of the matching nodes. If two or more nodes define the same attribute, children nodes have more priority.
+  - the name of **children nodes** in YAML files can be arbitrary.
 
-Due to the specialty of Regexp Tree dictionary, we only allow functions `dictGet`, `dictGetOrDefault` and `dictGetOrNull`.
+Due to the specialty of regexp tree dictionary, we only allow functions `dictGet`, `dictGetOrDefault` and `dictGetOrNull`.
 
 Example:
 
@@ -2258,17 +2258,17 @@ Result:
 
 ```text
 ┌─dictGet('regexp_dict', ('name', 'version'), '31/tclwebkit1024')─┐
-│ ('Andriod','12')                                                │
+│ ('Android','12')                                                │
 └─────────────────────────────────────────────────────────────────┘
 ```
 
-Explain:
+In this case, we match the regular expression `\d+/tclwebkit(?:\d+[\.\d]*)` in the top layer's second node, so the dictionary continues to look into the children nodes and find it matches `3[12]/tclwebkit`. As a result, the value of `name` is `Android` defined in the first layer and the value of `version` is `12` defined the child node.
 
-In this case, we match the regular expression `\d+/tclwebkit(?:\d+[\.\d]*)` in the first layer, so the dictionary will continue to look into the children nodes in the second layer and find it matches `3[12]/tclwebkit`. As a result, the value of `name` is `Andriod` defined in the first layer and the value of `version` is `12` defined in the second layer.
+With a powerful YAML configure file, we can use RepexpTree dictionary as a UA parser. We support [uap-core](https://github.com/ua-parser/uap-core) and demonstrate how to use it in the functional test [02504_regexp_dictionary_ua_parser](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh)
 
-### Use RegExpTree Dictionary on cloud
+### Use Regular Expression Tree Dictionary in ClickHouse Cloud
 
-We have shown how RegExpTree work in the local environment, but we cannot use `YAMLRegExpTree` in the cloud. If we have a local yaml file, we can use this file to create RegExpTree Dictionary in the local environment, then dump this dictionary to a csv file by the `dictionary` table function and [INTO OUTFILE](../statements/select/into-outfile.md) clause.
+We have shown how Regular Expression Tree work in the local environment, but we cannot use `YAMLRegExpTree` in the cloud. If we have a local YAML file, we can use this file to create Regular Expression Tree Dictionary in the local environment, then dump this dictionary to a csv file by the `dictionary` table function and [INTO OUTFILE](../statements/select/into-outfile.md) clause.
 
 ```sql
 select * from dictionary(regexp_dict) into outfile('regexp_dict.csv')
@@ -2278,7 +2278,7 @@ The content of csv file is:
 
 ```text
 1,0,"Linux/(\d+[\.\d]*).+tlinux","['version','name']","['\\1','TencentOS']"
-2,0,"(\d+)/tclwebkit(\d+[\.\d]*)","['comment','version','name']","['test $1 and $2','$1','Andriod']"
+2,0,"(\d+)/tclwebkit(\d+[\.\d]*)","['comment','version','name']","['test $1 and $2','$1','Android']"
 3,2,"33/tclwebkit","['version']","['13']"
 4,2,"3[12]/tclwebkit","['version']","['12']"
 5,2,"3[12]/tclwebkit","['version']","['11']"
@@ -2287,11 +2287,11 @@ The content of csv file is:
 
 The schema of dumped file is always
 
-- `id UInt64` represents the identify number of the RegexpTree node.
+- `id UInt64` represents the id of the RegexpTree node.
 - `parent_id UInt64` represents the id of the parent of a node.
 - `regexp String` represents the regular expression string.
-- `keys Array(String)` represents the names of user defined attributions.
-- `values Array(String)` represents the values of user defined attributions.
+- `keys Array(String)` represents the names of user defined attributes.
+- `values Array(String)` represents the values of user defined attributes.
 
 On the cloud, we can create a table `regexp_dictionary_source_table` with the above table structure.
 
@@ -2314,8 +2314,8 @@ clickhouse client \
     --secure \
     --password MY_PASSWORD \
     --query "
-    insert into regexp_dictionary_source_table 
-    select * from input ('id UInt64, parent_id UInt64, regexp String, keys Array(String), values Array(String)') 
+    INSERT INTO regexp_dictionary_source_table 
+    SELECT * FROM input ('id UInt64, parent_id UInt64, regexp String, keys Array(String), values Array(String)') 
     FORMAT CSV" < regexp_dict.csv
 ```
 
@@ -2333,10 +2333,6 @@ LIFETIME(0)
 LAYOUT(regexp_tree);
 ```
 
-### Use RegexpTree Dictionary as a UA Parser
-
-With a powerful yaml configure file, we can use RepexpTree dictionary as a UA parser. We support [uap-core](https://github.com/ua-parser/uap-core) and demonstrate how to use it in the functional test [02504_regexp_dictionary_ua_parser](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh)
-
 ## Embedded Dictionaries {#embedded-dictionaries}
 
 <SelfManaged />
diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_table_source.reference b/tests/queries/0_stateless/02504_regexp_dictionary_table_source.reference
index 86a74291b07..4e72cf4ce37 100644
--- a/tests/queries/0_stateless/02504_regexp_dictionary_table_source.reference
+++ b/tests/queries/0_stateless/02504_regexp_dictionary_table_source.reference
@@ -1,11 +1,11 @@
 1	0	Linux/(\\d+[\\.\\d]*).+tlinux	['version','name']	['\\1','TencentOS']
-2	0	(\\d+)/tclwebkit(\\d+[\\.\\d]*)	['comment','version','name']	['test $1 and $2','$1','Andriod']
+2	0	(\\d+)/tclwebkit(\\d+[\\.\\d]*)	['comment','version','name']	['test $1 and $2','$1','Android']
 3	2	33/tclwebkit	['version']	['13']
 4	2	3[12]/tclwebkit	['version']	['12']
 5	2	3[12]/tclwebkit	['version']	['11']
 6	2	3[12]/tclwebkit	['version']	['10']
 ('TencentOS',101,'nothing')
-('Andriod',13,'test 33 and 11.10')
+('Android',13,'test 33 and 11.10')
 ('',NULL,'nothing')
 ('',0,'default')
 30/tclwebkit0
@@ -23,22 +23,22 @@
 42/tclwebkit12
 43/tclwebkit13
 44/tclwebkit14
-('Andriod',30)
-('Andriod',12)
-('Andriod',12)
-('Andriod',13)
-('Andriod',34)
-('Andriod',35)
-('Andriod',36)
-('Andriod',37)
-('Andriod',38)
-('Andriod',39)
-('Andriod',40)
-('Andriod',41)
-('Andriod',42)
-('Andriod',43)
-('Andriod',44)
-('Andriod1',33,'matched 3')
-1	0	(\\d+)/tclwebkit	['version','name']	['$1','Andriod']
+('Android',30)
+('Android',12)
+('Android',12)
+('Android',13)
+('Android',34)
+('Android',35)
+('Android',36)
+('Android',37)
+('Android',38)
+('Android',39)
+('Android',40)
+('Android',41)
+('Android',42)
+('Android',43)
+('Android',44)
+('Android1',33,'matched 3')
+1	0	(\\d+)/tclwebkit	['version','name']	['$1','Android']
 2	0	33/tclwebkit	['comment','version']	['matched 3','13']
-3	1	33/tclwebkit	['name']	['Andriod1']
+3	1	33/tclwebkit	['name']	['Android1']
diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql b/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql
index 15e8adce403..42d7acbf057 100644
--- a/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql
+++ b/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql
@@ -1,7 +1,7 @@
 -- Tags: use-vectorscan
 
-DROP TABLE IF EXISTS regexp_dictionary_source_table;
 DROP DICTIONARY IF EXISTS regexp_dict1;
+DROP TABLE IF EXISTS regexp_dictionary_source_table;
 
 CREATE TABLE regexp_dictionary_source_table
 (
@@ -15,7 +15,7 @@ CREATE TABLE regexp_dictionary_source_table
 -- test back reference.
 
 INSERT INTO regexp_dictionary_source_table VALUES (1, 0, 'Linux/(\d+[\.\d]*).+tlinux', ['name', 'version'], ['TencentOS', '\1'])
-INSERT INTO regexp_dictionary_source_table VALUES (2, 0, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Andriod', '$1', 'test $1 and $2'])
+INSERT INTO regexp_dictionary_source_table VALUES (2, 0, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Android', '$1', 'test $1 and $2'])
 INSERT INTO regexp_dictionary_source_table VALUES (3, 2, '33/tclwebkit', ['version'], ['13'])
 INSERT INTO regexp_dictionary_source_table VALUES (4, 2, '3[12]/tclwebkit', ['version'], ['12'])
 INSERT INTO regexp_dictionary_source_table VALUES (5, 2, '3[12]/tclwebkit', ['version'], ['11'])
@@ -65,14 +65,14 @@ SYSTEM RELOAD dictionary regexp_dict1; -- { serverError 489  }
 truncate table regexp_dictionary_source_table;
 
 INSERT INTO regexp_dictionary_source_table VALUES (1, 2, 'Linux/(\d+[\.\d]*).+tlinux', ['name', 'version'], ['TencentOS', '\1'])
-INSERT INTO regexp_dictionary_source_table VALUES (2, 3, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Andriod', '$1', 'test $1 and $2'])
-INSERT INTO regexp_dictionary_source_table VALUES (3, 1, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Andriod', '$1', 'test $1 and $2'])
+INSERT INTO regexp_dictionary_source_table VALUES (2, 3, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Android', '$1', 'test $1 and $2'])
+INSERT INTO regexp_dictionary_source_table VALUES (3, 1, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Android', '$1', 'test $1 and $2'])
 SYSTEM RELOAD dictionary regexp_dict1; -- { serverError 489  }
 
 -- test priority
 truncate table regexp_dictionary_source_table;
-INSERT INTO regexp_dictionary_source_table VALUES (1, 0, '(\d+)/tclwebkit', ['name', 'version'], ['Andriod', '$1']);
-INSERT INTO regexp_dictionary_source_table VALUES (3, 1, '33/tclwebkit', ['name'], ['Andriod1']); -- child has more priority than parents.
+INSERT INTO regexp_dictionary_source_table VALUES (1, 0, '(\d+)/tclwebkit', ['name', 'version'], ['Android', '$1']);
+INSERT INTO regexp_dictionary_source_table VALUES (3, 1, '33/tclwebkit', ['name'], ['Android1']); -- child has more priority than parents.
 INSERT INTO regexp_dictionary_source_table VALUES (2, 0, '33/tclwebkit', ['version', 'comment'], ['13', 'matched 3']); -- larger id has lower priority than small id.
 SYSTEM RELOAD dictionary regexp_dict1;
 select dictGet(regexp_dict1, ('name', 'version', 'comment'), '33/tclwebkit');
@@ -82,6 +82,6 @@ SYSTEM RELOAD dictionary regexp_dict1; -- { serverError 489 }
 
 select * from dictionary(regexp_dict1);
 
+DROP DICTIONARY IF EXISTS regexp_dict1;
 DROP TABLE IF EXISTS regexp_dictionary_source_table;
 DROP TABLE IF EXISTS needle_table;
-DROP DICTIONARY IF EXISTS regexp_dict1;

From 6a136897e375e61bf59a25c6dc65e4e2570e7641 Mon Sep 17 00:00:00 2001
From: rfraposa <richraposa@gmail.com>
Date: Wed, 17 May 2023 13:23:53 -0600
Subject: [PATCH 606/689] Create reddit-comments.md

---
 .../example-datasets/reddit-comments.md       | 636 ++++++++++++++++++
 1 file changed, 636 insertions(+)
 create mode 100644 docs/en/getting-started/example-datasets/reddit-comments.md

diff --git a/docs/en/getting-started/example-datasets/reddit-comments.md b/docs/en/getting-started/example-datasets/reddit-comments.md
new file mode 100644
index 00000000000..e1e372746c9
--- /dev/null
+++ b/docs/en/getting-started/example-datasets/reddit-comments.md
@@ -0,0 +1,636 @@
+---
+slug: /en/getting-started/example-datasets/reddit-comments
+sidebar_label: Reddit comments
+---
+
+# Reddit comments dataset
+
+This dataset contains publicly-available comments on Reddit that go back to December, 2005, to March, 2023, and contains over 7B rows of data. The raw data is in JSON format in compressed `.zst` files and the rows look like the following:
+
+```json
+{"controversiality":0,"body":"A look at Vietnam and Mexico exposes the myth of market liberalisation.","subreddit_id":"t5_6","link_id":"t3_17863","stickied":false,"subreddit":"reddit.com","score":2,"ups":2,"author_flair_css_class":null,"created_utc":1134365188,"author_flair_text":null,"author":"frjo","id":"c13","edited":false,"parent_id":"t3_17863","gilded":0,"distinguished":null,"retrieved_on":1473738411}
+{"created_utc":1134365725,"author_flair_css_class":null,"score":1,"ups":1,"subreddit":"reddit.com","stickied":false,"link_id":"t3_17866","subreddit_id":"t5_6","controversiality":0,"body":"The site states \"What can I use it for? Meeting notes, Reports, technical specs Sign-up sheets, proposals and much more...\", just like any other new breeed of sites that want us to store everything we have on the web. And they even guarantee multiple levels of security and encryption etc. But what prevents these web site operators fom accessing and/or stealing Meeting notes, Reports, technical specs Sign-up sheets, proposals and much more, for competitive or personal gains...? I am pretty sure that most of them are honest, but what's there to prevent me from setting up a good useful site and stealing all your data? Call me paranoid - I am.","retrieved_on":1473738411,"distinguished":null,"gilded":0,"id":"c14","edited":false,"parent_id":"t3_17866","author":"zse7zse","author_flair_text":null}
+{"gilded":0,"distinguished":null,"retrieved_on":1473738411,"author":"[deleted]","author_flair_text":null,"edited":false,"id":"c15","parent_id":"t3_17869","subreddit":"reddit.com","score":0,"ups":0,"created_utc":1134366848,"author_flair_css_class":null,"body":"Jython related topics by Frank Wierzbicki","controversiality":0,"subreddit_id":"t5_6","stickied":false,"link_id":"t3_17869"}
+{"gilded":0,"retrieved_on":1473738411,"distinguished":null,"author_flair_text":null,"author":"[deleted]","edited":false,"parent_id":"t3_17870","id":"c16","subreddit":"reddit.com","created_utc":1134367660,"author_flair_css_class":null,"score":1,"ups":1,"body":"[deleted]","controversiality":0,"stickied":false,"link_id":"t3_17870","subreddit_id":"t5_6"}
+{"gilded":0,"retrieved_on":1473738411,"distinguished":null,"author_flair_text":null,"author":"rjoseph","edited":false,"id":"c17","parent_id":"t3_17817","subreddit":"reddit.com","author_flair_css_class":null,"created_utc":1134367754,"score":1,"ups":1,"body":"Saft is by far the best extension you could tak onto your Safari","controversiality":0,"link_id":"t3_17817","stickied":false,"subreddit_id":"t5_6"}
+```
+
+A shoutout to Percona for the [motivation behind ingesting this dataset](https://www.percona.com/blog/big-data-set-reddit-comments-analyzing-clickhouse/), which we have downloaded and stored in an S3 bucket.
+
+:::note
+The following commands were executed on ClickHouse Cloud. To run this on your own cluster, replace `default` in the `s3Cluster` function call with the name of your cluster. If you do not have a cluster, then replace the `s3Cluster` function with the `s3` function.
+:::
+
+1. Let's create a table for the Reddit data:
+
+```sql
+CREATE TABLE reddit
+(
+    subreddit LowCardinality(String),
+    subreddit_id LowCardinality(String),
+    subreddit_type Enum('public' = 1, 'restricted' = 2, 'user' = 3, 'archived' = 4, 'gold_restricted' = 5, 'private' = 6),
+    author LowCardinality(String),
+    body String CODEC(ZSTD(6)),
+    created_date Date DEFAULT toDate(created_utc),
+    created_utc DateTime,
+    retrieved_on DateTime,
+    id String,
+    parent_id String,
+    link_id String,
+    score Int32,
+    total_awards_received UInt16,
+    controversiality UInt8,
+    gilded UInt8,
+    collapsed_because_crowd_control UInt8,
+    collapsed_reason Enum('' = 0, 'comment score below threshold' = 1, 'may be sensitive content' = 2, 'potentially toxic' = 3, 'potentially toxic content' = 4),
+    distinguished Enum('' = 0, 'moderator' = 1, 'admin' = 2, 'special' = 3),
+    removal_reason Enum('' = 0, 'legal' = 1),
+    author_created_utc DateTime,
+    author_fullname LowCardinality(String),
+    author_patreon_flair UInt8,
+    author_premium UInt8,
+    can_gild UInt8,
+    can_mod_post UInt8,
+    collapsed UInt8,
+    is_submitter UInt8,
+    _edited String,
+    locked UInt8,
+    quarantined UInt8,
+    no_follow UInt8,
+    send_replies UInt8,
+    stickied UInt8,
+    author_flair_text LowCardinality(String)
+)
+ENGINE = MergeTree
+ORDER BY (subreddit, created_date, author);
+```
+
+:::note
+The names of the files in S3 start with `RC_YYYY-MM` where `YYYY-MM` goes from `2005-12` to `2023-02`. The compression changes a couple of times though, so the file extensions are not consistent. For example:
+
+- the file names are initially `RC_2005-12.bz2` to `RC_2017-11.bz2`
+- then they look like `RC_2017-12.xz` to `RC_2018-09.xz`
+- and finally `RC_2018-10.zst` to `RC_2023-02.zst`
+:::
+
+2. We are going to start with one month of data, but if you want to simply insert every row - skip ahead to step 8 below. The following file has 86M records from December, 2017:
+
+```sql
+INSERT INTO reddit
+    SELECT *
+    FROM s3Cluster(
+        'default',
+        'https://clickhouse-public-datasets.s3.eu-central-1.amazonaws.com/reddit/original/RC_2017-12.xz',
+        'JSONEachRow'
+    );
+```
+
+If you do not have a cluster, use `s3` instead of `s3Cluster`:
+
+```sql
+INSERT INTO reddit
+    SELECT *
+    FROM s3(
+        'https://clickhouse-public-datasets.s3.eu-central-1.amazonaws.com/reddit/original/RC_2017-12.xz',
+        'JSONEachRow'
+    );
+```
+
+3. It will take a while depending on your resources, but when it's done verify it worked:
+
+```sql
+SELECT formatReadableQuantity(count())
+FROM reddit;
+```
+
+```response
+┌─formatReadableQuantity(count())─┐
+│ 85.97 million                   │
+└─────────────────────────────────┘
+```
+
+4. Let's see how many unique subreddits were in December of 2017:
+
+```sql
+SELECT uniqExact(subreddit)
+FROM reddit;
+```
+
+```response
+┌─uniqExact(subreddit)─┐
+│                91613 │
+└──────────────────────┘
+
+1 row in set. Elapsed: 1.572 sec. Processed 85.97 million rows, 367.43 MB (54.71 million rows/s., 233.80 MB/s.)
+```
+
+5. This query returns the top 10 subreddits (in terms of number of comments):
+
+```sql
+SELECT
+    subreddit,
+    count() AS c
+FROM reddit
+GROUP BY subreddit
+ORDER BY c DESC
+LIMIT 20;
+```
+
+```response
+┌─subreddit───────┬───────c─┐
+│ AskReddit       │ 5245881 │
+│ politics        │ 1753120 │
+│ nfl             │ 1220266 │
+│ nba             │  960388 │
+│ The_Donald      │  931857 │
+│ news            │  796617 │
+│ worldnews       │  765709 │
+│ CFB             │  710360 │
+│ gaming          │  602761 │
+│ movies          │  601966 │
+│ soccer          │  590628 │
+│ Bitcoin         │  583783 │
+│ pics            │  563408 │
+│ StarWars        │  562514 │
+│ funny           │  547563 │
+│ leagueoflegends │  517213 │
+│ teenagers       │  492020 │
+│ DestinyTheGame  │  477377 │
+│ todayilearned   │  472650 │
+│ videos          │  450581 │
+└─────────────────┴─────────┘
+
+20 rows in set. Elapsed: 0.368 sec. Processed 85.97 million rows, 367.43 MB (233.34 million rows/s., 997.25 MB/s.)
+```
+
+6. Here are the top 10 authors in December of 2017, in terms of number of comments posted:
+
+```sql
+SELECT
+    author,
+    count() AS c
+FROM reddit
+GROUP BY author
+ORDER BY c DESC
+LIMIT 10;
+```
+
+```response
+┌─author──────────┬───────c─┐
+│ [deleted]       │ 5913324 │
+│ AutoModerator   │  784886 │
+│ ImagesOfNetwork │   83241 │
+│ BitcoinAllBot   │   54484 │
+│ imguralbumbot   │   45822 │
+│ RPBot           │   29337 │
+│ WikiTextBot     │   25982 │
+│ Concise_AMA_Bot │   19974 │
+│ MTGCardFetcher  │   19103 │
+│ TotesMessenger  │   19057 │
+└─────────────────┴─────────┘
+
+10 rows in set. Elapsed: 8.143 sec. Processed 85.97 million rows, 711.05 MB (10.56 million rows/s., 87.32 MB/s.)
+```
+
+7.  We already inserted some data, but we will start over:
+
+```sql
+TRUNCATE TABLE reddit;
+```
+
+8. This is a fun dataset and it looks like we can find some great information, so let's go ahead and insert the entire dataset from 2005 to 2023. When you're ready, run this command to insert all the rows. (It takes a while - up to 17 hours!)
+
+```sql
+INSERT INTO reddit
+SELECT *
+FROM s3Cluster(
+    'default',
+    'https://clickhouse-public-datasets.s3.amazonaws.com/reddit/original/RC*',
+    'JSONEachRow'
+    )
+SETTINGS zstd_window_log_max = 31;
+```
+
+The response looks like:
+
+```response
+0 rows in set. Elapsed: 61187.839 sec. Processed 6.74 billion rows, 2.06 TB (110.17 thousand rows/s., 33.68 MB/s.)
+```
+
+8. Let's see how many rows were inserted and how much disk space the table is using:
+
+
+```sql
+SELECT
+    sum(rows) AS count,
+    formatReadableQuantity(count),
+    formatReadableSize(sum(bytes)) AS disk_size,
+    formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed_size
+FROM system.parts
+WHERE (table = 'reddit') AND active
+```
+
+Notice the compression of disk storage is about 1/3 of the uncompressed size:
+
+```response
+┌──────count─┬─formatReadableQuantity(sum(rows))─┬─disk_size──┬─uncompressed_size─┐
+│ 6739503568 │ 6.74 billion                      │ 501.10 GiB │ 1.51 TiB          │
+└────────────┴───────────────────────────────────┴────────────┴───────────────────┘
+
+1 row in set. Elapsed: 0.010 sec.
+```
+
+9. The following query shows how many comments, authors and subreddits we have for each month:
+
+```sql
+SELECT
+    toStartOfMonth(created_utc) AS firstOfMonth,
+    count() AS c,
+    bar(c, 0, 50000000, 25) AS bar_count,
+    uniq(author) AS authors,
+    bar(authors, 0, 5000000, 25) AS bar_authors,
+    uniq(subreddit) AS subreddits,
+    bar(subreddits, 0, 100000, 25) AS bar_subreddits
+FROM reddit
+GROUP BY firstOfMonth
+ORDER BY firstOfMonth ASC;
+```
+
+This is a substantial query that has to process all 6.74 billion rows, but we still get an impressive response time (about 3 minutes):
+
+```response
+┌─firstOfMonth─┬─────────c─┬─bar_count─────────────────┬─authors─┬─bar_authors───────────────┬─subreddits─┬─bar_subreddits────────────┐
+│   2005-12-01 │      1075 │                           │     394 │                           │          1 │                           │
+│   2006-01-01 │      3666 │                           │     791 │                           │          2 │                           │
+│   2006-02-01 │      9095 │                           │    1464 │                           │         18 │                           │
+│   2006-03-01 │     13859 │                           │    1958 │                           │         15 │                           │
+│   2006-04-01 │     19090 │                           │    2334 │                           │         21 │                           │
+│   2006-05-01 │     26859 │                           │    2698 │                           │         21 │                           │
+│   2006-06-01 │     29163 │                           │    3043 │                           │         19 │                           │
+│   2006-07-01 │     37031 │                           │    3532 │                           │         22 │                           │
+│   2006-08-01 │     50559 │                           │    4750 │                           │         24 │                           │
+│   2006-09-01 │     50675 │                           │    4908 │                           │         21 │                           │
+│   2006-10-01 │     54148 │                           │    5654 │                           │         31 │                           │
+│   2006-11-01 │     62021 │                           │    6490 │                           │         23 │                           │
+│   2006-12-01 │     61018 │                           │    6707 │                           │         24 │                           │
+│   2007-01-01 │     81341 │                           │    7931 │                           │         23 │                           │
+│   2007-02-01 │     95634 │                           │    9020 │                           │         21 │                           │
+│   2007-03-01 │    112444 │                           │   10842 │                           │         23 │                           │
+│   2007-04-01 │    126773 │                           │   10701 │                           │         26 │                           │
+│   2007-05-01 │    170097 │                           │   11365 │                           │         25 │                           │
+│   2007-06-01 │    178800 │                           │   11267 │                           │         22 │                           │
+│   2007-07-01 │    203319 │                           │   12482 │                           │         25 │                           │
+│   2007-08-01 │    225111 │                           │   14124 │                           │         30 │                           │
+│   2007-09-01 │    259497 │ ▏                         │   15416 │                           │         33 │                           │
+│   2007-10-01 │    274170 │ ▏                         │   15302 │                           │         36 │                           │
+│   2007-11-01 │    372983 │ ▏                         │   15134 │                           │         43 │                           │
+│   2007-12-01 │    363390 │ ▏                         │   15915 │                           │         31 │                           │
+│   2008-01-01 │    452990 │ ▏                         │   18857 │                           │        126 │                           │
+│   2008-02-01 │    441768 │ ▏                         │   18266 │                           │        173 │                           │
+│   2008-03-01 │    463728 │ ▏                         │   18947 │                           │        292 │                           │
+│   2008-04-01 │    468317 │ ▏                         │   18590 │                           │        323 │                           │
+│   2008-05-01 │    536380 │ ▎                         │   20861 │                           │        375 │                           │
+│   2008-06-01 │    577684 │ ▎                         │   22557 │                           │        575 │ ▏                         │
+│   2008-07-01 │    592610 │ ▎                         │   23123 │                           │        657 │ ▏                         │
+│   2008-08-01 │    595959 │ ▎                         │   23729 │                           │        707 │ ▏                         │
+│   2008-09-01 │    680892 │ ▎                         │   26374 │ ▏                         │        801 │ ▏                         │
+│   2008-10-01 │    789874 │ ▍                         │   28970 │ ▏                         │        893 │ ▏                         │
+│   2008-11-01 │    792310 │ ▍                         │   30272 │ ▏                         │       1024 │ ▎                         │
+│   2008-12-01 │    850359 │ ▍                         │   34073 │ ▏                         │       1103 │ ▎                         │
+│   2009-01-01 │   1051649 │ ▌                         │   38978 │ ▏                         │       1316 │ ▎                         │
+│   2009-02-01 │    944711 │ ▍                         │   43390 │ ▏                         │       1132 │ ▎                         │
+│   2009-03-01 │   1048643 │ ▌                         │   46516 │ ▏                         │       1203 │ ▎                         │
+│   2009-04-01 │   1094599 │ ▌                         │   48284 │ ▏                         │       1334 │ ▎                         │
+│   2009-05-01 │   1201257 │ ▌                         │   52512 │ ▎                         │       1395 │ ▎                         │
+│   2009-06-01 │   1258750 │ ▋                         │   57728 │ ▎                         │       1473 │ ▎                         │
+│   2009-07-01 │   1470290 │ ▋                         │   60098 │ ▎                         │       1686 │ ▍                         │
+│   2009-08-01 │   1750688 │ ▉                         │   67347 │ ▎                         │       1777 │ ▍                         │
+│   2009-09-01 │   2032276 │ █                         │   78051 │ ▍                         │       1784 │ ▍                         │
+│   2009-10-01 │   2242017 │ █                         │   93409 │ ▍                         │       2071 │ ▌                         │
+│   2009-11-01 │   2207444 │ █                         │   95940 │ ▍                         │       2141 │ ▌                         │
+│   2009-12-01 │   2560510 │ █▎                        │  104239 │ ▌                         │       2141 │ ▌                         │
+│   2010-01-01 │   2884096 │ █▍                        │  114314 │ ▌                         │       2313 │ ▌                         │
+│   2010-02-01 │   2687779 │ █▎                        │  115683 │ ▌                         │       2522 │ ▋                         │
+│   2010-03-01 │   3228254 │ █▌                        │  125775 │ ▋                         │       2890 │ ▋                         │
+│   2010-04-01 │   3209898 │ █▌                        │  128936 │ ▋                         │       3170 │ ▊                         │
+│   2010-05-01 │   3267363 │ █▋                        │  131851 │ ▋                         │       3166 │ ▊                         │
+│   2010-06-01 │   3532867 │ █▊                        │  139522 │ ▋                         │       3301 │ ▊                         │
+│   2010-07-01 │   4032737 │ ██                        │  153451 │ ▊                         │       3662 │ ▉                         │
+│   2010-08-01 │   4247982 │ ██                        │  164071 │ ▊                         │       3653 │ ▉                         │
+│   2010-09-01 │   4704069 │ ██▎                       │  186613 │ ▉                         │       4009 │ █                         │
+│   2010-10-01 │   5032368 │ ██▌                       │  203800 │ █                         │       4154 │ █                         │
+│   2010-11-01 │   5689002 │ ██▊                       │  226134 │ █▏                        │       4383 │ █                         │
+│   2010-12-01 │   5972642 │ ██▉                       │  245824 │ █▏                        │       4692 │ █▏                        │
+│   2011-01-01 │   6603329 │ ███▎                      │  270025 │ █▎                        │       5141 │ █▎                        │
+│   2011-02-01 │   6363114 │ ███▏                      │  277593 │ █▍                        │       5202 │ █▎                        │
+│   2011-03-01 │   7556165 │ ███▊                      │  314748 │ █▌                        │       5445 │ █▎                        │
+│   2011-04-01 │   7571398 │ ███▊                      │  329920 │ █▋                        │       6128 │ █▌                        │
+│   2011-05-01 │   8803949 │ ████▍                     │  365013 │ █▊                        │       6834 │ █▋                        │
+│   2011-06-01 │   9766511 │ ████▉                     │  393945 │ █▉                        │       7519 │ █▉                        │
+│   2011-07-01 │  10557466 │ █████▎                    │  424235 │ ██                        │       8293 │ ██                        │
+│   2011-08-01 │  12316144 │ ██████▏                   │  475326 │ ██▍                       │       9657 │ ██▍                       │
+│   2011-09-01 │  12150412 │ ██████                    │  503142 │ ██▌                       │      10278 │ ██▌                       │
+│   2011-10-01 │  13470278 │ ██████▋                   │  548801 │ ██▋                       │      10922 │ ██▋                       │
+│   2011-11-01 │  13621533 │ ██████▊                   │  574435 │ ██▊                       │      11572 │ ██▉                       │
+│   2011-12-01 │  14509469 │ ███████▎                  │  622849 │ ███                       │      12335 │ ███                       │
+│   2012-01-01 │  16350205 │ ████████▏                 │  696110 │ ███▍                      │      14281 │ ███▌                      │
+│   2012-02-01 │  16015695 │ ████████                  │  722892 │ ███▌                      │      14949 │ ███▋                      │
+│   2012-03-01 │  17881943 │ ████████▉                 │  789664 │ ███▉                      │      15795 │ ███▉                      │
+│   2012-04-01 │  19044534 │ █████████▌                │  842491 │ ████▏                     │      16440 │ ████                      │
+│   2012-05-01 │  20388260 │ ██████████▏               │  886176 │ ████▍                     │      16974 │ ████▏                     │
+│   2012-06-01 │  21897913 │ ██████████▉               │  946798 │ ████▋                     │      17952 │ ████▍                     │
+│   2012-07-01 │  24087517 │ ████████████              │ 1018636 │ █████                     │      19069 │ ████▊                     │
+│   2012-08-01 │  25703326 │ ████████████▊             │ 1094445 │ █████▍                    │      20553 │ █████▏                    │
+│   2012-09-01 │  23419524 │ ███████████▋              │ 1088491 │ █████▍                    │      20831 │ █████▏                    │
+│   2012-10-01 │  24788236 │ ████████████▍             │ 1131885 │ █████▋                    │      21868 │ █████▍                    │
+│   2012-11-01 │  24648302 │ ████████████▎             │ 1167608 │ █████▊                    │      21791 │ █████▍                    │
+│   2012-12-01 │  26080276 │ █████████████             │ 1218402 │ ██████                    │      22622 │ █████▋                    │
+│   2013-01-01 │  30365867 │ ███████████████▏          │ 1341703 │ ██████▋                   │      24696 │ ██████▏                   │
+│   2013-02-01 │  27213960 │ █████████████▌            │ 1304756 │ ██████▌                   │      24514 │ ██████▏                   │
+│   2013-03-01 │  30771274 │ ███████████████▍          │ 1391703 │ ██████▉                   │      25730 │ ██████▍                   │
+│   2013-04-01 │  33259557 │ ████████████████▋         │ 1485971 │ ███████▍                  │      27294 │ ██████▊                   │
+│   2013-05-01 │  33126225 │ ████████████████▌         │ 1506473 │ ███████▌                  │      27299 │ ██████▊                   │
+│   2013-06-01 │  32648247 │ ████████████████▎         │ 1506650 │ ███████▌                  │      27450 │ ██████▊                   │
+│   2013-07-01 │  34922133 │ █████████████████▍        │ 1561771 │ ███████▊                  │      28294 │ ███████                   │
+│   2013-08-01 │  34766579 │ █████████████████▍        │ 1589781 │ ███████▉                  │      28943 │ ███████▏                  │
+│   2013-09-01 │  31990369 │ ███████████████▉          │ 1570342 │ ███████▊                  │      29408 │ ███████▎                  │
+│   2013-10-01 │  35940040 │ █████████████████▉        │ 1683770 │ ████████▍                 │      30273 │ ███████▌                  │
+│   2013-11-01 │  37396497 │ ██████████████████▋       │ 1757467 │ ████████▊                 │      31173 │ ███████▊                  │
+│   2013-12-01 │  39810216 │ ███████████████████▉      │ 1846204 │ █████████▏                │      32326 │ ████████                  │
+│   2014-01-01 │  42420655 │ █████████████████████▏    │ 1927229 │ █████████▋                │      35603 │ ████████▉                 │
+│   2014-02-01 │  38703362 │ ███████████████████▎      │ 1874067 │ █████████▎                │      37007 │ █████████▎                │
+│   2014-03-01 │  42459956 │ █████████████████████▏    │ 1959888 │ █████████▊                │      37948 │ █████████▍                │
+│   2014-04-01 │  42440735 │ █████████████████████▏    │ 1951369 │ █████████▊                │      38362 │ █████████▌                │
+│   2014-05-01 │  42514094 │ █████████████████████▎    │ 1970197 │ █████████▊                │      39078 │ █████████▊                │
+│   2014-06-01 │  41990650 │ ████████████████████▉     │ 1943850 │ █████████▋                │      38268 │ █████████▌                │
+│   2014-07-01 │  46868899 │ ███████████████████████▍  │ 2059346 │ ██████████▎               │      40634 │ ██████████▏               │
+│   2014-08-01 │  46990813 │ ███████████████████████▍  │ 2117335 │ ██████████▌               │      41764 │ ██████████▍               │
+│   2014-09-01 │  44992201 │ ██████████████████████▍   │ 2124708 │ ██████████▌               │      41890 │ ██████████▍               │
+│   2014-10-01 │  47497520 │ ███████████████████████▋  │ 2206535 │ ███████████               │      43109 │ ██████████▊               │
+│   2014-11-01 │  46118074 │ ███████████████████████   │ 2239747 │ ███████████▏              │      43718 │ ██████████▉               │
+│   2014-12-01 │  48807699 │ ████████████████████████▍ │ 2372945 │ ███████████▊              │      43823 │ ██████████▉               │
+│   2015-01-01 │  53851542 │ █████████████████████████ │ 2499536 │ ████████████▍             │      47172 │ ███████████▊              │
+│   2015-02-01 │  48342747 │ ████████████████████████▏ │ 2448496 │ ████████████▏             │      47229 │ ███████████▊              │
+│   2015-03-01 │  54564441 │ █████████████████████████ │ 2550534 │ ████████████▊             │      48156 │ ████████████              │
+│   2015-04-01 │  55005780 │ █████████████████████████ │ 2609443 │ █████████████             │      49865 │ ████████████▍             │
+│   2015-05-01 │  54504410 │ █████████████████████████ │ 2585535 │ ████████████▉             │      50137 │ ████████████▌             │
+│   2015-06-01 │  54258492 │ █████████████████████████ │ 2595129 │ ████████████▉             │      49598 │ ████████████▍             │
+│   2015-07-01 │  58451788 │ █████████████████████████ │ 2720026 │ █████████████▌            │      55022 │ █████████████▊            │
+│   2015-08-01 │  58075327 │ █████████████████████████ │ 2743994 │ █████████████▋            │      55302 │ █████████████▊            │
+│   2015-09-01 │  55574825 │ █████████████████████████ │ 2672793 │ █████████████▎            │      53960 │ █████████████▍            │
+│   2015-10-01 │  59494045 │ █████████████████████████ │ 2816426 │ ██████████████            │      70210 │ █████████████████▌        │
+│   2015-11-01 │  57117500 │ █████████████████████████ │ 2847146 │ ██████████████▏           │      71363 │ █████████████████▊        │
+│   2015-12-01 │  58523312 │ █████████████████████████ │ 2854840 │ ██████████████▎           │      94559 │ ███████████████████████▋  │
+│   2016-01-01 │  61991732 │ █████████████████████████ │ 2920366 │ ██████████████▌           │     108438 │ █████████████████████████ │
+│   2016-02-01 │  59189875 │ █████████████████████████ │ 2854683 │ ██████████████▎           │     109916 │ █████████████████████████ │
+│   2016-03-01 │  63918864 │ █████████████████████████ │ 2969542 │ ██████████████▊           │      84787 │ █████████████████████▏    │
+│   2016-04-01 │  64271256 │ █████████████████████████ │ 2999086 │ ██████████████▉           │      61647 │ ███████████████▍          │
+│   2016-05-01 │  65212004 │ █████████████████████████ │ 3034674 │ ███████████████▏          │      67465 │ ████████████████▊         │
+│   2016-06-01 │  65867743 │ █████████████████████████ │ 3057604 │ ███████████████▎          │      75170 │ ██████████████████▊       │
+│   2016-07-01 │  66974735 │ █████████████████████████ │ 3199374 │ ███████████████▉          │      77732 │ ███████████████████▍      │
+│   2016-08-01 │  69654819 │ █████████████████████████ │ 3239957 │ ████████████████▏         │      63080 │ ███████████████▊          │
+│   2016-09-01 │  67024973 │ █████████████████████████ │ 3190864 │ ███████████████▉          │      62324 │ ███████████████▌          │
+│   2016-10-01 │  71826553 │ █████████████████████████ │ 3284340 │ ████████████████▍         │      62549 │ ███████████████▋          │
+│   2016-11-01 │  71022319 │ █████████████████████████ │ 3300822 │ ████████████████▌         │      69718 │ █████████████████▍        │
+│   2016-12-01 │  72942967 │ █████████████████████████ │ 3430324 │ █████████████████▏        │      71705 │ █████████████████▉        │
+│   2017-01-01 │  78946585 │ █████████████████████████ │ 3572093 │ █████████████████▊        │      78198 │ ███████████████████▌      │
+│   2017-02-01 │  70609487 │ █████████████████████████ │ 3421115 │ █████████████████         │      69823 │ █████████████████▍        │
+│   2017-03-01 │  79723106 │ █████████████████████████ │ 3638122 │ ██████████████████▏       │      73865 │ ██████████████████▍       │
+│   2017-04-01 │  77478009 │ █████████████████████████ │ 3620591 │ ██████████████████        │      74387 │ ██████████████████▌       │
+│   2017-05-01 │  79810360 │ █████████████████████████ │ 3650820 │ ██████████████████▎       │      74356 │ ██████████████████▌       │
+│   2017-06-01 │  79901711 │ █████████████████████████ │ 3737614 │ ██████████████████▋       │      72114 │ ██████████████████        │
+│   2017-07-01 │  81798725 │ █████████████████████████ │ 3872330 │ ███████████████████▎      │      76052 │ ███████████████████       │
+│   2017-08-01 │  84658503 │ █████████████████████████ │ 3960093 │ ███████████████████▊      │      77798 │ ███████████████████▍      │
+│   2017-09-01 │  83165192 │ █████████████████████████ │ 3880501 │ ███████████████████▍      │      78402 │ ███████████████████▌      │
+│   2017-10-01 │  85828912 │ █████████████████████████ │ 3980335 │ ███████████████████▉      │      80685 │ ████████████████████▏     │
+│   2017-11-01 │  84965681 │ █████████████████████████ │ 4026749 │ ████████████████████▏     │      82659 │ ████████████████████▋     │
+│   2017-12-01 │  85973810 │ █████████████████████████ │ 4196354 │ ████████████████████▉     │      91984 │ ██████████████████████▉   │
+│   2018-01-01 │  91558594 │ █████████████████████████ │ 4364443 │ █████████████████████▊    │     102577 │ █████████████████████████ │
+│   2018-02-01 │  86467179 │ █████████████████████████ │ 4277899 │ █████████████████████▍    │     104610 │ █████████████████████████ │
+│   2018-03-01 │  96490262 │ █████████████████████████ │ 4422470 │ ██████████████████████    │     112559 │ █████████████████████████ │
+│   2018-04-01 │  98101232 │ █████████████████████████ │ 4572434 │ ██████████████████████▊   │     105284 │ █████████████████████████ │
+│   2018-05-01 │ 100109100 │ █████████████████████████ │ 4698908 │ ███████████████████████▍  │     103910 │ █████████████████████████ │
+│   2018-06-01 │ 100009462 │ █████████████████████████ │ 4697426 │ ███████████████████████▍  │     101107 │ █████████████████████████ │
+│   2018-07-01 │ 108151359 │ █████████████████████████ │ 5099492 │ █████████████████████████ │     106184 │ █████████████████████████ │
+│   2018-08-01 │ 107330940 │ █████████████████████████ │ 5084082 │ █████████████████████████ │     109985 │ █████████████████████████ │
+│   2018-09-01 │ 104473929 │ █████████████████████████ │ 5011953 │ █████████████████████████ │     109710 │ █████████████████████████ │
+│   2018-10-01 │ 112346556 │ █████████████████████████ │ 5320405 │ █████████████████████████ │     112533 │ █████████████████████████ │
+│   2018-11-01 │ 112573001 │ █████████████████████████ │ 5353282 │ █████████████████████████ │     112211 │ █████████████████████████ │
+│   2018-12-01 │ 121953600 │ █████████████████████████ │ 5611543 │ █████████████████████████ │     118291 │ █████████████████████████ │
+│   2019-01-01 │ 129386587 │ █████████████████████████ │ 6016687 │ █████████████████████████ │     125725 │ █████████████████████████ │
+│   2019-02-01 │ 120645639 │ █████████████████████████ │ 5974488 │ █████████████████████████ │     125420 │ █████████████████████████ │
+│   2019-03-01 │ 137650471 │ █████████████████████████ │ 6410197 │ █████████████████████████ │     135924 │ █████████████████████████ │
+│   2019-04-01 │ 138473643 │ █████████████████████████ │ 6416384 │ █████████████████████████ │     139844 │ █████████████████████████ │
+│   2019-05-01 │ 142463421 │ █████████████████████████ │ 6574836 │ █████████████████████████ │     142012 │ █████████████████████████ │
+│   2019-06-01 │ 134172939 │ █████████████████████████ │ 6601267 │ █████████████████████████ │     140997 │ █████████████████████████ │
+│   2019-07-01 │ 145965083 │ █████████████████████████ │ 6901822 │ █████████████████████████ │     147802 │ █████████████████████████ │
+│   2019-08-01 │ 146854393 │ █████████████████████████ │ 6993882 │ █████████████████████████ │     151888 │ █████████████████████████ │
+│   2019-09-01 │ 137540219 │ █████████████████████████ │ 7001362 │ █████████████████████████ │     148839 │ █████████████████████████ │
+│   2019-10-01 │ 129771456 │ █████████████████████████ │ 6825690 │ █████████████████████████ │     144453 │ █████████████████████████ │
+│   2019-11-01 │ 107990259 │ █████████████████████████ │ 6368286 │ █████████████████████████ │     141768 │ █████████████████████████ │
+│   2019-12-01 │ 112895934 │ █████████████████████████ │ 6640902 │ █████████████████████████ │     148277 │ █████████████████████████ │
+│   2020-01-01 │  54354879 │ █████████████████████████ │ 4782339 │ ███████████████████████▉  │     111658 │ █████████████████████████ │
+│   2020-02-01 │  22696923 │ ███████████▎              │ 3135175 │ ███████████████▋          │      79521 │ ███████████████████▉      │
+│   2020-03-01 │   3466677 │ █▋                        │  987960 │ ████▉                     │      40901 │ ██████████▏               │
+└──────────────┴───────────┴───────────────────────────┴─────────┴───────────────────────────┴────────────┴───────────────────────────┘
+
+172 rows in set. Elapsed: 184.809 sec. Processed 6.74 billion rows, 89.56 GB (36.47 million rows/s., 484.62 MB/s.)
+```
+
+10. Here are the top 10 subreddits of 2022:
+
+```sql
+SELECT
+    subreddit,
+    count() AS count
+FROM reddit
+WHERE toYear(created_utc) = 2022
+GROUP BY subreddit
+ORDER BY count DESC
+LIMIT 10;
+```
+
+The response is:
+
+```response
+┌─subreddit────────┬───count─┐
+│ AskReddit        │ 3858203 │
+│ politics         │ 1356782 │
+│ memes            │ 1249120 │
+│ nfl              │  883667 │
+│ worldnews        │  866065 │
+│ teenagers        │  777095 │
+│ AmItheAsshole    │  752720 │
+│ dankmemes        │  657932 │
+│ nba              │  514184 │
+│ unpopularopinion │  473649 │
+└──────────────────┴─────────┘
+
+10 rows in set. Elapsed: 27.824 sec. Processed 6.74 billion rows, 53.26 GB (242.22 million rows/s., 1.91 GB/s.)
+```
+
+11. Let's see which subreddits had the biggest increase in commnents from 2018 to 2019:
+
+```sql
+SELECT
+    subreddit,
+    newcount - oldcount AS diff
+FROM
+(
+    SELECT
+        subreddit,
+        count(*) AS newcount
+    FROM reddit
+    WHERE toYear(created_utc) = 2019
+    GROUP BY subreddit
+)
+ALL INNER JOIN
+(
+    SELECT
+        subreddit,
+        count(*) AS oldcount
+    FROM reddit
+    WHERE toYear(created_utc) = 2018
+    GROUP BY subreddit
+) USING (subreddit)
+ORDER BY diff DESC
+LIMIT 50
+SETTINGS joined_subquery_requires_alias = 0;
+```
+
+It looks like memes and teenagers were busy on Reddit in 2019:
+
+```response
+┌─subreddit────────────┬─────diff─┐
+│ memes                │ 15368369 │
+│ AskReddit            │ 14663662 │
+│ teenagers            │ 12266991 │
+│ AmItheAsshole        │ 11561538 │
+│ dankmemes            │ 11305158 │
+│ unpopularopinion     │  6332772 │
+│ PewdiepieSubmissions │  5930818 │
+│ Market76             │  5014668 │
+│ relationship_advice  │  3776383 │
+│ freefolk             │  3169236 │
+│ Minecraft            │  3160241 │
+│ classicwow           │  2907056 │
+│ Animemes             │  2673398 │
+│ gameofthrones        │  2402835 │
+│ PublicFreakout       │  2267605 │
+│ ShitPostCrusaders    │  2207266 │
+│ RoastMe              │  2195715 │
+│ gonewild             │  2148649 │
+│ AnthemTheGame        │  1803818 │
+│ entitledparents      │  1706270 │
+│ MortalKombat         │  1679508 │
+│ Cringetopia          │  1620555 │
+│ pokemon              │  1615266 │
+│ HistoryMemes         │  1608289 │
+│ Brawlstars           │  1574977 │
+│ iamatotalpieceofshit │  1558315 │
+│ trashy               │  1518549 │
+│ ChapoTrapHouse       │  1505748 │
+│ Pikabu               │  1501001 │
+│ Showerthoughts       │  1475101 │
+│ cursedcomments       │  1465607 │
+│ ukpolitics           │  1386043 │
+│ wallstreetbets       │  1384431 │
+│ interestingasfuck    │  1378900 │
+│ wholesomememes       │  1353333 │
+│ AskOuija             │  1233263 │
+│ borderlands3         │  1197192 │
+│ aww                  │  1168257 │
+│ insanepeoplefacebook │  1155473 │
+│ FortniteCompetitive  │  1122778 │
+│ EpicSeven            │  1117380 │
+│ FreeKarma4U          │  1116423 │
+│ YangForPresidentHQ   │  1086700 │
+│ SquaredCircle        │  1044089 │
+│ MurderedByWords      │  1042511 │
+│ AskMen               │  1024434 │
+│ thedivision          │  1016634 │
+│ barstoolsports       │   985032 │
+│ nfl                  │   978340 │
+│ BattlefieldV         │   971408 │
+└──────────────────────┴──────────┘
+
+50 rows in set. Elapsed: 65.954 sec. Processed 13.48 billion rows, 79.67 GB (204.37 million rows/s., 1.21 GB/s.)
+```
+
+12. One more query: let's compare ClickHouse mentions to other technologies like Snowflake and Postgres. This query is a big one because it has to search all the comments three times for a substring, and unfortunately ClickHouse user are obviously not very active on Reddit yet:
+
+```sql
+SELECT
+    toStartOfQuarter(created_utc) AS quarter,
+    sum(if(positionCaseInsensitive(body, 'clickhouse') > 0, 1, 0)) AS clickhouse,
+    sum(if(positionCaseInsensitive(body, 'snowflake') > 0, 1, 0)) AS snowflake,
+    sum(if(positionCaseInsensitive(body, 'postgres') > 0, 1, 0)) AS postgres
+FROM reddit
+GROUP BY quarter
+ORDER BY quarter ASC;
+```
+
+```response
+┌────Quarter─┬─clickhouse─┬─snowflake─┬─postgres─┐
+│ 2005-10-01 │          0 │         0 │        0 │
+│ 2006-01-01 │          0 │         2 │       23 │
+│ 2006-04-01 │          0 │         2 │       24 │
+│ 2006-07-01 │          0 │         4 │       13 │
+│ 2006-10-01 │          0 │        23 │       73 │
+│ 2007-01-01 │          0 │        14 │       91 │
+│ 2007-04-01 │          0 │        10 │       59 │
+│ 2007-07-01 │          0 │        39 │      116 │
+│ 2007-10-01 │          0 │        45 │      125 │
+│ 2008-01-01 │          0 │        53 │      234 │
+│ 2008-04-01 │          0 │        79 │      303 │
+│ 2008-07-01 │          0 │       102 │      174 │
+│ 2008-10-01 │          0 │       156 │      323 │
+│ 2009-01-01 │          0 │       206 │      208 │
+│ 2009-04-01 │          0 │       178 │      417 │
+│ 2009-07-01 │          0 │       300 │      295 │
+│ 2009-10-01 │          0 │       633 │      589 │
+│ 2010-01-01 │          0 │       555 │      501 │
+│ 2010-04-01 │          0 │       587 │      469 │
+│ 2010-07-01 │          0 │       770 │      821 │
+│ 2010-10-01 │          0 │      1480 │      550 │
+│ 2011-01-01 │          0 │      1482 │      568 │
+│ 2011-04-01 │          0 │      1558 │      406 │
+│ 2011-07-01 │          0 │      2163 │      628 │
+│ 2011-10-01 │          0 │      4064 │      566 │
+│ 2012-01-01 │          0 │      4621 │      662 │
+│ 2012-04-01 │          0 │      5737 │      785 │
+│ 2012-07-01 │          0 │      6097 │     1127 │
+│ 2012-10-01 │          0 │      7986 │      600 │
+│ 2013-01-01 │          0 │      9704 │      839 │
+│ 2013-04-01 │          0 │      8161 │      853 │
+│ 2013-07-01 │          0 │      9704 │     1028 │
+│ 2013-10-01 │          0 │     12879 │     1404 │
+│ 2014-01-01 │          0 │     12317 │     1548 │
+│ 2014-04-01 │          0 │     13181 │     1577 │
+│ 2014-07-01 │          0 │     15640 │     1710 │
+│ 2014-10-01 │          0 │     19479 │     1959 │
+│ 2015-01-01 │          0 │     20411 │     2104 │
+│ 2015-04-01 │          1 │     20309 │     9112 │
+│ 2015-07-01 │          0 │     20325 │     4771 │
+│ 2015-10-01 │          0 │     25087 │     3030 │
+│ 2016-01-01 │          0 │     23462 │     3126 │
+│ 2016-04-01 │          3 │     25496 │     2757 │
+│ 2016-07-01 │          4 │     28233 │     2928 │
+│ 2016-10-01 │          2 │     45445 │     2449 │
+│ 2017-01-01 │          9 │     76019 │     2808 │
+│ 2017-04-01 │          9 │     67919 │     2803 │
+│ 2017-07-01 │         13 │     68974 │     2771 │
+│ 2017-10-01 │         12 │     69730 │     2906 │
+│ 2018-01-01 │         17 │     67476 │     3152 │
+│ 2018-04-01 │          3 │     67139 │     3986 │
+│ 2018-07-01 │         14 │     67979 │     3609 │
+│ 2018-10-01 │         28 │     74147 │     3850 │
+│ 2019-01-01 │         14 │     80250 │     4305 │
+│ 2019-04-01 │         30 │     70307 │     3872 │
+│ 2019-07-01 │         33 │     77149 │     4164 │
+│ 2019-10-01 │         13 │     76746 │     3541 │
+│ 2020-01-01 │         16 │     54475 │      846 │
+└────────────┴────────────┴───────────┴──────────┘
+
+58 rows in set. Elapsed: 2663.751 sec. Processed 6.74 billion rows, 1.21 TB (2.53 million rows/s., 454.37 MB/s.)
+```
\ No newline at end of file

From e2e3a03dbe1e1dc5a4f2d39532bf072c9e6bffa3 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 17 May 2023 22:33:30 +0200
Subject: [PATCH 607/689] Revert "`groupArray` returns cannot be nullable"

---
 src/AggregateFunctions/AggregateFunctionGroupArray.cpp       | 2 +-
 src/AggregateFunctions/AggregateFunctionNull.cpp             | 2 +-
 tests/queries/0_stateless/00529_orantius.reference           | 2 +-
 tests/queries/0_stateless/01664_array_slice_ubsan.reference  | 2 +-
 .../queries/0_stateless/02713_group_array_nullable.reference | 3 ---
 tests/queries/0_stateless/02713_group_array_nullable.sql     | 5 -----
 6 files changed, 4 insertions(+), 12 deletions(-)
 delete mode 100644 tests/queries/0_stateless/02713_group_array_nullable.reference
 delete mode 100644 tests/queries/0_stateless/02713_group_array_nullable.sql

diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp
index 67cfa3f7356..15f500b8bb6 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp
@@ -121,7 +121,7 @@ AggregateFunctionPtr createAggregateFunctionGroupArraySample(
 
 void registerAggregateFunctionGroupArray(AggregateFunctionFactory & factory)
 {
-    AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = true };
+    AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
 
     factory.registerFunction("groupArray", { createAggregateFunctionGroupArray<false>, properties });
     factory.registerFunction("groupArraySample", { createAggregateFunctionGroupArraySample, properties });
diff --git a/src/AggregateFunctions/AggregateFunctionNull.cpp b/src/AggregateFunctions/AggregateFunctionNull.cpp
index 19c66db98cd..3d3d7af3026 100644
--- a/src/AggregateFunctions/AggregateFunctionNull.cpp
+++ b/src/AggregateFunctions/AggregateFunctionNull.cpp
@@ -72,7 +72,7 @@ public:
         {
             /// Currently the only functions that returns not-NULL on all NULL arguments are count and uniq, and they returns UInt64.
             if (properties.returns_default_when_only_null)
-                return std::make_shared<AggregateFunctionNothing>(arguments, params, nested_function->getResultType());
+                return std::make_shared<AggregateFunctionNothing>(arguments, params, std::make_shared<DataTypeUInt64>());
             else
                 return std::make_shared<AggregateFunctionNothing>(arguments, params, std::make_shared<DataTypeNullable>(std::make_shared<DataTypeNothing>()));
         }
diff --git a/tests/queries/0_stateless/00529_orantius.reference b/tests/queries/0_stateless/00529_orantius.reference
index 1deecf44752..865659048cc 100644
--- a/tests/queries/0_stateless/00529_orantius.reference
+++ b/tests/queries/0_stateless/00529_orantius.reference
@@ -3,7 +3,7 @@
 1
 1
 [[1],[-1]]
-[]
+\N
 1
 42	42
 [NULL,'','',NULL]
diff --git a/tests/queries/0_stateless/01664_array_slice_ubsan.reference b/tests/queries/0_stateless/01664_array_slice_ubsan.reference
index beb31c4a02b..abae410d95c 100644
--- a/tests/queries/0_stateless/01664_array_slice_ubsan.reference
+++ b/tests/queries/0_stateless/01664_array_slice_ubsan.reference
@@ -1 +1 @@
-['\0','\0','\0']
+[0,0,0]
diff --git a/tests/queries/0_stateless/02713_group_array_nullable.reference b/tests/queries/0_stateless/02713_group_array_nullable.reference
deleted file mode 100644
index a163ccade3a..00000000000
--- a/tests/queries/0_stateless/02713_group_array_nullable.reference
+++ /dev/null
@@ -1,3 +0,0 @@
-[1]
-[0,1,2,3,4,5,6,7,8,9]
-[8,9]
diff --git a/tests/queries/0_stateless/02713_group_array_nullable.sql b/tests/queries/0_stateless/02713_group_array_nullable.sql
deleted file mode 100644
index a0cb3075252..00000000000
--- a/tests/queries/0_stateless/02713_group_array_nullable.sql
+++ /dev/null
@@ -1,5 +0,0 @@
-SET aggregate_functions_null_for_empty = 1;
-	
-SELECT groupArray(1);
-SELECT groupArray(number) FROM numbers(10);
-SELECT groupArrayLast(2)(number) FROM numbers(10);

From 612b79868b6a5e9f8c91ddf0caea6f72e97c0320 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Wed, 17 May 2023 20:40:51 +0000
Subject: [PATCH 608/689] test added

---
 .../0_stateless/02751_protobuf_ipv6.reference      |  2 ++
 tests/queries/0_stateless/02751_protobuf_ipv6.sh   | 14 ++++++++++++++
 .../format_schemas/02751_protobuf_ipv6.proto       |  6 ++++++
 3 files changed, 22 insertions(+)
 create mode 100644 tests/queries/0_stateless/02751_protobuf_ipv6.reference
 create mode 100755 tests/queries/0_stateless/02751_protobuf_ipv6.sh
 create mode 100644 tests/queries/0_stateless/format_schemas/02751_protobuf_ipv6.proto

diff --git a/tests/queries/0_stateless/02751_protobuf_ipv6.reference b/tests/queries/0_stateless/02751_protobuf_ipv6.reference
new file mode 100644
index 00000000000..0318b49c77e
--- /dev/null
+++ b/tests/queries/0_stateless/02751_protobuf_ipv6.reference
@@ -0,0 +1,2 @@
+::ffff:1.2.3.4
+::ffff:1.2.3.4
diff --git a/tests/queries/0_stateless/02751_protobuf_ipv6.sh b/tests/queries/0_stateless/02751_protobuf_ipv6.sh
new file mode 100755
index 00000000000..ecf565d9db4
--- /dev/null
+++ b/tests/queries/0_stateless/02751_protobuf_ipv6.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+SCHEMADIR=$CURDIR/format_schemas
+
+
+echo 121a1000000000000000000000ffff01020304 | xxd -r -p | $CLICKHOUSE_LOCAL --input-format Protobuf --format_schema="$SCHEMADIR/02751_protobuf_ipv6:Message" --structure="ipv6_bytes IPv6" -q "select * from table"
+
+$CLICKHOUSE_LOCAL -q "select '::ffff:1.2.3.4'::IPv6 as ipv6_bytes format Protobuf settings format_schema = '$SCHEMADIR/02751_protobuf_ipv6:Message'" | $CLICKHOUSE_LOCAL --input-format Protobuf --format_schema="$SCHEMADIR/02751_protobuf_ipv6:Message" --structure="ipv6_bytes IPv6"  -q "select * from table"
+
diff --git a/tests/queries/0_stateless/format_schemas/02751_protobuf_ipv6.proto b/tests/queries/0_stateless/format_schemas/02751_protobuf_ipv6.proto
new file mode 100644
index 00000000000..8e6f115f2d7
--- /dev/null
+++ b/tests/queries/0_stateless/format_schemas/02751_protobuf_ipv6.proto
@@ -0,0 +1,6 @@
+syntax = "proto3";
+
+message Message
+{
+    bytes ipv6_bytes = 3;
+}

From 855c95f6268b1f31cdadb4b55b4b4ada5db7847a Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Wed, 17 May 2023 22:46:09 +0200
Subject: [PATCH 609/689] Update src/Interpreters/Cache/Metadata.cpp

Co-authored-by: Igor Nikonov <954088+devcrafter@users.noreply.github.com>
---
 src/Interpreters/Cache/Metadata.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp
index e0b82763a08..843ffd45b63 100644
--- a/src/Interpreters/Cache/Metadata.cpp
+++ b/src/Interpreters/Cache/Metadata.cpp
@@ -284,7 +284,7 @@ void CacheMetadata::doCleanup()
         {
             /// Key prefix directory can become non-empty just now, it is expected.
             if (e.code() == std::errc::directory_not_empty)
-                return;
+                continue;
             LOG_ERROR(log, "Error while removing key {}: {}", cleanup_key, getCurrentExceptionMessage(true));
             chassert(false);
         }

From d294ecbc162e129e6f63a1d9ec4c9c80b994cddb Mon Sep 17 00:00:00 2001
From: libin <imbingo123@gmail.com>
Date: Thu, 18 May 2023 15:50:19 +0800
Subject: [PATCH 610/689] Update grant.md

docs: Modifying grant example
---
 docs/zh/sql-reference/statements/grant.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/zh/sql-reference/statements/grant.md b/docs/zh/sql-reference/statements/grant.md
index 12ad2e0fe25..7e7cdbff350 100644
--- a/docs/zh/sql-reference/statements/grant.md
+++ b/docs/zh/sql-reference/statements/grant.md
@@ -55,7 +55,7 @@ GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION
 
 同样 `john` 有权执行 `GRANT OPTION`，因此他能给其它账号进行和自己账号权限范围相同的授权。
 
-可以使用`*` 号代替表或库名进行授权操作。例如， `GRANT SELECT ONdb.* TO john` 操作运行 `john`对 `db`库的所有表执行 `SELECT`查询。同样，你可以忽略库名。在这种情形下，权限将指向当前的数据库。例如， `GRANT SELECT ON* to john` 对当前数据库的所有表指定授权， `GARNT SELECT ON mytable to john`对当前数据库的 `mytable`表进行授权。
+可以使用`*` 号代替表或库名进行授权操作。例如， `GRANT SELECT ONdb.* TO john` 操作运行 `john`对 `db`库的所有表执行 `SELECT`查询。同样，你可以忽略库名。在这种情形下，权限将指向当前的数据库。例如， `GRANT SELECT ON* to john` 对当前数据库的所有表指定授权， `GRANT SELECT ON mytable to john`对当前数据库的 `mytable`表进行授权。
 
 访问 `systen`数据库总是被允许的（因为这个数据库用来处理sql操作）
 可以一次给多个账号进行多种授权操作。 `GRANT SELECT,INSERT ON *.* TO john,robin` 允许 `john`和`robin` 账号对任意数据库的任意表执行 `INSERT`和 `SELECT`操作。

From f98c337d2f9580fa65c8d21de447ab6e8fe3d781 Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Thu, 18 May 2023 14:53:46 +0200
Subject: [PATCH 611/689] Fix stack-use-after-scope in resource manager test
 (#49908)

* Fix stack-use-after-scope in resource manager test

* fix
---
 .../gtest_resource_manager_hierarchical.cpp   | 37 +++++++++----------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/IO/Resource/tests/gtest_resource_manager_hierarchical.cpp b/src/IO/Resource/tests/gtest_resource_manager_hierarchical.cpp
index b113da31d59..43773559f03 100644
--- a/src/IO/Resource/tests/gtest_resource_manager_hierarchical.cpp
+++ b/src/IO/Resource/tests/gtest_resource_manager_hierarchical.cpp
@@ -47,9 +47,18 @@ TEST(IOResourceDynamicResourceManager, Smoke)
 
 TEST(IOResourceDynamicResourceManager, Fairness)
 {
-    constexpr size_t T = 3; // threads per queue
-    int N = 100; // requests per thread
-    ResourceTest t(2 * T + 1);
+    // Total cost for A and B cannot differ for more than 1 (every request has cost equal to 1).
+    // Requests from A use `value = 1` and from B `value = -1` is used.
+    std::atomic<Int64> unfairness = 0;
+    auto fairness_diff = [&] (Int64 value)
+    {
+        Int64 cur_unfairness = unfairness.fetch_add(value, std::memory_order_relaxed) + value;
+        EXPECT_NEAR(cur_unfairness, 0, 1);
+    };
+
+    constexpr size_t threads_per_queue = 3;
+    int requests_per_thread = 100;
+    ResourceTest t(2 * threads_per_queue + 1);
 
     t.update(R"CONFIG(
         <clickhouse>
@@ -70,24 +79,14 @@ TEST(IOResourceDynamicResourceManager, Fairness)
         </clickhouse>
     )CONFIG");
 
-
-    // Total cost for A and B cannot differ for more than 1 (every request has cost equal to 1).
-    // Requests from A use `value = 1` and from B `value = -1` is used.
-    std::atomic<Int64> unfairness = 0;
-    auto fairness_diff = [&] (Int64 value)
-    {
-        Int64 cur_unfairness = unfairness.fetch_add(value, std::memory_order_relaxed) + value;
-        EXPECT_NEAR(cur_unfairness, 0, 1);
-    };
-
-    for (int thr = 0; thr < T; thr++)
+    for (int thread = 0; thread < threads_per_queue; thread++)
     {
         t.threads.emplace_back([&]
         {
             ClassifierPtr c = t.manager->acquire("A");
             ResourceLink link = c->get("res1");
-            t.startBusyPeriod(link, 1, N);
-            for (int req = 0; req < N; req++)
+            t.startBusyPeriod(link, 1, requests_per_thread);
+            for (int request = 0; request < requests_per_thread; request++)
             {
                 TestGuard g(t, link, 1);
                 fairness_diff(1);
@@ -95,14 +94,14 @@ TEST(IOResourceDynamicResourceManager, Fairness)
         });
     }
 
-    for (int thr = 0; thr < T; thr++)
+    for (int thread = 0; thread < threads_per_queue; thread++)
     {
         t.threads.emplace_back([&]
         {
             ClassifierPtr c = t.manager->acquire("B");
             ResourceLink link = c->get("res1");
-            t.startBusyPeriod(link, 1, N);
-            for (int req = 0; req < N; req++)
+            t.startBusyPeriod(link, 1, requests_per_thread);
+            for (int request = 0; request < requests_per_thread; request++)
             {
                 TestGuard g(t, link, 1);
                 fairness_diff(-1);

From 94fe22493527634853febfb848260c084c17a368 Mon Sep 17 00:00:00 2001
From: Denny Crane <denis.zhuravlov@gmail.com>
Date: Thu, 18 May 2023 10:06:59 -0300
Subject: [PATCH 612/689] Update partition.md

---
 docs/en/sql-reference/statements/alter/partition.md | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md
index ce0bc1ea528..a0aa74e6d25 100644
--- a/docs/en/sql-reference/statements/alter/partition.md
+++ b/docs/en/sql-reference/statements/alter/partition.md
@@ -284,13 +284,17 @@ Manipulates data in the specifies partition matching the specified filtering exp
 Syntax:
 
 ``` sql
-ALTER TABLE [db.]table [ON CLUSTER cluster] UPDATE column1 = expr1 [, ...] [IN PARTITION partition_id] WHERE filter_expr
+ALTER TABLE [db.]table [ON CLUSTER cluster] UPDATE column1 = expr1 [, ...] [IN PARTITION partition_expr] WHERE filter_expr
 ```
 
 ### Example
 
 ``` sql
+-- using partition name
 ALTER TABLE mt UPDATE x = x + 1 IN PARTITION 2 WHERE p = 2;
+
+-- using partition id
+ALTER TABLE mt UPDATE x = x + 1 IN PARTITION ID '2' WHERE p = 2;
 ```
 
 ### See Also
@@ -304,13 +308,17 @@ Deletes data in the specifies partition matching the specified filtering express
 Syntax:
 
 ``` sql
-ALTER TABLE [db.]table [ON CLUSTER cluster] DELETE [IN PARTITION partition_id] WHERE filter_expr
+ALTER TABLE [db.]table [ON CLUSTER cluster] DELETE [IN PARTITION partition_expr] WHERE filter_expr
 ```
 
 ### Example
 
 ``` sql
+-- using partition name
 ALTER TABLE mt DELETE IN PARTITION 2 WHERE p = 2;
+
+-- using partition id
+ALTER TABLE mt DELETE IN PARTITION ID '2' WHERE p = 2;
 ```
 
 ### See Also

From 30083351f5769d781a2dacccc8c259e846518956 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Thu, 18 May 2023 14:42:48 +0000
Subject: [PATCH 613/689] test fix

---
 tests/queries/0_stateless/02751_protobuf_ipv6.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02751_protobuf_ipv6.sh b/tests/queries/0_stateless/02751_protobuf_ipv6.sh
index ecf565d9db4..f93963aa6c6 100755
--- a/tests/queries/0_stateless/02751_protobuf_ipv6.sh
+++ b/tests/queries/0_stateless/02751_protobuf_ipv6.sh
@@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 SCHEMADIR=$CURDIR/format_schemas
 
 
-echo 121a1000000000000000000000ffff01020304 | xxd -r -p | $CLICKHOUSE_LOCAL --input-format Protobuf --format_schema="$SCHEMADIR/02751_protobuf_ipv6:Message" --structure="ipv6_bytes IPv6" -q "select * from table"
+echo -ne '\x12\x1a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x01\x02\x03\x04' | $CLICKHOUSE_LOCAL --input-format Protobuf --format_schema="$SCHEMADIR/02751_protobuf_ipv6:Message" --structure="ipv6_bytes IPv6" -q "select * from table"
 
 $CLICKHOUSE_LOCAL -q "select '::ffff:1.2.3.4'::IPv6 as ipv6_bytes format Protobuf settings format_schema = '$SCHEMADIR/02751_protobuf_ipv6:Message'" | $CLICKHOUSE_LOCAL --input-format Protobuf --format_schema="$SCHEMADIR/02751_protobuf_ipv6:Message" --structure="ipv6_bytes IPv6"  -q "select * from table"
 

From 6b4dcbd3ed5f3f00322b86cf82780509f93ea038 Mon Sep 17 00:00:00 2001
From: Amos Bird <amosbird@gmail.com>
Date: Thu, 18 May 2023 23:23:39 +0800
Subject: [PATCH 614/689] Use PROJECT_*_DIR instead of CMAKE_*_DIR.

---
 CMakeLists.txt                                   |  8 ++++----
 cmake/add_check.cmake                            |  4 ++--
 cmake/git.cmake                                  | 10 +++++-----
 cmake/print_flags.cmake                          |  6 +++---
 cmake/sanitize.cmake                             |  6 +++---
 cmake/version.cmake                              |  2 +-
 contrib/avro-cmake/CMakeLists.txt                |  2 +-
 contrib/cassandra-cmake/CMakeLists.txt           |  2 +-
 contrib/cctz-cmake/CMakeLists.txt                |  2 +-
 contrib/libuv-cmake/CMakeLists.txt               |  4 ++--
 contrib/mariadb-connector-c-cmake/CMakeLists.txt |  2 +-
 contrib/snappy-cmake/CMakeLists.txt              |  2 +-
 contrib/zlib-ng-cmake/CMakeLists.txt             |  2 +-
 programs/self-extracting/CMakeLists.txt          |  6 +++---
 src/CMakeLists.txt                               |  2 +-
 15 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 26188cb7110..56bf3e1c3f8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -259,8 +259,8 @@ endif ()
 option (ENABLE_BUILD_PATH_MAPPING "Enable remapping of file source paths in debug info, predefined preprocessor macros, and __builtin_FILE(). It's used to generate reproducible builds. See https://reproducible-builds.org/docs/build-path" ${ENABLE_BUILD_PATH_MAPPING_DEFAULT})
 
 if (ENABLE_BUILD_PATH_MAPPING)
-    set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
-    set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
+    set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${PROJECT_SOURCE_DIR}=.")
+    set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${PROJECT_SOURCE_DIR}=.")
 endif ()
 
 option (ENABLE_BUILD_PROFILING "Enable profiling of build time" OFF)
@@ -557,7 +557,7 @@ if (NATIVE_BUILD_TARGETS
 )
     message (STATUS "Building native targets...")
 
-    set (NATIVE_BUILD_DIR "${CMAKE_BINARY_DIR}/native")
+    set (NATIVE_BUILD_DIR "${PROJECT_BINARY_DIR}/native")
 
     execute_process(
         COMMAND ${CMAKE_COMMAND} -E make_directory "${NATIVE_BUILD_DIR}"
@@ -571,7 +571,7 @@ if (NATIVE_BUILD_TARGETS
             # Avoid overriding .cargo/config.toml with native toolchain.
             "-DENABLE_RUST=OFF"
             "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=${ENABLE_CLICKHOUSE_SELF_EXTRACTING}"
-        ${CMAKE_SOURCE_DIR}
+        ${PROJECT_SOURCE_DIR}
         WORKING_DIRECTORY "${NATIVE_BUILD_DIR}"
         COMMAND_ECHO STDOUT)
 
diff --git a/cmake/add_check.cmake b/cmake/add_check.cmake
index c6abbcdb321..ba30ee8676f 100644
--- a/cmake/add_check.cmake
+++ b/cmake/add_check.cmake
@@ -5,11 +5,11 @@ if (NOT TARGET check)
     if (CMAKE_CONFIGURATION_TYPES)
         add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND}
             --force-new-ctest-process --output-on-failure --build-config "$<CONFIGURATION>"
-            WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+            WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
     else ()
         add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND}
             --force-new-ctest-process --output-on-failure
-            WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+            WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
     endif ()
 endif ()
 
diff --git a/cmake/git.cmake b/cmake/git.cmake
index 397ec3cd081..a4b3bd4bdab 100644
--- a/cmake/git.cmake
+++ b/cmake/git.cmake
@@ -5,14 +5,14 @@ if (Git_FOUND)
   # Commit hash + whether the building workspace was dirty or not
   execute_process(COMMAND
     "${GIT_EXECUTABLE}" rev-parse HEAD
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     OUTPUT_VARIABLE GIT_HASH
     ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
 
   # Branch name
   execute_process(COMMAND
     "${GIT_EXECUTABLE}" rev-parse --abbrev-ref HEAD
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     OUTPUT_VARIABLE GIT_BRANCH
     ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
 
@@ -20,14 +20,14 @@ if (Git_FOUND)
   SET(ENV{TZ} "UTC")
   execute_process(COMMAND
     "${GIT_EXECUTABLE}" log -1 --format=%ad --date=iso-local
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     OUTPUT_VARIABLE GIT_DATE
     ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
 
   # Subject of the commit
   execute_process(COMMAND
     "${GIT_EXECUTABLE}" log -1 --format=%s
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     OUTPUT_VARIABLE GIT_COMMIT_SUBJECT
     ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
 
@@ -35,7 +35,7 @@ if (Git_FOUND)
 
   execute_process(
     COMMAND ${GIT_EXECUTABLE} status
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_STRIP_TRAILING_WHITESPACE)
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_STRIP_TRAILING_WHITESPACE)
 else()
   message(STATUS "Git could not be found.")
 endif()
diff --git a/cmake/print_flags.cmake b/cmake/print_flags.cmake
index 66f2a8bfbc7..869764602d4 100644
--- a/cmake/print_flags.cmake
+++ b/cmake/print_flags.cmake
@@ -7,6 +7,6 @@ message (STATUS "compiler CXX = ${CMAKE_CXX_COMPILER} ${FULL_CXX_FLAGS}")
 message (STATUS "LINKER_FLAGS = ${FULL_EXE_LINKER_FLAGS}")
 
 # Reproducible builds
-string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_C_FLAGS_NORMALIZED "${FULL_C_FLAGS}")
-string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_CXX_FLAGS_NORMALIZED "${FULL_CXX_FLAGS}")
-string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_EXE_LINKER_FLAGS_NORMALIZED "${FULL_EXE_LINKER_FLAGS}")
+string (REPLACE "${PROJECT_SOURCE_DIR}" "." FULL_C_FLAGS_NORMALIZED "${FULL_C_FLAGS}")
+string (REPLACE "${PROJECT_SOURCE_DIR}" "." FULL_CXX_FLAGS_NORMALIZED "${FULL_CXX_FLAGS}")
+string (REPLACE "${PROJECT_SOURCE_DIR}" "." FULL_EXE_LINKER_FLAGS_NORMALIZED "${FULL_EXE_LINKER_FLAGS}")
diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake
index b2fbdb256fd..17ce8a7db29 100644
--- a/cmake/sanitize.cmake
+++ b/cmake/sanitize.cmake
@@ -29,14 +29,14 @@ if (SANITIZE)
 
         # Linking can fail due to relocation overflows (see #49145), caused by too big object files / libraries.
         # Work around this with position-independent builds (-fPIC and -fpie), this is slightly slower than non-PIC/PIE but that's okay.
-        set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-use-after-dtor -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fPIC -fpie -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/msan_suppressions.txt")
+        set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-use-after-dtor -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fPIC -fpie -fsanitize-blacklist=${PROJECT_SOURCE_DIR}/tests/msan_suppressions.txt")
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}")
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}")
 
     elseif (SANITIZE STREQUAL "thread")
         set (TSAN_FLAGS "-fsanitize=thread")
         if (COMPILER_CLANG)
-            set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/tsan_suppressions.txt")
+            set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-blacklist=${PROJECT_SOURCE_DIR}/tests/tsan_suppressions.txt")
         endif()
 
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}")
@@ -54,7 +54,7 @@ if (SANITIZE)
             set(UBSAN_FLAGS "${UBSAN_FLAGS} -fno-sanitize=unsigned-integer-overflow")
         endif()
         if (COMPILER_CLANG)
-            set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt")
+            set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-blacklist=${PROJECT_SOURCE_DIR}/tests/ubsan_suppressions.txt")
         endif()
 
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}")
diff --git a/cmake/version.cmake b/cmake/version.cmake
index acaa772ff2f..9ca21556f4d 100644
--- a/cmake/version.cmake
+++ b/cmake/version.cmake
@@ -1,4 +1,4 @@
-include(${CMAKE_SOURCE_DIR}/cmake/autogenerated_versions.txt)
+include(${PROJECT_SOURCE_DIR}/cmake/autogenerated_versions.txt)
 
 set(VERSION_EXTRA "" CACHE STRING "")
 set(VERSION_TWEAK "" CACHE STRING "")
diff --git a/contrib/avro-cmake/CMakeLists.txt b/contrib/avro-cmake/CMakeLists.txt
index 25474650d0e..63b3854eef9 100644
--- a/contrib/avro-cmake/CMakeLists.txt
+++ b/contrib/avro-cmake/CMakeLists.txt
@@ -6,7 +6,7 @@ if (NOT ENABLE_AVRO)
     return()
 endif()
 
-set(AVROCPP_ROOT_DIR "${CMAKE_SOURCE_DIR}/contrib/avro/lang/c++")
+set(AVROCPP_ROOT_DIR "${PROJECT_SOURCE_DIR}/contrib/avro/lang/c++")
 set(AVROCPP_INCLUDE_DIR "${AVROCPP_ROOT_DIR}/api")
 set(AVROCPP_SOURCE_DIR "${AVROCPP_ROOT_DIR}/impl")
 
diff --git a/contrib/cassandra-cmake/CMakeLists.txt b/contrib/cassandra-cmake/CMakeLists.txt
index 59ff908b63a..32611e0e151 100644
--- a/contrib/cassandra-cmake/CMakeLists.txt
+++ b/contrib/cassandra-cmake/CMakeLists.txt
@@ -18,7 +18,7 @@ endif()
 # Need to use C++17 since the compilation is not possible with C++20 currently.
 set (CMAKE_CXX_STANDARD 17)
 
-set(CASS_ROOT_DIR ${CMAKE_SOURCE_DIR}/contrib/cassandra)
+set(CASS_ROOT_DIR ${PROJECT_SOURCE_DIR}/contrib/cassandra)
 set(CASS_SRC_DIR "${CASS_ROOT_DIR}/src")
 set(CASS_INCLUDE_DIR "${CASS_ROOT_DIR}/include")
 
diff --git a/contrib/cctz-cmake/CMakeLists.txt b/contrib/cctz-cmake/CMakeLists.txt
index f1ef9b53f7d..10070fbd949 100644
--- a/contrib/cctz-cmake/CMakeLists.txt
+++ b/contrib/cctz-cmake/CMakeLists.txt
@@ -26,7 +26,7 @@ endif ()
 # StorageSystemTimeZones.generated.cpp is autogenerated each time during a build
 # data in this file will be used to populate the system.time_zones table, this is specific to OS_LINUX
 # as the library that's built using embedded tzdata is also specific to OS_LINUX
-set(SYSTEM_STORAGE_TZ_FILE "${CMAKE_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp")
+set(SYSTEM_STORAGE_TZ_FILE "${PROJECT_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp")
 # remove existing copies so that its generated fresh on each build.
 file(REMOVE ${SYSTEM_STORAGE_TZ_FILE})
 
diff --git a/contrib/libuv-cmake/CMakeLists.txt b/contrib/libuv-cmake/CMakeLists.txt
index fb88799ed38..928fdcdd7e6 100644
--- a/contrib/libuv-cmake/CMakeLists.txt
+++ b/contrib/libuv-cmake/CMakeLists.txt
@@ -1,7 +1,7 @@
 # This file is a modified version of contrib/libuv/CMakeLists.txt
 
-set (SOURCE_DIR "${CMAKE_SOURCE_DIR}/contrib/libuv")
-set (BINARY_DIR "${CMAKE_BINARY_DIR}/contrib/libuv")
+set (SOURCE_DIR "${PROJECT_SOURCE_DIR}/contrib/libuv")
+set (BINARY_DIR "${PROJECT_BINARY_DIR}/contrib/libuv")
 
 set(uv_sources
     src/fs-poll.c
diff --git a/contrib/mariadb-connector-c-cmake/CMakeLists.txt b/contrib/mariadb-connector-c-cmake/CMakeLists.txt
index 50287c54ac1..18d1510a57b 100644
--- a/contrib/mariadb-connector-c-cmake/CMakeLists.txt
+++ b/contrib/mariadb-connector-c-cmake/CMakeLists.txt
@@ -15,7 +15,7 @@ endif()
 
 # This is the LGPL libmariadb project.
 
-set(CC_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/mariadb-connector-c)
+set(CC_SOURCE_DIR ${PROJECT_SOURCE_DIR}/contrib/mariadb-connector-c)
 set(CC_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
 
 set(WITH_SSL ON)
diff --git a/contrib/snappy-cmake/CMakeLists.txt b/contrib/snappy-cmake/CMakeLists.txt
index 50cdc8732a1..f406de0e343 100644
--- a/contrib/snappy-cmake/CMakeLists.txt
+++ b/contrib/snappy-cmake/CMakeLists.txt
@@ -1,4 +1,4 @@
-set (SOURCE_DIR "${CMAKE_SOURCE_DIR}/contrib/snappy")
+set (SOURCE_DIR "${PROJECT_SOURCE_DIR}/contrib/snappy")
 
 if (ARCH_S390X)
     set (SNAPPY_IS_BIG_ENDIAN 1)
diff --git a/contrib/zlib-ng-cmake/CMakeLists.txt b/contrib/zlib-ng-cmake/CMakeLists.txt
index aa067ba37e0..79f343bfc75 100644
--- a/contrib/zlib-ng-cmake/CMakeLists.txt
+++ b/contrib/zlib-ng-cmake/CMakeLists.txt
@@ -1,4 +1,4 @@
-set (SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/zlib-ng)
+set (SOURCE_DIR ${PROJECT_SOURCE_DIR}/contrib/zlib-ng)
 
 add_definitions(-DZLIB_COMPAT)
 add_definitions(-DWITH_GZFILEOP)
diff --git a/programs/self-extracting/CMakeLists.txt b/programs/self-extracting/CMakeLists.txt
index 2cc26926b38..f3ff0bbcd78 100644
--- a/programs/self-extracting/CMakeLists.txt
+++ b/programs/self-extracting/CMakeLists.txt
@@ -4,10 +4,10 @@ if (NOT(
         AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR
     )
 )
-    set (COMPRESSOR "${CMAKE_BINARY_DIR}/native/utils/self-extracting-executable/pre_compressor")
-    set (DECOMPRESSOR "--decompressor=${CMAKE_BINARY_DIR}/utils/self-extracting-executable/decompressor")
+    set (COMPRESSOR "${PROJECT_BINARY_DIR}/native/utils/self-extracting-executable/pre_compressor")
+    set (DECOMPRESSOR "--decompressor=${PROJECT_BINARY_DIR}/utils/self-extracting-executable/decompressor")
 else ()
-    set (COMPRESSOR "${CMAKE_BINARY_DIR}/utils/self-extracting-executable/compressor")
+    set (COMPRESSOR "${PROJECT_BINARY_DIR}/utils/self-extracting-executable/compressor")
 endif ()
 
 add_custom_target (self-extracting ALL
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index b3f4fbb7420..87a2979ecd1 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -527,7 +527,7 @@ target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::fast_float)
 
 if (USE_ORC)
     dbms_target_link_libraries(PUBLIC ${ORC_LIBRARIES})
-    dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR} "${CMAKE_BINARY_DIR}/contrib/orc/c++/include")
+    dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR} "${PROJECT_BINARY_DIR}/contrib/orc/c++/include")
 endif ()
 
 if (TARGET ch_contrib::rocksdb)

From 73661c3a4635398de2d4783f0e2c17fe59265258 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 18 May 2023 18:18:30 +0200
Subject: [PATCH 615/689] Move tunnings for woboq codebrowser to cmake out from
 build.sh

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 CMakeLists.txt                   | 6 ++++++
 docker/test/codebrowser/build.sh | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 26188cb7110..54d1ae2bfcb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -102,6 +102,12 @@ if (ENABLE_FUZZING)
     set (ENABLE_PROTOBUF 1)
 endif()
 
+option (ENABLE_WOBOQ_CODEBROWSER "Build for woboq codebrowser" OFF)
+
+if (ENABLE_WOBOQ_CODEBROWSER)
+    set (ENABLE_EMBEDDED_COMPILER 0)
+endif()
+
 # Global libraries
 # See:
 # - default_libs.cmake
diff --git a/docker/test/codebrowser/build.sh b/docker/test/codebrowser/build.sh
index 5ab9de5a453..d76d0c3a039 100755
--- a/docker/test/codebrowser/build.sh
+++ b/docker/test/codebrowser/build.sh
@@ -15,7 +15,7 @@ nproc=$(($(nproc) + 2)) # increase parallelism
 read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
 
 mkdir -p "$BUILD_DIRECTORY" && cd "$BUILD_DIRECTORY"
-cmake "$SOURCE_DIRECTORY" -DCMAKE_CXX_COMPILER="/usr/bin/clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="/usr/bin/clang-${LLVM_VERSION}" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 "${CMAKE_FLAGS[@]}"
+cmake "$SOURCE_DIRECTORY" -DCMAKE_CXX_COMPILER="/usr/bin/clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="/usr/bin/clang-${LLVM_VERSION}" -DENABLE_WOBOQ_CODEBROWSER=ON "${CMAKE_FLAGS[@]}"
 mkdir -p "$HTML_RESULT_DIRECTORY"
 echo 'Filter out too noisy "Error: filename" lines and keep them in full codebrowser_generator.log'
 /woboq_codebrowser/generator/codebrowser_generator -b "$BUILD_DIRECTORY" -a \

From 0f7a310a6775728fd71c4784c5e7a8e776338ca5 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 18 May 2023 18:21:19 +0200
Subject: [PATCH 616/689] Fix woboq codebrowser build with
 -Wno-poison-system-directories

woboq codebrowser uses clang tooling, which adds clang system includes
(in Linux::AddClangSystemIncludeArgs()), because none of (-nostdinc,
-nobuiltininc) is set.

And later it will complain with -Wpoison-system-directories for added by
itself includes in InitHeaderSearch::AddUnmappedPath(), because they are
starts from one of the following:
- /usr/include
- /usr/local/include

The interesting thing here is that it got broken only after upgrading to
llvm 16 (in #49678), and the reason for this is that clang 15 build has
system includes that does not trigger the warning -
"/usr/lib/clang/15.0.7/include", while clang 16 has
"/usr/include/clang/16.0.4/include"

So let's simply disable this warning, but only for woboq.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 CMakeLists.txt | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 54d1ae2bfcb..ef3f34204b2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -106,6 +106,11 @@ option (ENABLE_WOBOQ_CODEBROWSER "Build for woboq codebrowser" OFF)
 
 if (ENABLE_WOBOQ_CODEBROWSER)
     set (ENABLE_EMBEDDED_COMPILER 0)
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-poison-system-directories")
+    # woboq codebrowser uses clang tooling, and they could add default system
+    # clang includes, and later clang will warn for those added by itself
+    # includes.
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-poison-system-directories")
 endif()
 
 # Global libraries

From e7b6056bbbab56b2fe8d0e9f6243bba355744f31 Mon Sep 17 00:00:00 2001
From: Denny Crane <denis.zhuravlov@gmail.com>
Date: Thu, 18 May 2023 15:18:55 -0300
Subject: [PATCH 617/689] test for #46128

---
 ...with_short_circuit_functins_mutations.reference |  3 +++
 ...ality_with_short_circuit_functins_mutations.sql | 14 ++++++++++++++
 2 files changed, 17 insertions(+)
 create mode 100644 tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins_mutations.reference
 create mode 100644 tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins_mutations.sql

diff --git a/tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins_mutations.reference b/tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins_mutations.reference
new file mode 100644
index 00000000000..9a6e97d4503
--- /dev/null
+++ b/tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins_mutations.reference
@@ -0,0 +1,3 @@
+0	xxxx	yyyy
+1	yyyy	yyyy
+2	xxxx	yyyy
diff --git a/tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins_mutations.sql b/tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins_mutations.sql
new file mode 100644
index 00000000000..9d183dde91d
--- /dev/null
+++ b/tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins_mutations.sql
@@ -0,0 +1,14 @@
+drop table if exists issue_46128;
+
+create table issue_46128 (
+	id Int64,
+	a LowCardinality(Nullable(String)),
+	b LowCardinality(Nullable(String))
+) Engine = MergeTree order by id
+as  select number%100, 'xxxx', 'yyyy' from numbers(10);
+
+ALTER TABLE issue_46128 UPDATE a = b WHERE id= 1 settings mutations_sync=2;
+
+select * from issue_46128 where id  <= 2 order by id;
+
+drop table issue_46128;

From 8dc59c1efea2dfaa56ec5a0728a362714e2ab206 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Thu, 18 May 2023 21:40:20 +0000
Subject: [PATCH 618/689] Fix test_insert_same_partition_and_merge failing if
 one Azure request attempt fails

---
 src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp             | 2 +-
 tests/integration/test_merge_tree_azure_blob_storage/test.py | 4 ++--
 tests/integration/test_merge_tree_s3/test.py                 | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp
index f8ca6b9ab07..44185f74f60 100644
--- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp
+++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp
@@ -51,7 +51,7 @@ void WriteBufferFromAzureBlobStorage::execWithRetry(std::function<void()> func,
         if (i == num_tries - 1)
             throw;
 
-        LOG_DEBUG(log, "Write at attempt {} for blob `{}` failed: {}", i + 1, blob_path, e.Message);
+        LOG_DEBUG(log, "Write at attempt {} for blob `{}` failed: {} {}", i + 1, blob_path, e.what(), e.Message);
     };
 
     for (size_t i = 0; i < num_tries; ++i)
diff --git a/tests/integration/test_merge_tree_azure_blob_storage/test.py b/tests/integration/test_merge_tree_azure_blob_storage/test.py
index bcb62c3181d..8bf4df17c39 100644
--- a/tests/integration/test_merge_tree_azure_blob_storage/test.py
+++ b/tests/integration/test_merge_tree_azure_blob_storage/test.py
@@ -203,7 +203,7 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical):
     node.query(f"SYSTEM START MERGES {TABLE_NAME}")
 
     # Wait for merges and old parts deletion
-    for attempt in range(0, 10):
+    for attempt in range(0, 60):
         parts_count = azure_query(
             node,
             f"SELECT COUNT(*) FROM system.parts WHERE table = '{TABLE_NAME}' FORMAT Values",
@@ -211,7 +211,7 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical):
         if parts_count == "(1)":
             break
 
-        if attempt == 9:
+        if attempt == 59:
             assert parts_count == "(1)"
 
         time.sleep(1)
diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py
index 7fbe8c8e99b..ee774f6632b 100644
--- a/tests/integration/test_merge_tree_s3/test.py
+++ b/tests/integration/test_merge_tree_s3/test.py
@@ -232,7 +232,7 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical, node_name):
     node.query("SYSTEM START MERGES s3_test")
 
     # Wait for merges and old parts deletion
-    for attempt in range(0, 10):
+    for attempt in range(0, 60):
         parts_count = node.query(
             "SELECT COUNT(*) FROM system.parts WHERE table = 's3_test' and active = 1 FORMAT Values"
         )
@@ -240,7 +240,7 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical, node_name):
         if parts_count == "(1)":
             break
 
-        if attempt == 9:
+        if attempt == 59:
             assert parts_count == "(1)"
 
         time.sleep(1)

From e84f0895e72f25c7200993a1fd8ac8e11bd7f9b9 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Tue, 16 May 2023 19:27:30 +0000
Subject: [PATCH 619/689] Support hardlinking parts transactionally

---
 .../MergeTree/DataPartStorageOnDiskBase.cpp   | 27 +++++++---
 .../MergeTree/DataPartStorageOnDiskBase.h     |  3 +-
 src/Storages/MergeTree/IDataPartStorage.h     |  8 ++-
 src/Storages/MergeTree/IMergeTreeDataPart.h   |  2 +-
 src/Storages/MergeTree/localBackup.cpp        | 50 ++++++++++++++-----
 src/Storages/MergeTree/localBackup.h          |  8 +--
 tests/integration/test_grpc_protocol/test.py  |  2 +-
 .../test_grpc_protocol_ssl/test.py            |  2 +-
 tests/integration/test_server_reload/test.py  |  2 +-
 9 files changed, 76 insertions(+), 28 deletions(-)

diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
index ebe55ea7dc7..cfc3ff58f81 100644
--- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
+++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
@@ -411,25 +411,38 @@ MutableDataPartStoragePtr DataPartStorageOnDiskBase::freeze(
     bool make_source_readonly,
     std::function<void(const DiskPtr &)> save_metadata_callback,
     bool copy_instead_of_hardlink,
-    const NameSet & files_to_copy_instead_of_hardlinks) const
+    const NameSet & files_to_copy_instead_of_hardlinks,
+    DiskTransactionPtr external_transaction) const
 {
     auto disk = volume->getDisk();
-    disk->createDirectories(to);
+    if (external_transaction)
+        external_transaction->createDirectories(to);
+    else
+        disk->createDirectories(to);
 
-    localBackup(disk, getRelativePath(), fs::path(to) / dir_path, make_source_readonly, {}, copy_instead_of_hardlink, files_to_copy_instead_of_hardlinks);
+    localBackup(disk, getRelativePath(), fs::path(to) / dir_path, make_source_readonly, {}, copy_instead_of_hardlink, files_to_copy_instead_of_hardlinks, external_transaction);
 
     if (save_metadata_callback)
         save_metadata_callback(disk);
 
-    disk->removeFileIfExists(fs::path(to) / dir_path / "delete-on-destroy.txt");
-    disk->removeFileIfExists(fs::path(to) / dir_path / "txn_version.txt");
-    disk->removeFileIfExists(fs::path(to) / dir_path / IMergeTreeDataPart::METADATA_VERSION_FILE_NAME);
+    if (external_transaction)
+    {
+        external_transaction->removeFileIfExists(fs::path(to) / dir_path / "delete-on-destroy.txt");
+        external_transaction->removeFileIfExists(fs::path(to) / dir_path / "txn_version.txt");
+        external_transaction->removeFileIfExists(fs::path(to) / dir_path / IMergeTreeDataPart::METADATA_VERSION_FILE_NAME);
+    }
+    else
+    {
+        disk->removeFileIfExists(fs::path(to) / dir_path / "delete-on-destroy.txt");
+        disk->removeFileIfExists(fs::path(to) / dir_path / "txn_version.txt");
+        disk->removeFileIfExists(fs::path(to) / dir_path / IMergeTreeDataPart::METADATA_VERSION_FILE_NAME);
+    }
 
     auto single_disk_volume = std::make_shared<SingleDiskVolume>(disk->getName(), disk, 0);
 
     /// Do not initialize storage in case of DETACH because part may be broken.
     bool to_detached = dir_path.starts_with("detached/");
-    return create(single_disk_volume, to, dir_path, /*initialize=*/ !to_detached);
+    return create(single_disk_volume, to, dir_path, /*initialize=*/ !to_detached && !external_transaction);
 }
 
 MutableDataPartStoragePtr DataPartStorageOnDiskBase::clonePart(
diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h
index 09eb7f008bc..6b27b7296fc 100644
--- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h
+++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h
@@ -64,7 +64,8 @@ public:
         bool make_source_readonly,
         std::function<void(const DiskPtr &)> save_metadata_callback,
         bool copy_instead_of_hardlink,
-        const NameSet & files_to_copy_instead_of_hardlinks) const override;
+        const NameSet & files_to_copy_instead_of_hardlinks,
+        DiskTransactionPtr external_transaction) const override;
 
     MutableDataPartStoragePtr clonePart(
         const std::string & to,
diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h
index 7c85469d890..f160254350d 100644
--- a/src/Storages/MergeTree/IDataPartStorage.h
+++ b/src/Storages/MergeTree/IDataPartStorage.h
@@ -11,6 +11,7 @@
 #include <memory>
 #include <optional>
 #include <Common/ZooKeeper/ZooKeeper.h>
+#include <Disks/IDiskTransaction.h>
 
 namespace DB
 {
@@ -212,13 +213,18 @@ public:
     /// implementation which relies on paths of some blobs in S3. For example if we want to hardlink
     /// the whole part during mutation we shouldn't hardlink checksums.txt, because otherwise
     /// zero-copy locks for different parts will be on the same path in zookeeper.
+    ///
+    /// If `external_transaction` is provided, the disk operations (creating directories, hardlinking,
+    /// etc) won't be applied immediately; instead, they'll be added to external_transaction, which the
+    /// caller then needs to commit.
     virtual std::shared_ptr<IDataPartStorage> freeze(
         const std::string & to,
         const std::string & dir_path,
         bool make_source_readonly,
         std::function<void(const DiskPtr &)> save_metadata_callback,
         bool copy_instead_of_hardlink,
-        const NameSet & files_to_copy_instead_of_hardlinks) const = 0;
+        const NameSet & files_to_copy_instead_of_hardlinks,
+        DiskTransactionPtr external_transaction = nullptr) const = 0;
 
     /// Make a full copy of a data part into 'to/dir_path' (possibly to a different disk).
     virtual std::shared_ptr<IDataPartStorage> clonePart(
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h
index ff40c1da8d1..388d96314c0 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.h
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.h
@@ -161,7 +161,7 @@ public:
     void remove();
 
     /// Initialize columns (from columns.txt if exists, or create from column files if not).
-    /// Load checksums from checksums.txt if exists. Load index if required.
+    /// Load various metadata into memory: checksums from checksums.txt, index if required, etc.
     void loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency);
     void appendFilesOfColumnsChecksumsIndexes(Strings & files, bool include_projection = false) const;
 
diff --git a/src/Storages/MergeTree/localBackup.cpp b/src/Storages/MergeTree/localBackup.cpp
index 3b05e3df8d3..6faacf3c066 100644
--- a/src/Storages/MergeTree/localBackup.cpp
+++ b/src/Storages/MergeTree/localBackup.cpp
@@ -17,9 +17,10 @@ namespace
 {
 
 void localBackupImpl(
-    const DiskPtr & disk, const String & source_path,
+    const DiskPtr & disk, IDiskTransaction * transaction, const String & source_path,
     const String & destination_path, bool make_source_readonly, size_t level,
-    std::optional<size_t> max_level, const NameSet & files_to_copy_instead_of_hardlinks)
+    std::optional<size_t> max_level, bool copy_instead_of_hardlinks,
+    const NameSet & files_to_copy_instead_of_hardlinks)
 {
     if (max_level && level > *max_level)
         return;
@@ -27,7 +28,10 @@ void localBackupImpl(
     if (level >= 1000)
         throw DB::Exception(DB::ErrorCodes::TOO_DEEP_RECURSION, "Too deep recursion");
 
-    disk->createDirectories(destination_path);
+    if (transaction)
+        transaction->createDirectories(destination_path);
+    else
+        disk->createDirectories(destination_path);
 
     for (auto it = disk->iterateDirectory(source_path); it->isValid(); it->next())
     {
@@ -37,15 +41,36 @@ void localBackupImpl(
         if (!disk->isDirectory(source))
         {
             if (make_source_readonly)
-                disk->setReadOnly(source);
-            if (files_to_copy_instead_of_hardlinks.contains(it->name()))
-                disk->copyFile(source, *disk, destination);
+            {
+                if (transaction)
+                    transaction->setReadOnly(source);
+                else
+                    disk->setReadOnly(source);
+            }
+            if (copy_instead_of_hardlinks || files_to_copy_instead_of_hardlinks.contains(it->name()))
+            {
+                if (transaction)
+                {
+                    transaction->copyFile(source, destination);
+                }
+                else
+                {
+                    disk->copyFile(source, *disk, destination);
+                }
+            }
             else
-                disk->createHardLink(source, destination);
+            {
+                if (transaction)
+                    transaction->createHardLink(source, destination);
+                else
+                    disk->createHardLink(source, destination);
+            }
         }
         else
         {
-            localBackupImpl(disk, source, destination, make_source_readonly, level + 1, max_level, files_to_copy_instead_of_hardlinks);
+            localBackupImpl(
+                disk, transaction, source, destination, make_source_readonly, level + 1, max_level,
+                copy_instead_of_hardlinks, files_to_copy_instead_of_hardlinks);
         }
     }
 }
@@ -89,7 +114,7 @@ private:
 void localBackup(
     const DiskPtr & disk, const String & source_path,
     const String & destination_path, bool make_source_readonly,
-    std::optional<size_t> max_level, bool copy_instead_of_hardlinks, const NameSet & files_to_copy_intead_of_hardlinks)
+    std::optional<size_t> max_level, bool copy_instead_of_hardlinks, const NameSet & files_to_copy_intead_of_hardlinks, DiskTransactionPtr disk_transaction)
 {
     if (disk->exists(destination_path) && !disk->isDirectoryEmpty(destination_path))
     {
@@ -100,7 +125,8 @@ void localBackup(
     size_t try_no = 0;
     const size_t max_tries = 10;
 
-    CleanupOnFail cleanup([disk, destination_path]() { disk->removeRecursive(destination_path); });
+    CleanupOnFail cleanup(disk_transaction ? std::function<void()>([]{}) :
+                          [disk, destination_path]() { disk->removeRecursive(destination_path); });
 
     /** Files in the directory can be permanently added and deleted.
       * If some file is deleted during an attempt to make a backup, then try again,
@@ -110,10 +136,10 @@ void localBackup(
     {
         try
         {
-            if (copy_instead_of_hardlinks)
+            if (copy_instead_of_hardlinks && !disk_transaction)
                 disk->copyDirectoryContent(source_path, disk, destination_path);
             else
-                localBackupImpl(disk, source_path, destination_path, make_source_readonly, 0, max_level, files_to_copy_intead_of_hardlinks);
+                localBackupImpl(disk, disk_transaction.get(), source_path, destination_path, make_source_readonly, 0, max_level, copy_instead_of_hardlinks, files_to_copy_intead_of_hardlinks);
         }
         catch (const DB::ErrnoException & e)
         {
diff --git a/src/Storages/MergeTree/localBackup.h b/src/Storages/MergeTree/localBackup.h
index 74b188daff6..89906bf1d75 100644
--- a/src/Storages/MergeTree/localBackup.h
+++ b/src/Storages/MergeTree/localBackup.h
@@ -9,7 +9,7 @@ namespace DB
 
 /** Creates a local (at the same mount point) backup (snapshot) directory.
   *
-  * In the specified destination directory, it creates a hard links on all source-directory files
+  * In the specified destination directory, it creates hard links on all source-directory files
   *  and in all nested directories, with saving (creating) all relative paths;
   *  and also `chown`, removing the write permission.
   *
@@ -17,9 +17,11 @@ namespace DB
   *  and is intended to be used as a simple means of protection against a human or program error,
   *  but not from a hardware failure.
   *
-  *  If max_level is specified, than only files which depth relative source_path less or equal max_level will be copied.
+  * If max_level is specified, than only files with depth relative source_path less or equal max_level will be copied.
   *  So, if max_level=0 than only direct file child are copied.
+  *
+  * If `transaction` is provided, the changes will be added to it instead of performend on disk.
   */
-    void localBackup(const DiskPtr & disk, const String & source_path, const String & destination_path, bool make_source_readonly = true, std::optional<size_t> max_level = {}, bool copy_instead_of_hardlinks = false, const NameSet & files_to_copy_intead_of_hardlinks = {});
+    void localBackup(const DiskPtr & disk, const String & source_path, const String & destination_path, bool make_source_readonly = true, std::optional<size_t> max_level = {}, bool copy_instead_of_hardlinks = false, const NameSet & files_to_copy_intead_of_hardlinks = {}, DiskTransactionPtr disk_transaction = nullptr);
 
 }
diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py
index 254b78667d5..dead4d447ec 100644
--- a/tests/integration/test_grpc_protocol/test.py
+++ b/tests/integration/test_grpc_protocol/test.py
@@ -42,7 +42,7 @@ node = cluster.add_instance(
     main_configs=["configs/grpc_config.xml"],
     # Bug in TSAN reproduces in this test https://github.com/grpc/grpc/issues/29550#issuecomment-1188085387
     env_variables={
-        "TSAN_OPTIONS": "report_atomic_races=0 " + os.getenv("TSAN_OPTIONS")
+        "TSAN_OPTIONS": "report_atomic_races=0 " + os.getenv("TSAN_OPTIONS", default="")
     },
 )
 main_channel = None
diff --git a/tests/integration/test_grpc_protocol_ssl/test.py b/tests/integration/test_grpc_protocol_ssl/test.py
index f1a4475c1a5..4c7fe99f49d 100644
--- a/tests/integration/test_grpc_protocol_ssl/test.py
+++ b/tests/integration/test_grpc_protocol_ssl/test.py
@@ -44,7 +44,7 @@ node = cluster.add_instance(
     ],
     # Bug in TSAN reproduces in this test https://github.com/grpc/grpc/issues/29550#issuecomment-1188085387
     env_variables={
-        "TSAN_OPTIONS": "report_atomic_races=0 " + os.getenv("TSAN_OPTIONS")
+        "TSAN_OPTIONS": "report_atomic_races=0 " + os.getenv("TSAN_OPTIONS", default="")
     },
 )
 
diff --git a/tests/integration/test_server_reload/test.py b/tests/integration/test_server_reload/test.py
index 1429713cb84..da5208799ac 100644
--- a/tests/integration/test_server_reload/test.py
+++ b/tests/integration/test_server_reload/test.py
@@ -36,7 +36,7 @@ instance = cluster.add_instance(
     with_zookeeper=True,
     # Bug in TSAN reproduces in this test https://github.com/grpc/grpc/issues/29550#issuecomment-1188085387
     env_variables={
-        "TSAN_OPTIONS": "report_atomic_races=0 " + os.getenv("TSAN_OPTIONS")
+        "TSAN_OPTIONS": "report_atomic_races=0 " + os.getenv("TSAN_OPTIONS", default="")
     },
 )
 

From c4f23e87f10be8f48c99be671a272347715676f9 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 18:56:56 +0200
Subject: [PATCH 620/689] Export grower_type in HashTable

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Common/HashTable/HashTable.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h
index 8542ddae144..439b96d6976 100644
--- a/src/Common/HashTable/HashTable.h
+++ b/src/Common/HashTable/HashTable.h
@@ -753,6 +753,7 @@ protected:
 
 public:
     using key_type = Key;
+    using grower_type = Grower;
     using mapped_type = typename Cell::mapped_type;
     using value_type = typename Cell::value_type;
     using cell_type = Cell;

From b44497fd4c1908b656bdd22538595084be63c9fe Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 1 May 2023 16:33:46 +0200
Subject: [PATCH 621/689] Introduce PackedHashMap (HashMap with structure
 without padding)

In case of you have HashMap with <UInt64, UInt16> as <Key, Value> the
overhead of 38% can be crutial, especially if you have tons of keys.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Common/HashTable/HashMap.h         | 18 +++++-----
 src/Common/HashTable/PackedHashMap.h   | 46 ++++++++++++++++++++++++++
 src/Interpreters/examples/hash_map.cpp | 31 +++++++++++++++++
 3 files changed, 87 insertions(+), 8 deletions(-)
 create mode 100644 src/Common/HashTable/PackedHashMap.h

diff --git a/src/Common/HashTable/HashMap.h b/src/Common/HashTable/HashMap.h
index f0045306fe8..5f4cb396822 100644
--- a/src/Common/HashTable/HashMap.h
+++ b/src/Common/HashTable/HashMap.h
@@ -9,6 +9,8 @@
 /** NOTE HashMap could only be used for memmoveable (position independent) types.
   * Example: std::string is not position independent in libstdc++ with C++11 ABI or in libc++.
   * Also, key in hash table must be of type, that zero bytes is compared equals to zero key.
+  *
+  * Please keep in sync with PackedHashMap.h
   */
 
 namespace DB
@@ -53,13 +55,13 @@ PairNoInit<std::decay_t<First>, std::decay_t<Second>> makePairNoInit(First && fi
 }
 
 
-template <typename Key, typename TMapped, typename Hash, typename TState = HashTableNoState>
+template <typename Key, typename TMapped, typename Hash, typename TState = HashTableNoState, typename Pair = PairNoInit<Key, TMapped>>
 struct HashMapCell
 {
     using Mapped = TMapped;
     using State = TState;
 
-    using value_type = PairNoInit<Key, Mapped>;
+    using value_type = Pair;
     using mapped_type = Mapped;
     using key_type = Key;
 
@@ -151,14 +153,14 @@ struct HashMapCell
 namespace std
 {
 
-    template <typename Key, typename TMapped, typename Hash, typename TState>
-    struct tuple_size<HashMapCell<Key, TMapped, Hash, TState>> : std::integral_constant<size_t, 2> { };
+    template <typename Key, typename TMapped, typename Hash, typename TState, typename Pair>
+    struct tuple_size<HashMapCell<Key, TMapped, Hash, TState, Pair>> : std::integral_constant<size_t, 2> { };
 
-    template <typename Key, typename TMapped, typename Hash, typename TState>
-    struct tuple_element<0, HashMapCell<Key, TMapped, Hash, TState>> { using type = Key; };
+    template <typename Key, typename TMapped, typename Hash, typename TState, typename Pair>
+    struct tuple_element<0, HashMapCell<Key, TMapped, Hash, TState, Pair>> { using type = Key; };
 
-    template <typename Key, typename TMapped, typename Hash, typename TState>
-    struct tuple_element<1, HashMapCell<Key, TMapped, Hash, TState>> { using type = TMapped; };
+    template <typename Key, typename TMapped, typename Hash, typename TState, typename Pair>
+    struct tuple_element<1, HashMapCell<Key, TMapped, Hash, TState, Pair>> { using type = TMapped; };
 }
 
 template <typename Key, typename TMapped, typename Hash, typename TState = HashTableNoState>
diff --git a/src/Common/HashTable/PackedHashMap.h b/src/Common/HashTable/PackedHashMap.h
new file mode 100644
index 00000000000..6b52a04aa49
--- /dev/null
+++ b/src/Common/HashTable/PackedHashMap.h
@@ -0,0 +1,46 @@
+#pragma once
+
+/// Packed versions HashMap, please keep in sync with HashMap.h
+
+#include <Common/HashTable/HashMap.h>
+
+/// A pair that does not initialize the elements, if not needed.
+///
+/// NOTE: makePairNoInit() is omitted for PackedPairNoInit since it is not
+/// required for PackedHashMap (see mergeBlockWithPipe() for details)
+template <typename First, typename Second>
+struct PackedPairNoInit
+{
+    First first;
+    Second second;
+
+    PackedPairNoInit() {} /// NOLINT
+
+    template <typename FirstValue>
+    PackedPairNoInit(FirstValue && first_, NoInitTag)
+        : first(std::forward<FirstValue>(first_))
+    {
+    }
+
+    template <typename FirstValue, typename SecondValue>
+    PackedPairNoInit(FirstValue && first_, SecondValue && second_)
+        : first(std::forward<FirstValue>(first_))
+        , second(std::forward<SecondValue>(second_))
+    {
+    }
+} __attribute__((packed));
+
+/// Packed HashMap - HashMap with structure without padding
+///
+/// Sometimes padding in structure can be crucial, consider the following
+/// example <UInt64, UInt16> as <Key, Value> in this case the padding overhead
+/// is 0.375, and this can be major in case of lots of keys.
+///
+/// Note, there is no need to provide PackedHashSet, since it cannot have padding.
+template <
+    typename Key,
+    typename Mapped,
+    typename Hash = DefaultHash<Key>,
+    typename Grower = HashTableGrower<>,
+    typename Allocator = HashTableAllocator>
+using PackedHashMap = HashMapTable<Key, HashMapCell<Key, Mapped, Hash, HashTableNoState, PackedPairNoInit<Key, Mapped>>, Hash, Grower, Allocator>;
diff --git a/src/Interpreters/examples/hash_map.cpp b/src/Interpreters/examples/hash_map.cpp
index fe2cbda019d..c0016bf8cd8 100644
--- a/src/Interpreters/examples/hash_map.cpp
+++ b/src/Interpreters/examples/hash_map.cpp
@@ -15,6 +15,7 @@
 #include <IO/ReadBufferFromFile.h>
 #include <Compression/CompressedReadBuffer.h>
 #include <Common/HashTable/HashMap.h>
+#include <Common/HashTable/PackedHashMap.h>
 #include <AggregateFunctions/IAggregateFunction.h>
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 #include <DataTypes/DataTypesNumber.h>
@@ -278,5 +279,35 @@ int main(int argc, char ** argv)
             << std::endl;
     }
 
+    if (argc < 3 || std::stol(argv[2]) == 7)
+    {
+        Stopwatch watch;
+
+        PackedHashMap<Key, Value> map;
+        PackedHashMap<Key, Value>::LookupResult it;
+        bool inserted;
+
+        for (size_t i = 0; i < n; ++i)
+        {
+            map.emplace(data[i], it, inserted);
+            if (inserted)
+            {
+                new (&it->getMapped()) Value;
+                std::swap(it->getMapped(), value);
+                INIT
+            }
+        }
+
+        watch.stop();
+        std::cerr << std::fixed << std::setprecision(2)
+            << "PackedHashMap. Size: " << map.size()
+            << ", elapsed: " << watch.elapsedSeconds()
+            << " (" << n / watch.elapsedSeconds() << " elem/sec.)"
+#ifdef DBMS_HASH_MAP_COUNT_COLLISIONS
+            << ", collisions: " << map.getCollisions()
+#endif
+            << std::endl;
+    }
+
     return 0;
 }

From 7b5d156cc5e4d53091a1d2930fe6def0fafe8b5e Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 1 May 2023 18:01:01 +0200
Subject: [PATCH 622/689] Optimize SPARSE_HASHED layout (by using
 PackedHashMap)

In case you want dictionary optimized for memory, SPARSE_HASHED is not
always gives you what you need.

Consider the following example <UInt64, UInt16> as <Key, Value>, but
this pair will also have a 6 byte padding (on amd64), so this is almost
40% of space wastage.

And because of this padding, even google::sparse_hash_map, does not make
picture better, in fact, sparse_hash_map is not very friendly to memory
allocators (especially jemalloc).

Here are some numbers for dictionary with 1e9 elements and UInt64 as
key, and UInt16 as value:

settings                         | load (sec) | read (sec) | read (million rows/s) | bytes_allocated | RSS
HASHED upstream                  | -          | -          | -                     | -               | 35GiB
SPARSE_HASHED upstream           | -          | -          | -                     | -               | 26GiB
-                                | -          | -          | -                     | -               | -
sparse_hash_map glibc hashbench  | -          | -          | -                     | -               | 17.5GiB
sparse_hash_map packed allocator | 101.878    | 231.48     | 4.32                  | -               | 17.7GiB
PackedHashMap                    | 15.514     | 42.35      | 23.61                 | 20GiB           | 22GiB

As you can see PackedHashMap looks way more better then HASHED, and
even better then SPARSE_HASHED, but slightly worse then sparse_hash_map
with packed allocator (it is done with a custom patch to google
sparse_hash_map).

v2: rebase on top of bucket_count fix
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Dictionaries/HashedDictionary.cpp         |  74 ++--------
 src/Dictionaries/HashedDictionary.h           |  44 +-----
 .../HashedDictionaryCollectionTraits.h        | 102 +++++++++++++
 .../HashedDictionaryCollectionType.h          | 134 ++++++++++++++++++
 .../performance/hashed_dictionary_sharded.xml |   2 +-
 5 files changed, 249 insertions(+), 107 deletions(-)
 create mode 100644 src/Dictionaries/HashedDictionaryCollectionTraits.h
 create mode 100644 src/Dictionaries/HashedDictionaryCollectionType.h

diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp
index 191adab8983..048c378b917 100644
--- a/src/Dictionaries/HashedDictionary.cpp
+++ b/src/Dictionaries/HashedDictionary.cpp
@@ -21,6 +21,7 @@
 #include <Dictionaries/DictionarySource.h>
 #include <Dictionaries/DictionaryFactory.h>
 #include <Dictionaries/HierarchyDictionariesUtils.h>
+#include <Dictionaries/HashedDictionaryCollectionTraits.h>
 
 namespace CurrentMetrics
 {
@@ -28,21 +29,6 @@ namespace CurrentMetrics
     extern const Metric HashedDictionaryThreadsActive;
 }
 
-namespace
-{
-
-/// NOTE: Trailing return type is explicitly specified for SFINAE.
-
-/// google::sparse_hash_map
-template <typename T> auto getKeyFromCell(const T & value) -> decltype(value->first) { return value->first; } // NOLINT
-template <typename T> auto getValueFromCell(const T & value) -> decltype(value->second) { return value->second; } // NOLINT
-
-/// HashMap
-template <typename T> auto getKeyFromCell(const T & value) -> decltype(value->getKey()) { return value->getKey(); } // NOLINT
-template <typename T> auto getValueFromCell(const T & value) -> decltype(value->getMapped()) { return value->getMapped(); } // NOLINT
-
-}
-
 namespace DB
 {
 
@@ -242,10 +228,7 @@ HashedDictionary<dictionary_key_type, sparse, sharded>::~HashedDictionary()
                 CurrentThread::attachToGroupIfDetached(thread_group);
             setThreadName("HashedDictDtor");
 
-            if constexpr (sparse)
-                container.clear();
-            else
-                container.clearAndShrink();
+            clearContainer(container);
         });
 
         ++hash_tables_count;
@@ -834,12 +817,7 @@ void HashedDictionary<dictionary_key_type, sparse, sharded>::resize(size_t added
     if (unlikely(attributes_size == 0))
     {
         size_t reserve_size = added_rows + no_attributes_containers.front().size();
-
-        if constexpr (sparse)
-            no_attributes_containers.front().resize(reserve_size);
-        else
-            no_attributes_containers.front().reserve(reserve_size);
-
+        resizeContainer(no_attributes_containers.front(), reserve_size);
         return;
     }
 
@@ -849,11 +827,7 @@ void HashedDictionary<dictionary_key_type, sparse, sharded>::resize(size_t added
         {
             auto & container = containers.front();
             size_t reserve_size = added_rows + container.size();
-
-            if constexpr (sparse)
-                container.resize(reserve_size);
-            else
-                container.reserve(reserve_size);
+            resizeContainer(container, reserve_size);
         });
     }
 }
@@ -973,25 +947,9 @@ void HashedDictionary<dictionary_key_type, sparse, sharded>::calculateBytesAlloc
         {
             for (const auto & container : containers)
             {
-                using ContainerType = std::decay_t<decltype(container)>;
-                using AttributeValueType = typename ContainerType::mapped_type;
-
                 bytes_allocated += sizeof(container);
-
-                if constexpr (sparse || std::is_same_v<AttributeValueType, Field>)
-                {
-                    /// bucket_count() - Returns table size, that includes empty and deleted
-                    /// size()         - Returns table size, without empty and deleted
-                    /// and since this is sparsehash, empty cells should not be significant,
-                    /// and since items cannot be removed from the dictionary, deleted is also not important.
-                    bytes_allocated += container.size() * (sizeof(KeyType) + sizeof(AttributeValueType));
-                    bucket_count += container.bucket_count();
-                }
-                else
-                {
-                    bytes_allocated += container.getBufferSizeInBytes();
-                    bucket_count += container.getBufferSizeInCells();
-                }
+                bytes_allocated += getBufferSizeInBytes(container);
+                bucket_count += getBufferSizeInCells(container);
             }
         });
 
@@ -1010,17 +968,8 @@ void HashedDictionary<dictionary_key_type, sparse, sharded>::calculateBytesAlloc
         for (const auto & container : no_attributes_containers)
         {
             bytes_allocated += sizeof(container);
-
-            if constexpr (sparse)
-            {
-                bytes_allocated += container.size() * (sizeof(KeyType));
-                bucket_count += container.bucket_count();
-            }
-            else
-            {
-                bytes_allocated += container.getBufferSizeInBytes();
-                bucket_count += container.getBufferSizeInCells();
-            }
+            bytes_allocated += getBufferSizeInBytes(container);
+            bucket_count += getBufferSizeInCells(container);
         }
     }
 
@@ -1078,12 +1027,7 @@ Pipe HashedDictionary<dictionary_key_type, sparse, sharded>::read(const Names &
             keys.reserve(keys.size() + container.size());
 
             for (const auto & key : container)
-            {
-                if constexpr (sparse)
-                    keys.emplace_back(key);
-                else
-                    keys.emplace_back(key.getKey());
-            }
+                keys.emplace_back(getSetKeyFromCell(key));
         }
     }
 
diff --git a/src/Dictionaries/HashedDictionary.h b/src/Dictionaries/HashedDictionary.h
index 67683679647..08f5dfaa40d 100644
--- a/src/Dictionaries/HashedDictionary.h
+++ b/src/Dictionaries/HashedDictionary.h
@@ -4,17 +4,14 @@
 #include <memory>
 #include <variant>
 #include <optional>
-#include <sparsehash/sparse_hash_map>
-#include <sparsehash/sparse_hash_set>
 
-#include <Common/HashTable/HashMap.h>
-#include <Common/HashTable/HashSet.h>
 #include <Core/Block.h>
 
 #include <Dictionaries/DictionaryStructure.h>
 #include <Dictionaries/IDictionary.h>
 #include <Dictionaries/IDictionarySource.h>
 #include <Dictionaries/DictionaryHelpers.h>
+#include <Dictionaries/HashedDictionaryCollectionType.h>
 
 /** This dictionary stores all content in a hash table in memory
   * (a separate Key -> Value map for each attribute)
@@ -136,42 +133,7 @@ public:
 
 private:
     template <typename Value>
-    using CollectionTypeNonSparse = std::conditional_t<
-        dictionary_key_type == DictionaryKeyType::Simple,
-        HashMap<UInt64, Value, DefaultHash<UInt64>>,
-        HashMapWithSavedHash<StringRef, Value, DefaultHash<StringRef>>>;
-
-    using NoAttributesCollectionTypeNonSparse = std::conditional_t<
-        dictionary_key_type == DictionaryKeyType::Simple,
-        HashSet<UInt64, DefaultHash<UInt64>>,
-        HashSetWithSavedHash<StringRef, DefaultHash<StringRef>>>;
-
-    /// Here we use sparse_hash_map with DefaultHash<> for the following reasons:
-    ///
-    /// - DefaultHash<> is used for HashMap
-    /// - DefaultHash<> (from HashTable/Hash.h> works better then std::hash<>
-    ///   in case of sequential set of keys, but with random access to this set, i.e.
-    ///
-    ///       SELECT number FROM numbers(3000000) ORDER BY rand()
-    ///
-    ///   And even though std::hash<> works better in some other cases,
-    ///   DefaultHash<> is preferred since the difference for this particular
-    ///   case is significant, i.e. it can be 10x+.
-    template <typename Value>
-    using CollectionTypeSparse = std::conditional_t<
-        dictionary_key_type == DictionaryKeyType::Simple,
-        google::sparse_hash_map<UInt64, Value, DefaultHash<KeyType>>,
-        google::sparse_hash_map<StringRef, Value, DefaultHash<KeyType>>>;
-
-    using NoAttributesCollectionTypeSparse = google::sparse_hash_set<KeyType, DefaultHash<KeyType>>;
-
-    template <typename Value>
-    using CollectionType = std::conditional_t<sparse, CollectionTypeSparse<Value>, CollectionTypeNonSparse<Value>>;
-
-    template <typename Value>
-    using CollectionsHolder = std::vector<CollectionType<Value>>;
-
-    using NoAttributesCollectionType = std::conditional_t<sparse, NoAttributesCollectionTypeSparse, NoAttributesCollectionTypeNonSparse>;
+    using CollectionsHolder = std::vector<typename HashedDictionaryMapType<dictionary_key_type, sparse, KeyType, Value>::Type>;
 
     using NullableSet = HashSet<KeyType, DefaultHash<KeyType>>;
     using NullableSets = std::vector<NullableSet>;
@@ -269,7 +231,7 @@ private:
 
     BlockPtr update_field_loaded_block;
     std::vector<std::unique_ptr<Arena>> string_arenas;
-    std::vector<NoAttributesCollectionType> no_attributes_containers;
+    std::vector<typename HashedDictionarySetType<dictionary_key_type, sparse, KeyType>::Type> no_attributes_containers;
     DictionaryHierarchicalParentToChildIndexPtr hierarchical_index;
 };
 
diff --git a/src/Dictionaries/HashedDictionaryCollectionTraits.h b/src/Dictionaries/HashedDictionaryCollectionTraits.h
new file mode 100644
index 00000000000..20fc882a2fe
--- /dev/null
+++ b/src/Dictionaries/HashedDictionaryCollectionTraits.h
@@ -0,0 +1,102 @@
+#pragma once
+
+#include <type_traits>
+#include <sparsehash/sparse_hash_map>
+#include <Common/HashTable/Hash.h>
+#include <Common/HashTable/HashMap.h>
+#include <Common/HashTable/HashSet.h>
+#include <Common/HashTable/PackedHashMap.h>
+
+namespace DB
+{
+
+/// sparse_hash_map/sparse_hash_set
+template <typename C>
+concept IsGoogleSparseHashTable = std::is_same_v<C, google::sparse_hash_map<
+    typename C::key_type,
+    typename C::mapped_type,
+    /// HashFcn is not exported in sparse_hash_map is public type
+    DefaultHash<typename C::key_type>>>;
+
+template <typename V>
+concept IsStdMapCell = requires (V v)
+{
+    v->first;
+    v->second;
+};
+
+/// HashMap/HashMapWithSavedHash/HashSet/HashMapWithSavedHash/PackedHashMap and their Cells
+template <typename C>
+concept IsBuiltinHashTable = (
+    std::is_same_v<C, HashMapWithSavedHash<
+        typename C::key_type,
+        typename C::mapped_type,
+        DefaultHash<typename C::key_type>,
+        typename C::grower_type>> ||
+    std::is_same_v<C, HashMap<
+        typename C::key_type,
+        typename C::mapped_type,
+        DefaultHash<typename C::key_type>,
+        typename C::grower_type>> ||
+    std::is_same_v<C, PackedHashMap<
+        typename C::key_type,
+        typename C::mapped_type,
+        DefaultHash<typename C::key_type>,
+        typename C::grower_type>> ||
+    std::is_same_v<C, HashSetWithSavedHash<
+        typename C::key_type,
+        DefaultHash<typename C::key_type>,
+        typename C::grower_type>> ||
+    std::is_same_v<C, HashSet<
+        typename C::key_type,
+        DefaultHash<typename C::key_type>,
+        typename C::grower_type>>
+);
+
+template <typename V>
+concept IsBuiltinSetCell = requires (V v)
+{
+    v.getKey();
+};
+
+template <typename V>
+concept IsBuiltinMapCell = requires (V v)
+{
+    v->getKey();
+    v->getMapped();
+};
+
+// NOLINTBEGIN(*)
+
+/// google::sparse_hash_map
+template <typename T> auto getSetKeyFromCell(const T & value) { return value; }
+template <typename T> auto getKeyFromCell(const T & value) requires (IsStdMapCell<T>) { return value->first; }
+template <typename T> auto getValueFromCell(const T & value) requires (IsStdMapCell<T>) { return value->second; }
+
+/// size() - returns table size, without empty and deleted
+/// and since this is sparsehash, empty cells should not be significant,
+/// and since items cannot be removed from the dictionary, deleted is also not important.
+///
+/// NOTE: for google::sparse_hash_set value_type is Key, for sparse_hash_map
+/// value_type is std::pair<Key, Value>, and now we correctly takes into
+/// account padding in structures, if any.
+template <typename C> auto getBufferSizeInBytes(const C & c) requires (IsGoogleSparseHashTable<C>) { return c.size() * sizeof(typename C::value_type); }
+/// bucket_count() - Returns table size, that includes empty and deleted
+template <typename C> auto getBufferSizeInCells(const C & c) requires (IsGoogleSparseHashTable<C>) { return c.bucket_count(); }
+
+template <typename C> auto resizeContainer(C & c, size_t size) requires (IsGoogleSparseHashTable<C>) { return c.resize(size); }
+template <typename C> auto clearContainer(C & c) requires (IsGoogleSparseHashTable<C>) { return c.clear(); }
+
+/// HashMap
+template <typename T> auto getSetKeyFromCell(const T & value) requires (IsBuiltinSetCell<T>) { return value.getKey(); }
+template <typename T> auto getKeyFromCell(const T & value) requires (IsBuiltinMapCell<T>) { return value->getKey(); }
+template <typename T> auto getValueFromCell(const T & value) requires (IsBuiltinMapCell<T>) { return value->getMapped(); }
+
+template <typename C> auto getBufferSizeInBytes(const C & c) requires (IsBuiltinHashTable<C>) { return c.getBufferSizeInBytes(); }
+template <typename C> auto getBufferSizeInCells(const C & c) requires (IsBuiltinHashTable<C>) { return c.getBufferSizeInCells(); }
+template <typename C> auto resizeContainer(C & c, size_t size) requires (IsBuiltinHashTable<C>) { return c.reserve(size); }
+template <typename C> void clearContainer(C & c) requires (IsBuiltinHashTable<C>) { return c.clearAndShrink(); }
+
+// NOLINTEND(*)
+
+}
diff --git a/src/Dictionaries/HashedDictionaryCollectionType.h b/src/Dictionaries/HashedDictionaryCollectionType.h
new file mode 100644
index 00000000000..05bdfbbf00f
--- /dev/null
+++ b/src/Dictionaries/HashedDictionaryCollectionType.h
@@ -0,0 +1,134 @@
+#pragma once
+
+#include <Dictionaries/IDictionary.h>
+#include <Common/HashTable/PackedHashMap.h>
+#include <Common/HashTable/HashMap.h>
+#include <Common/HashTable/HashSet.h>
+#include <sparsehash/sparse_hash_map>
+#include <sparsehash/sparse_hash_set>
+#include <type_traits>
+
+namespace DB
+{
+
+/// HashMap with packed structure is better than google::sparse_hash_map if the
+/// <K, V> pair is small, for the sizeof(std::pair<K, V>) == 16, RSS for hash
+/// table with 1e9 elements will be:
+///
+/// - google::sparse_hash_map             : 26GiB
+/// - HashMap                             : 35GiB
+/// - PackedHashMap                       : 22GiB
+/// - google::sparse_hash_map<packed_pair>: 17GiB
+///
+/// Also note here sizeof(std::pair<>) was used since google::sparse_hash_map
+/// uses it to store <K, V>, yes we can modify google::sparse_hash_map to work
+/// with packed analog of std::pair, but the allocator overhead is still
+/// significant, because of tons of reallocations (and those cannot be solved
+/// with reserve() due to some internals of google::sparse_hash_map) and poor
+/// jemalloc support of such pattern, which results in 33% fragmentation (in
+/// comparison with glibc).
+///
+/// Plus since google::sparse_hash_map cannot use packed structure, it will
+/// have the same memory footprint for everything from UInt8 to UInt64 values
+/// and so on.
+///
+/// Returns true hen google::sparse_hash_map should be used, otherwise
+/// PackedHashMap should be used instead.
+template <typename K, typename V>
+constexpr bool useSparseHashForHashedDictionary()
+{
+    return sizeof(PackedPairNoInit<K, V>) > 16;
+}
+
+///
+/// Map (dictionary with attributes)
+///
+
+/// Type of the hash table for the dictionary.
+template <DictionaryKeyType dictionary_key_type, bool sparse, typename Key, typename Value>
+struct HashedDictionaryMapType;
+
+/// Default implementation using builtin HashMap (for HASHED layout).
+template <DictionaryKeyType dictionary_key_type, typename Key, typename Value>
+struct HashedDictionaryMapType<dictionary_key_type, /* sparse= */ false, Key, Value>
+{
+    using Type = std::conditional_t<
+        dictionary_key_type == DictionaryKeyType::Simple,
+        HashMap<UInt64, Value, DefaultHash<UInt64>>,
+        HashMapWithSavedHash<StringRef, Value, DefaultHash<StringRef>>>;
+};
+
+/// Implementations for SPARSE_HASHED layout.
+template <DictionaryKeyType dictionary_key_type, typename Key, typename Value, bool use_sparse_hash>
+struct HashedDictionarySparseMapType;
+
+/// Implementation based on google::sparse_hash_map for SPARSE_HASHED.
+template <DictionaryKeyType dictionary_key_type, typename Key, typename Value>
+struct HashedDictionarySparseMapType<dictionary_key_type, Key, Value, /* use_sparse_hash= */ true>
+{
+    /// Here we use sparse_hash_map with DefaultHash<> for the following reasons:
+    ///
+    /// - DefaultHash<> is used for HashMap
+    /// - DefaultHash<> (from HashTable/Hash.h> works better then std::hash<>
+    ///   in case of sequential set of keys, but with random access to this set, i.e.
+    ///
+    ///       SELECT number FROM numbers(3000000) ORDER BY rand()
+    ///
+    ///   And even though std::hash<> works better in some other cases,
+    ///   DefaultHash<> is preferred since the difference for this particular
+    ///   case is significant, i.e. it can be 10x+.
+    using Type = std::conditional_t<
+        dictionary_key_type == DictionaryKeyType::Simple,
+        google::sparse_hash_map<UInt64, Value, DefaultHash<Key>>,
+        google::sparse_hash_map<StringRef, Value, DefaultHash<Key>>>;
+};
+
+/// Implementation based on PackedHashMap for SPARSE_HASHED.
+template <DictionaryKeyType dictionary_key_type, typename Key, typename Value>
+struct HashedDictionarySparseMapType<dictionary_key_type, Key, Value, /* use_sparse_hash= */ false>
+{
+    using Type = std::conditional_t<
+        dictionary_key_type == DictionaryKeyType::Simple,
+        PackedHashMap<UInt64, Value, DefaultHash<UInt64>>,
+        PackedHashMap<StringRef, Value, DefaultHash<StringRef>>>;
+};
+template <DictionaryKeyType dictionary_key_type, typename Key, typename Value>
+struct HashedDictionaryMapType<dictionary_key_type, /* sparse= */ true, Key, Value>
+    : public HashedDictionarySparseMapType<
+        dictionary_key_type, Key, Value,
+        /* use_sparse_hash= */ useSparseHashForHashedDictionary<Key, Value>()>
+{};
+
+///
+/// Set (dictionary with attributes)
+///
+
+/// Type of the hash table for the dictionary.
+template <DictionaryKeyType dictionary_key_type, bool sparse, typename Key>
+struct HashedDictionarySetType;
+
+/// Default implementation using builtin HashMap (for HASHED layout).
+template <DictionaryKeyType dictionary_key_type, typename Key>
+struct HashedDictionarySetType<dictionary_key_type, /* sparse= */ false, Key>
+{
+    using Type = std::conditional_t<
+        dictionary_key_type == DictionaryKeyType::Simple,
+        HashSet<UInt64, DefaultHash<UInt64>>,
+        HashSetWithSavedHash<StringRef, DefaultHash<StringRef>>>;
+};
+
+/// Implementation for SPARSE_HASHED.
+///
+/// NOTE: There is no implementation based on google::sparse_hash_set since
+/// PackedHashMap is more optimal anyway (see comments for
+/// useSparseHashForHashedDictionary()).
+template <DictionaryKeyType dictionary_key_type, typename Key>
+struct HashedDictionarySetType<dictionary_key_type, /* sparse= */ true, Key>
+{
+    using Type = std::conditional_t<
+        dictionary_key_type == DictionaryKeyType::Simple,
+        HashSet<UInt64, DefaultHash<UInt64>>,
+        HashSet<StringRef, DefaultHash<StringRef>>>;
+};
+
+}
diff --git a/tests/performance/hashed_dictionary_sharded.xml b/tests/performance/hashed_dictionary_sharded.xml
index 98c65fc49ea..d229ce02c49 100644
--- a/tests/performance/hashed_dictionary_sharded.xml
+++ b/tests/performance/hashed_dictionary_sharded.xml
@@ -22,7 +22,7 @@
         CREATE TABLE simple_key_dictionary_source_table
         (
             id UInt64,
-            value_int UInt64
+            value_int UInt16
         ) ENGINE = Memory
     </create_query>
 

From fb6f7631c20c4a1cc23b4743491804536d0b67ae Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 1 May 2023 20:21:04 +0200
Subject: [PATCH 623/689] Add ability to pass grower for HashTable during
 creation

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Common/HashTable/HashTable.h | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h
index 439b96d6976..0e8510d1c8d 100644
--- a/src/Common/HashTable/HashTable.h
+++ b/src/Common/HashTable/HashTable.h
@@ -771,6 +771,14 @@ public:
         alloc(grower);
     }
 
+    explicit HashTable(const Grower & grower_)
+        : grower(grower_)
+    {
+        if (Cell::need_zero_value_storage)
+            this->zeroValue()->setZero();
+        alloc(grower);
+    }
+
     HashTable(size_t reserve_for_num_elements) /// NOLINT
     {
         if (Cell::need_zero_value_storage)

From 8c6d691f52dc09958088f6b0b5f6820c4a70b47d Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 1 May 2023 21:06:18 +0200
Subject: [PATCH 624/689] Use HashTable constructor in HashSet

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Common/HashTable/HashSet.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Common/HashTable/HashSet.h b/src/Common/HashTable/HashSet.h
index 8f3761599ab..c25bfb14d9c 100644
--- a/src/Common/HashTable/HashSet.h
+++ b/src/Common/HashTable/HashSet.h
@@ -41,6 +41,8 @@ public:
     using Base = HashTable<Key, TCell, Hash, Grower, Allocator>;
     using typename Base::LookupResult;
 
+    using Base::Base;
+
     void merge(const Self & rhs)
     {
         if (!this->hasZero() && rhs.hasZero())

From 3698302ddb32b489aa7e3ad556318f8e0a39f0c0 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 1 May 2023 21:28:20 +0200
Subject: [PATCH 625/689] Accept float values for dictionary layouts
 configurations

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Dictionaries/getDictionaryConfigurationFromAST.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp
index 05065df5251..0b7352e9cbb 100644
--- a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp
+++ b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp
@@ -156,11 +156,11 @@ void buildLayoutConfiguration(
 
         const auto value_field = value_literal->value;
 
-        if (value_field.getType() != Field::Types::UInt64 && value_field.getType() != Field::Types::String)
+        if (value_field.getType() != Field::Types::UInt64 && value_field.getType() != Field::Types::Float64 && value_field.getType() != Field::Types::String)
         {
             throw DB::Exception(
                 ErrorCodes::BAD_ARGUMENTS,
-                "Dictionary layout parameter value must be an UInt64 or String, got '{}' instead",
+                "Dictionary layout parameter value must be an UInt64, Float64 or String, got '{}' instead",
                 value_field.getTypeName());
         }
 

From 2996b3860612b6990867c197d630fc50528ec5b8 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 1 May 2023 20:34:47 +0200
Subject: [PATCH 626/689] Add ability to configure maximum load factor for the
 HASHED/SPARSE_HASHED layout

As it turns out, HashMap/PackedHashMap works great even with max load
factor of 0.99. By "great" I mean it least it works faster then
google sparsehash, and not to mention it's friendliness to the memory
allocator (it has zero fragmentation since it works with a continuious
memory region, in comparison to the sparsehash that doing lots of
realloc, which jemalloc does not like, due to it's slabs).

Here is a table of different setups:

settings                         | load (sec) | read (sec) | read (million rows/s) | bytes_allocated | RSS
-                                | -          | -          | -                     | -               | -
HASHED upstream                  | -          | -          | -                     | -               | 35GiB
SPARSE_HASHED upstream           | -          | -          | -                     | -               | 26GiB
-                                | -          | -          | -                     | -               | -
sparse_hash_map glibc hashbench  | -          | -          | -                     | -               | 17.5GiB
sparse_hash_map packed allocator | 101.878    | 231.48     | 4.32                  | -               | 17.7GiB
PackedHashMap 0.5                | 15.514     | 42.35      | 23.61                 | 20GiB           | 22GiB
hashed 0.95                      | 34.903     | 115.615    | 8.65                  | 16GiB           | 18.7GiB
**PackedHashMap 0.95**           | **93.6**   | **19.883** | **10.68**             | **10GiB**       | **12.8GiB**
PackedHashMap 0.99               | 26.113     | 83.6       | 11.96                 | 10GiB           | 12.3GiB

As it shows, PackedHashMap with 0.95 max_load_factor, eats 2.6x less
memory then SPARSE_HASHED in upstream, and it also 2x faster for read!

v2: fix grower
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docs/en/sql-reference/dictionaries/index.md   |   8 ++
 src/Dictionaries/HashedDictionary.cpp         |  30 ++++-
 src/Dictionaries/HashedDictionary.h           |   1 +
 .../HashedDictionaryCollectionType.h          | 102 +++++++++++++++--
 .../hashed_dictionary_load_factor.xml         |  92 +++++++++++++++
 ...hed_dictionary_load_factor_smoke.reference |  12 ++
 ...40_hashed_dictionary_load_factor_smoke.sql | 107 ++++++++++++++++++
 ...41_hashed_dictionary_load_factor.reference |   4 +
 ...02741_hashed_dictionary_load_factor.sql.j2 |  39 +++++++
 9 files changed, 384 insertions(+), 11 deletions(-)
 create mode 100644 tests/performance/hashed_dictionary_load_factor.xml
 create mode 100644 tests/queries/0_stateless/02740_hashed_dictionary_load_factor_smoke.reference
 create mode 100644 tests/queries/0_stateless/02740_hashed_dictionary_load_factor_smoke.sql
 create mode 100644 tests/queries/0_stateless/02741_hashed_dictionary_load_factor.reference
 create mode 100644 tests/queries/0_stateless/02741_hashed_dictionary_load_factor.sql.j2

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index 5801b7866cb..dfda1ff9c04 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -284,6 +284,13 @@ Configuration example:
          10000 is good balance between memory and speed.
          Even for 10e10 elements and can handle all the load without starvation. -->
     <shard_load_queue_backlog>10000</shard_load_queue_backlog>
+    <!-- Maximum load factor of the hash table, with greater values, the memory
+         is utilized more efficiently (less memory is wasted) but read/performance
+         may deteriorate.
+
+         Valid values: [0.5, 0.99]
+         Default: 0.5 -->
+    <max_load_factor>0.5</max_load_factor>
   </hashed>
 </layout>
 ```
@@ -327,6 +334,7 @@ Configuration example:
   <complex_key_hashed>
     <shards>1</shards>
     <!-- <shard_load_queue_backlog>10000</shard_load_queue_backlog> -->
+    <!-- <max_load_factor>0.5</max_load_factor> -->
   </complex_key_hashed>
 </layout>
 ```
diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp
index 048c378b917..fa9a846195d 100644
--- a/src/Dictionaries/HashedDictionary.cpp
+++ b/src/Dictionaries/HashedDictionary.cpp
@@ -630,6 +630,8 @@ void HashedDictionary<dictionary_key_type, sparse, sharded>::createAttributes()
     const auto size = dict_struct.attributes.size();
     attributes.reserve(size);
 
+    HashTableGrowerWithMaxLoadFactor grower(configuration.max_load_factor);
+
     for (const auto & dictionary_attribute : dict_struct.attributes)
     {
         auto type_call = [&, this](const auto & dictionary_attribute_type)
@@ -639,8 +641,23 @@ void HashedDictionary<dictionary_key_type, sparse, sharded>::createAttributes()
             using ValueType = DictionaryValueType<AttributeType>;
 
             auto is_nullable_sets = dictionary_attribute.is_nullable ? std::make_optional<NullableSets>(configuration.shards) : std::optional<NullableSets>{};
-            Attribute attribute{dictionary_attribute.underlying_type, std::move(is_nullable_sets), CollectionsHolder<ValueType>(configuration.shards)};
-            attributes.emplace_back(std::move(attribute));
+            if constexpr (IsBuiltinHashTable<typename CollectionsHolder<ValueType>::value_type>)
+            {
+                CollectionsHolder<ValueType> collections;
+                collections.reserve(configuration.shards);
+                for (size_t i = 0; i < configuration.shards; ++i)
+                    collections.emplace_back(grower);
+
+                Attribute attribute{dictionary_attribute.underlying_type, std::move(is_nullable_sets), std::move(collections)};
+                attributes.emplace_back(std::move(attribute));
+            }
+            else
+            {
+                Attribute attribute{dictionary_attribute.underlying_type, std::move(is_nullable_sets), CollectionsHolder<ValueType>(configuration.shards)};
+                for (auto & container : std::get<CollectionsHolder<ValueType>>(attribute.containers))
+                    container.max_load_factor(configuration.max_load_factor);
+                attributes.emplace_back(std::move(attribute));
+            }
         };
 
         callOnDictionaryAttributeType(dictionary_attribute.underlying_type, type_call);
@@ -648,7 +665,9 @@ void HashedDictionary<dictionary_key_type, sparse, sharded>::createAttributes()
 
     if (unlikely(attributes.size()) == 0)
     {
-        no_attributes_containers.resize(configuration.shards);
+        no_attributes_containers.reserve(configuration.shards);
+        for (size_t i = 0; i < configuration.shards; ++i)
+            no_attributes_containers.emplace_back(grower);
     }
 
     string_arenas.resize(configuration.shards);
@@ -1136,9 +1155,14 @@ void registerDictionaryHashed(DictionaryFactory & factory)
         if (shard_load_queue_backlog <= 0)
             throw Exception(ErrorCodes::BAD_ARGUMENTS,"{}: SHARD_LOAD_QUEUE_BACKLOG parameter should be greater then zero", full_name);
 
+        float max_load_factor = static_cast<float>(config.getDouble(config_prefix + dictionary_layout_prefix + ".max_load_factor", 0.5));
+        if (max_load_factor < 0.5 || max_load_factor > 0.99)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: max_load_factor parameter should be within [0.5, 0.99], got {}", full_name, max_load_factor);
+
         HashedDictionaryConfiguration configuration{
             static_cast<UInt64>(shards),
             static_cast<UInt64>(shard_load_queue_backlog),
+            max_load_factor,
             require_nonempty,
             dict_lifetime,
         };
diff --git a/src/Dictionaries/HashedDictionary.h b/src/Dictionaries/HashedDictionary.h
index 08f5dfaa40d..bff22706435 100644
--- a/src/Dictionaries/HashedDictionary.h
+++ b/src/Dictionaries/HashedDictionary.h
@@ -25,6 +25,7 @@ struct HashedDictionaryConfiguration
 {
     const UInt64 shards;
     const UInt64 shard_load_queue_backlog;
+    const float max_load_factor;
     const bool require_nonempty;
     const DictionaryLifetime lifetime;
 };
diff --git a/src/Dictionaries/HashedDictionaryCollectionType.h b/src/Dictionaries/HashedDictionaryCollectionType.h
index 05bdfbbf00f..1ab0ff8b0e3 100644
--- a/src/Dictionaries/HashedDictionaryCollectionType.h
+++ b/src/Dictionaries/HashedDictionaryCollectionType.h
@@ -40,6 +40,92 @@ constexpr bool useSparseHashForHashedDictionary()
     return sizeof(PackedPairNoInit<K, V>) > 16;
 }
 
+/// Grower with custom fill limit/load factor (instead of default 50%).
+///
+/// It turns out that HashMap can outperform google::sparse_hash_map in case of
+/// the structure size of not big, in terms of speed *and* memory. Even 99% of
+/// max load factor was faster then google::sparse_hash_map in my simple tests
+/// (1e9 UInt64 keys with UInt16 values, randomly distributed).
+///
+/// And not to mention very high allocator memory fragmentation in
+/// google::sparse_hash_map.
+///
+/// Based on HashTableGrowerWithPrecalculation
+class alignas(64) HashTableGrowerWithMaxLoadFactor
+{
+    static constexpr size_t initial_size_degree = 8;
+    UInt8 size_degree = initial_size_degree;
+    size_t precalculated_mask = (1ULL << initial_size_degree) - 1;
+    size_t precalculated_max_fill = 1ULL << (initial_size_degree - 1);
+    float max_load_factor = 0.5;
+    /// HashTableGrowerWithPrecalculation has 23, but to decrease memory usage
+    /// at least slightly 19 is used here. Also note, that for dictionaries it
+    /// is not that important since they are not that frequently loaded.
+    static constexpr size_t max_size_degree_quadratic = 19;
+
+public:
+    static constexpr auto initial_count = 1ULL << initial_size_degree;
+
+    /// If collision resolution chains are contiguous, we can implement erase operation by moving the elements.
+    static constexpr auto performs_linear_probing_with_single_step = true;
+
+    HashTableGrowerWithMaxLoadFactor() = default;
+    explicit HashTableGrowerWithMaxLoadFactor(float max_load_factor_)
+        : max_load_factor(max_load_factor_)
+    {
+        increaseSizeDegree(0);
+    }
+
+    UInt8 sizeDegree() const { return size_degree; }
+
+    void increaseSizeDegree(UInt8 delta)
+    {
+        size_degree += delta;
+        precalculated_mask = (1ULL << size_degree) - 1;
+        precalculated_max_fill = static_cast<size_t>((1ULL << size_degree) * max_load_factor);
+    }
+
+    /// The size of the hash table in the cells.
+    size_t bufSize() const { return 1ULL << size_degree; }
+
+    /// From the hash value, get the cell number in the hash table.
+    size_t place(size_t x) const { return x & precalculated_mask; }
+
+    /// The next cell in the collision resolution chain.
+    size_t next(size_t pos) const { return (pos + 1) & precalculated_mask; }
+
+    /// Whether the hash table is sufficiently full. You need to increase the size of the hash table, or remove something unnecessary from it.
+    bool overflow(size_t elems) const { return elems > precalculated_max_fill; }
+
+    /// Increase the size of the hash table.
+    void increaseSize() { increaseSizeDegree(size_degree >= max_size_degree_quadratic ? 1 : 2); }
+
+    /// Set the buffer size by the number of elements in the hash table. Used when deserializing a hash table.
+    void set(size_t num_elems)
+    {
+        if (num_elems <= 1)
+            size_degree = initial_size_degree;
+        else if (initial_size_degree > static_cast<size_t>(log2(num_elems - 1)) + 2)
+            size_degree = initial_size_degree;
+        else
+        {
+            /// Slightly more optimal than HashTableGrowerWithPrecalculation
+            /// and takes into account max_load_factor.
+            size_degree = static_cast<size_t>(log2(num_elems - 1)) + 1;
+            if ((1ULL << size_degree) * max_load_factor < num_elems)
+                ++size_degree;
+        }
+        increaseSizeDegree(0);
+    }
+
+    void setBufSize(size_t buf_size_)
+    {
+        size_degree = static_cast<size_t>(log2(buf_size_ - 1) + 1);
+        increaseSizeDegree(0);
+    }
+};
+static_assert(sizeof(HashTableGrowerWithMaxLoadFactor) == 64);
+
 ///
 /// Map (dictionary with attributes)
 ///
@@ -54,8 +140,8 @@ struct HashedDictionaryMapType<dictionary_key_type, /* sparse= */ false, Key, Va
 {
     using Type = std::conditional_t<
         dictionary_key_type == DictionaryKeyType::Simple,
-        HashMap<UInt64, Value, DefaultHash<UInt64>>,
-        HashMapWithSavedHash<StringRef, Value, DefaultHash<StringRef>>>;
+        HashMap<UInt64, Value, DefaultHash<UInt64>, HashTableGrowerWithMaxLoadFactor>,
+        HashMapWithSavedHash<StringRef, Value, DefaultHash<StringRef>, HashTableGrowerWithMaxLoadFactor>>;
 };
 
 /// Implementations for SPARSE_HASHED layout.
@@ -89,8 +175,8 @@ struct HashedDictionarySparseMapType<dictionary_key_type, Key, Value, /* use_spa
 {
     using Type = std::conditional_t<
         dictionary_key_type == DictionaryKeyType::Simple,
-        PackedHashMap<UInt64, Value, DefaultHash<UInt64>>,
-        PackedHashMap<StringRef, Value, DefaultHash<StringRef>>>;
+        PackedHashMap<UInt64, Value, DefaultHash<UInt64>, HashTableGrowerWithMaxLoadFactor>,
+        PackedHashMap<StringRef, Value, DefaultHash<StringRef>, HashTableGrowerWithMaxLoadFactor>>;
 };
 template <DictionaryKeyType dictionary_key_type, typename Key, typename Value>
 struct HashedDictionaryMapType<dictionary_key_type, /* sparse= */ true, Key, Value>
@@ -113,8 +199,8 @@ struct HashedDictionarySetType<dictionary_key_type, /* sparse= */ false, Key>
 {
     using Type = std::conditional_t<
         dictionary_key_type == DictionaryKeyType::Simple,
-        HashSet<UInt64, DefaultHash<UInt64>>,
-        HashSetWithSavedHash<StringRef, DefaultHash<StringRef>>>;
+        HashSet<UInt64, DefaultHash<UInt64>, HashTableGrowerWithMaxLoadFactor>,
+        HashSetWithSavedHash<StringRef, DefaultHash<StringRef>, HashTableGrowerWithMaxLoadFactor>>;
 };
 
 /// Implementation for SPARSE_HASHED.
@@ -127,8 +213,8 @@ struct HashedDictionarySetType<dictionary_key_type, /* sparse= */ true, Key>
 {
     using Type = std::conditional_t<
         dictionary_key_type == DictionaryKeyType::Simple,
-        HashSet<UInt64, DefaultHash<UInt64>>,
-        HashSet<StringRef, DefaultHash<StringRef>>>;
+        HashSet<UInt64, DefaultHash<UInt64>, HashTableGrowerWithMaxLoadFactor>,
+        HashSet<StringRef, DefaultHash<StringRef>, HashTableGrowerWithMaxLoadFactor>>;
 };
 
 }
diff --git a/tests/performance/hashed_dictionary_load_factor.xml b/tests/performance/hashed_dictionary_load_factor.xml
new file mode 100644
index 00000000000..dfc5c64b133
--- /dev/null
+++ b/tests/performance/hashed_dictionary_load_factor.xml
@@ -0,0 +1,92 @@
+<test>
+    <substitutions>
+        <substitution>
+            <name>layout_suffix</name>
+            <values>
+                <value>HASHED</value>
+                <value>SPARSE_HASHED</value>
+            </values>
+        </substitution>
+
+        <substitution>
+            <name>load_factor</name>
+            <values>
+                <!-- 0. will be prepended -->
+                <value>5</value>
+                <value>7</value>
+                <value>99</value>
+            </values>
+        </substitution>
+    </substitutions>
+
+    <create_query>
+        CREATE TABLE simple_key_dictionary_source_table
+        (
+            id UInt64,
+            value_int UInt16
+        ) ENGINE = Memory
+    </create_query>
+
+    <create_query>
+        CREATE TABLE complex_key_dictionary_source_table
+        (
+            id UInt64,
+            id_key String,
+            value_int UInt64
+        ) ENGINE = Memory
+    </create_query>
+
+    <create_query>
+        CREATE DICTIONARY IF NOT EXISTS simple_key_{layout_suffix}_dictionary_l0_{load_factor}
+        (
+            id UInt64,
+            value_int UInt64
+        )
+        PRIMARY KEY id
+        SOURCE(CLICKHOUSE(TABLE 'simple_key_dictionary_source_table'))
+        LAYOUT({layout_suffix}(MAX_LOAD_FACTOR 0.{load_factor}))
+        LIFETIME(0)
+    </create_query>
+
+    <create_query>
+        CREATE DICTIONARY IF NOT EXISTS complex_key_{layout_suffix}_dictionary_l0_{load_factor}
+        (
+            id UInt64,
+            id_key String,
+            value_int UInt64
+        )
+        PRIMARY KEY id, id_key
+        SOURCE(CLICKHOUSE(TABLE 'complex_key_dictionary_source_table'))
+        LAYOUT(COMPLEX_KEY_{layout_suffix}(MAX_LOAD_FACTOR 0.{load_factor}))
+        LIFETIME(0)
+    </create_query>
+
+    <fill_query>INSERT INTO simple_key_dictionary_source_table SELECT number, number FROM numbers(3_000_000)</fill_query>
+    <fill_query>INSERT INTO complex_key_dictionary_source_table SELECT number, toString(number), number FROM numbers(2_000_000)</fill_query>
+
+    <fill_query>SYSTEM RELOAD DICTIONARY simple_key_{layout_suffix}_dictionary_l0_{load_factor}</fill_query>
+    <fill_query>SYSTEM RELOAD DICTIONARY complex_key_{layout_suffix}_dictionary_l0_{load_factor}</fill_query>
+
+    <query>SYSTEM RELOAD DICTIONARY simple_key_{layout_suffix}_dictionary_l0_{load_factor}</query>
+    <query>SYSTEM RELOAD DICTIONARY complex_key_{layout_suffix}_dictionary_l0_{load_factor}</query>
+
+    <query>
+        WITH rand64() % 3_000_000 as key
+        SELECT dictHas('default.simple_key_{layout_suffix}_dictionary_l0_{load_factor}', key)
+        FROM numbers(3_000_000)
+        FORMAT Null
+    </query>
+
+    <query>
+        WITH (rand64() % 2_000_000, toString(rand64() % 2_000_000)) as key
+        SELECT dictHas('default.complex_key_{layout_suffix}_dictionary_l0_{load_factor}', key)
+        FROM numbers(2_000_000)
+        FORMAT Null
+    </query>
+
+    <drop_query>DROP DICTIONARY simple_key_{layout_suffix}_dictionary_l0_{load_factor}</drop_query>
+    <drop_query>DROP DICTIONARY complex_key_{layout_suffix}_dictionary_l0_{load_factor}</drop_query>
+
+    <drop_query>DROP TABLE simple_key_dictionary_source_table</drop_query>
+    <drop_query>DROP TABLE complex_key_dictionary_source_table</drop_query>
+</test>
diff --git a/tests/queries/0_stateless/02740_hashed_dictionary_load_factor_smoke.reference b/tests/queries/0_stateless/02740_hashed_dictionary_load_factor_smoke.reference
new file mode 100644
index 00000000000..d43fa8e734c
--- /dev/null
+++ b/tests/queries/0_stateless/02740_hashed_dictionary_load_factor_smoke.reference
@@ -0,0 +1,12 @@
+CREATE DICTIONARY default.test_sparse_dictionary_load_factor\n(\n    `key` UInt64,\n    `value` UInt16\n)\nPRIMARY KEY key\nSOURCE(CLICKHOUSE(TABLE test_table))\nLIFETIME(MIN 0 MAX 0)\nLAYOUT(SPARSE_HASHED(MAX_LOAD_FACTOR 0.9))
+100000
+0
+CREATE DICTIONARY default.test_dictionary_load_factor\n(\n    `key` UInt64,\n    `value` UInt16\n)\nPRIMARY KEY key\nSOURCE(CLICKHOUSE(TABLE test_table))\nLIFETIME(MIN 0 MAX 0)\nLAYOUT(HASHED(MAX_LOAD_FACTOR 0.9))
+100000
+0
+CREATE DICTIONARY default.test_dictionary_load_factor_nullable\n(\n    `key` UInt64,\n    `value` Nullable(UInt16)\n)\nPRIMARY KEY key\nSOURCE(CLICKHOUSE(TABLE test_table_nullable))\nLIFETIME(MIN 0 MAX 0)\nLAYOUT(HASHED(MAX_LOAD_FACTOR 0.9))
+100000
+0
+CREATE DICTIONARY default.test_complex_dictionary_load_factor\n(\n    `key_1` UInt64,\n    `key_2` UInt64,\n    `value` UInt16\n)\nPRIMARY KEY key_1, key_2\nSOURCE(CLICKHOUSE(TABLE test_table_complex))\nLIFETIME(MIN 0 MAX 0)\nLAYOUT(COMPLEX_KEY_HASHED(MAX_LOAD_FACTOR 0.9))
+100000
+0
diff --git a/tests/queries/0_stateless/02740_hashed_dictionary_load_factor_smoke.sql b/tests/queries/0_stateless/02740_hashed_dictionary_load_factor_smoke.sql
new file mode 100644
index 00000000000..d4bb9a1b14a
--- /dev/null
+++ b/tests/queries/0_stateless/02740_hashed_dictionary_load_factor_smoke.sql
@@ -0,0 +1,107 @@
+DROP TABLE IF EXISTS test_table;
+CREATE TABLE test_table
+(
+    key UInt64,
+    value UInt16
+) ENGINE=Memory() AS SELECT number, number FROM numbers(1e5);
+
+DROP TABLE IF EXISTS test_table_nullable;
+CREATE TABLE test_table_nullable
+(
+    key UInt64,
+    value Nullable(UInt16)
+) ENGINE=Memory() AS SELECT number, number % 2 == 0 ? NULL : number FROM numbers(1e5);
+
+DROP TABLE IF EXISTS test_table_string;
+CREATE TABLE test_table_string
+(
+    key String,
+    value UInt16
+) ENGINE=Memory() AS SELECT 'foo' || number::String, number FROM numbers(1e5);
+
+DROP TABLE IF EXISTS test_table_complex;
+CREATE TABLE test_table_complex
+(
+    key_1 UInt64,
+    key_2 UInt64,
+    value UInt16
+) ENGINE=Memory() AS SELECT number, number, number FROM numbers(1e5);
+
+DROP DICTIONARY IF EXISTS test_sparse_dictionary_load_factor;
+CREATE DICTIONARY test_sparse_dictionary_load_factor
+(
+    key UInt64,
+    value UInt16
+) PRIMARY KEY key
+SOURCE(CLICKHOUSE(TABLE test_table))
+LAYOUT(SPARSE_HASHED(MAX_LOAD_FACTOR 0.90))
+LIFETIME(0);
+SHOW CREATE test_sparse_dictionary_load_factor;
+SYSTEM RELOAD DICTIONARY test_sparse_dictionary_load_factor;
+SELECT element_count FROM system.dictionaries WHERE database = currentDatabase() AND name = 'test_sparse_dictionary_load_factor';
+SELECT count() FROM test_table WHERE dictGet('test_sparse_dictionary_load_factor', 'value', key) != value;
+DROP DICTIONARY test_sparse_dictionary_load_factor;
+
+DROP DICTIONARY IF EXISTS test_dictionary_load_factor;
+CREATE DICTIONARY test_dictionary_load_factor
+(
+    key UInt64,
+    value UInt16
+) PRIMARY KEY key
+SOURCE(CLICKHOUSE(TABLE test_table))
+LAYOUT(HASHED(MAX_LOAD_FACTOR 0.90))
+LIFETIME(0);
+SHOW CREATE test_dictionary_load_factor;
+SYSTEM RELOAD DICTIONARY test_dictionary_load_factor;
+SELECT element_count FROM system.dictionaries WHERE database = currentDatabase() AND name = 'test_dictionary_load_factor';
+SELECT count() FROM test_table WHERE dictGet('test_dictionary_load_factor', 'value', key) != value;
+DROP DICTIONARY test_dictionary_load_factor;
+
+DROP DICTIONARY IF EXISTS test_dictionary_load_factor_nullable;
+CREATE DICTIONARY test_dictionary_load_factor_nullable
+(
+    key UInt64,
+    value Nullable(UInt16)
+) PRIMARY KEY key
+SOURCE(CLICKHOUSE(TABLE test_table_nullable))
+LAYOUT(HASHED(MAX_LOAD_FACTOR 0.90))
+LIFETIME(0);
+SHOW CREATE test_dictionary_load_factor_nullable;
+SYSTEM RELOAD DICTIONARY test_dictionary_load_factor_nullable;
+SELECT element_count FROM system.dictionaries WHERE database = currentDatabase() AND name = 'test_dictionary_load_factor_nullable';
+SELECT count() FROM test_table_nullable WHERE dictGet('test_dictionary_load_factor_nullable', 'value', key) != value;
+DROP DICTIONARY test_dictionary_load_factor_nullable;
+
+DROP DICTIONARY IF EXISTS test_complex_dictionary_load_factor;
+CREATE DICTIONARY test_complex_dictionary_load_factor
+(
+    key_1 UInt64,
+    key_2 UInt64,
+    value UInt16
+) PRIMARY KEY key_1, key_2
+SOURCE(CLICKHOUSE(TABLE test_table_complex))
+LAYOUT(COMPLEX_KEY_HASHED(MAX_LOAD_FACTOR 0.90))
+LIFETIME(0);
+SYSTEM RELOAD DICTIONARY test_complex_dictionary_load_factor;
+SHOW CREATE test_complex_dictionary_load_factor;
+SELECT element_count FROM system.dictionaries WHERE database = currentDatabase() and name = 'test_complex_dictionary_load_factor';
+SELECT count() FROM test_table_complex WHERE dictGet('test_complex_dictionary_load_factor', 'value', (key_1, key_2)) != value;
+DROP DICTIONARY test_complex_dictionary_load_factor;
+
+DROP DICTIONARY IF EXISTS test_dictionary_load_factor_string;
+CREATE DICTIONARY test_dictionary_load_factor_string
+(
+    key String,
+    value UInt16
+) PRIMARY KEY key
+SOURCE(CLICKHOUSE(TABLE test_table_string))
+LAYOUT(HASHED(MAX_LOAD_FACTOR 1))
+LIFETIME(0);
+-- should because of MAX_LOAD_FACTOR is 1 (maximum allowed value is 0.99)
+SYSTEM RELOAD DICTIONARY test_dictionary_load_factor_string; -- { serverError BAD_ARGUMENTS }
+DROP DICTIONARY test_dictionary_load_factor_string;
+
+DROP TABLE test_table;
+DROP TABLE test_table_nullable;
+DROP TABLE test_table_string;
+DROP TABLE test_table_complex;
diff --git a/tests/queries/0_stateless/02741_hashed_dictionary_load_factor.reference b/tests/queries/0_stateless/02741_hashed_dictionary_load_factor.reference
new file mode 100644
index 00000000000..abe891cbb9b
--- /dev/null
+++ b/tests/queries/0_stateless/02741_hashed_dictionary_load_factor.reference
@@ -0,0 +1,4 @@
+test_dictionary_hashed	1000000	0.4768	33558760
+test_dictionary_hashed_load_factor	1000000	0.9537	16781544
+test_dictionary_sparse_hashed	1000000	0.4768	20975848
+test_dictionary_sparse_hashed_load_factor	1000000	0.9537	10490088
diff --git a/tests/queries/0_stateless/02741_hashed_dictionary_load_factor.sql.j2 b/tests/queries/0_stateless/02741_hashed_dictionary_load_factor.sql.j2
new file mode 100644
index 00000000000..870acd54514
--- /dev/null
+++ b/tests/queries/0_stateless/02741_hashed_dictionary_load_factor.sql.j2
@@ -0,0 +1,39 @@
+{# vi: ft=sql #}
+
+{% for layout in ["hashed", "sparse_hashed"] %}
+DROP DICTIONARY IF EXISTS test_dictionary_{{layout}};
+DROP DICTIONARY IF EXISTS test_dictionary_{{layout}}_load_factor;
+DROP TABLE IF EXISTS test_table;
+
+CREATE TABLE test_table
+(
+    key UInt64,
+    value UInt16
+) ENGINE=Memory() AS SELECT number, number FROM numbers(1e6);
+
+CREATE DICTIONARY test_dictionary_{{layout}}
+(
+    key UInt64,
+    value UInt16
+) PRIMARY KEY key
+SOURCE(CLICKHOUSE(TABLE test_table))
+LAYOUT({{layout}}())
+LIFETIME(0);
+
+CREATE DICTIONARY test_dictionary_{{layout}}_load_factor
+(
+    key UInt64,
+    value UInt16
+) PRIMARY KEY key
+SOURCE(CLICKHOUSE(TABLE test_table))
+LAYOUT({{layout}}(MAX_LOAD_FACTOR 0.98))
+LIFETIME(0);
+
+SYSTEM RELOAD DICTIONARY test_dictionary_{{layout}};
+SYSTEM RELOAD DICTIONARY test_dictionary_{{layout}}_load_factor;
+SELECT name, element_count, round(load_factor, 4), bytes_allocated FROM system.dictionaries WHERE database = currentDatabase() ORDER BY name;
+
+DROP DICTIONARY IF EXISTS test_dictionary_{{layout}};
+DROP DICTIONARY IF EXISTS test_dictionary_{{layout}}_load_factor;
+DROP TABLE test_table;
+{% endfor %}

From 7c8d8eeb5677a22e324c1c2dd36c4036d21bce0e Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 2 May 2023 20:55:47 +0200
Subject: [PATCH 627/689] Use Cell::setMapped() over separate helper
 insertSetMapped()

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Common/HashTable/FixedHashTable.h    |  2 +-
 src/Common/HashTable/HashTable.h         | 15 ++-------------
 src/Common/HashTable/TwoLevelHashTable.h |  2 +-
 3 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/src/Common/HashTable/FixedHashTable.h b/src/Common/HashTable/FixedHashTable.h
index 7df90fd98b9..49675aaafbc 100644
--- a/src/Common/HashTable/FixedHashTable.h
+++ b/src/Common/HashTable/FixedHashTable.h
@@ -358,7 +358,7 @@ public:
         std::pair<LookupResult, bool> res;
         emplace(Cell::getKey(x), res.first, res.second);
         if (res.second)
-            insertSetMapped(res.first->getMapped(), x);
+            res.first->setMapped(x);
 
         return res;
     }
diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h
index 0e8510d1c8d..a48cb8d49f2 100644
--- a/src/Common/HashTable/HashTable.h
+++ b/src/Common/HashTable/HashTable.h
@@ -117,7 +117,7 @@ inline bool bitEquals(T && a, T && b)
   * 3) Hash tables that store the key and do not have a "mapped" value, e.g. the normal HashTable.
   *    GetKey returns the key, and GetMapped returns a zero void pointer. This simplifies generic
   *    code that works with mapped values: it can overload on the return type of GetMapped(), and
-  *    doesn't need other parameters. One example is insertSetMapped() function.
+  *    doesn't need other parameters. One example is Cell::setMapped() function.
   *
   * 4) Hash tables that store both the key and the "mapped" value, e.g. HashMap. Both GetKey and
   *    GetMapped are supported.
@@ -216,17 +216,6 @@ struct HashTableCell
 
 };
 
-/**
-  * A helper function for HashTable::insert() to set the "mapped" value.
-  * Overloaded on the mapped type, does nothing if it's VoidMapped.
-  */
-template <typename ValueType>
-void insertSetMapped(VoidMapped /* dest */, const ValueType & /* src */) {}
-
-template <typename MappedType, typename ValueType>
-void insertSetMapped(MappedType & dest, const ValueType & src) { dest = src.second; }
-
-
 /** Determines the size of the hash table, and when and how much it should be resized.
   * Has very small state (one UInt8) and useful for Set-s allocated in automatic memory (see uniqExact as an example).
   */
@@ -1046,7 +1035,7 @@ public:
         }
 
         if (res.second)
-            insertSetMapped(res.first->getMapped(), x);
+            res.first->setMapped(x);
 
         return res;
     }
diff --git a/src/Common/HashTable/TwoLevelHashTable.h b/src/Common/HashTable/TwoLevelHashTable.h
index bd4c4c366f2..a60655e4072 100644
--- a/src/Common/HashTable/TwoLevelHashTable.h
+++ b/src/Common/HashTable/TwoLevelHashTable.h
@@ -224,7 +224,7 @@ public:
         emplace(Cell::getKey(x), res.first, res.second, hash_value);
 
         if (res.second)
-            insertSetMapped(res.first->getMapped(), x);
+            res.first->setMapped(x);
 
         return res;
     }

From 65dd87d0da27feea18a3a5ea8b94c24213209357 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 2 May 2023 17:01:20 +0200
Subject: [PATCH 628/689] Fix "reference binding to misaligned address" in
 PackedHashMap

Use separate helpers that accept/return values, instead of reference,
anyway PackedHashMap is developed for small structure.

v0: fix for keys
v2: fix for values
v3: fix bitEquals
v4: fix for iterating over HashMap
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 base/base/StringRef.h                | 12 ++++++
 src/Common/HashTable/PackedHashMap.h | 63 +++++++++++++++++++++++++++-
 2 files changed, 74 insertions(+), 1 deletion(-)

diff --git a/base/base/StringRef.h b/base/base/StringRef.h
index a3e32ff5058..f428b7c747f 100644
--- a/base/base/StringRef.h
+++ b/base/base/StringRef.h
@@ -3,6 +3,7 @@
 #include <cassert>
 #include <stdexcept> // for std::logic_error
 #include <string>
+#include <type_traits>
 #include <vector>
 #include <functional>
 #include <iosfwd>
@@ -326,5 +327,16 @@ namespace ZeroTraits
     inline void set(StringRef & x) { x.size = 0; }
 }
 
+namespace PackedZeroTraits
+{
+    template <typename Second, template <typename, typename> class PackedPairNoInit>
+    inline bool check(const PackedPairNoInit<StringRef, Second> p)
+    { return 0 == p.key.size; }
+
+    template <typename Second, template <typename, typename> class PackedPairNoInit>
+    inline void set(PackedPairNoInit<StringRef, Second> & p)
+    { p.key.size = 0; }
+}
+
 
 std::ostream & operator<<(std::ostream & os, const StringRef & str);
diff --git a/src/Common/HashTable/PackedHashMap.h b/src/Common/HashTable/PackedHashMap.h
index 6b52a04aa49..6432c299b9d 100644
--- a/src/Common/HashTable/PackedHashMap.h
+++ b/src/Common/HashTable/PackedHashMap.h
@@ -30,6 +30,67 @@ struct PackedPairNoInit
     }
 } __attribute__((packed));
 
+/// The difference with ZeroTraits is that PackedZeroTraits accepts PackedPairNoInit instead of Key.
+namespace PackedZeroTraits
+{
+    template <typename First, typename Second, template <typename, typename> class PackedPairNoInit>
+    bool check(const PackedPairNoInit<First, Second> p) { return p.first == First{}; }
+
+    template <typename First, typename Second, template <typename, typename> class PackedPairNoInit>
+    void set(PackedPairNoInit<First, Second> & p) { p.first = First{}; }
+}
+
+/// setZero() should be overwritten to pass the pair instead of key, to avoid
+/// "reference binding to misaligned address" errors from UBsan.
+template <typename Key, typename TMapped, typename Hash, typename TState = HashTableNoState>
+struct PackedHashMapCell : public HashMapCell<Key, TMapped, Hash, TState, PackedPairNoInit<Key, TMapped>>
+{
+    using Base = HashMapCell<Key, TMapped, Hash, TState, PackedPairNoInit<Key, TMapped>>;
+    using State = typename Base::State;
+    using value_type = typename Base::value_type;
+    using key_type = typename Base::key_type;
+    using Mapped = typename Base::Mapped;
+
+    using Base::Base;
+
+    void setZero() { PackedZeroTraits::set(this->value); }
+
+    Key getKey() const { return this->value.first; }
+    static Key getKey(const value_type & value_) { return value_.first; }
+
+    Mapped & getMapped() { return this->value.second; }
+    Mapped getMapped() const { return this->value.second; }
+    value_type getValue() const { return this->value; }
+
+    bool keyEquals(const Key key_) const { return bitEqualsByValue(this->value.first, key_); }
+    bool keyEquals(const Key key_, size_t /*hash_*/) const { return bitEqualsByValue(this->value.first, key_); }
+    bool keyEquals(const Key key_, size_t /*hash_*/, const State & /*state*/) const { return bitEqualsByValue(this->value.first, key_); }
+
+    bool isZero(const State & state) const { return isZero(this->value.first, state); }
+    static bool isZero(const Key key, const State & /*state*/) { return ZeroTraits::check(key); }
+
+    static inline bool bitEqualsByValue(key_type a, key_type b) { return a == b; }
+
+    template <size_t I>
+    auto get() const
+    {
+        if constexpr (I == 0) return this->value.first;
+        else if constexpr (I == 1) return this->value.second;
+    }
+};
+
+namespace std
+{
+    template <typename Key, typename TMapped, typename Hash, typename TState>
+    struct tuple_size<PackedHashMapCell<Key, TMapped, Hash, TState>> : std::integral_constant<size_t, 2> { };
+
+    template <typename Key, typename TMapped, typename Hash, typename TState>
+    struct tuple_element<0, PackedHashMapCell<Key, TMapped, Hash, TState>> { using type = Key; };
+
+    template <typename Key, typename TMapped, typename Hash, typename TState>
+    struct tuple_element<1, PackedHashMapCell<Key, TMapped, Hash, TState>> { using type = TMapped; };
+}
+
 /// Packed HashMap - HashMap with structure without padding
 ///
 /// Sometimes padding in structure can be crucial, consider the following
@@ -43,4 +104,4 @@ template <
     typename Hash = DefaultHash<Key>,
     typename Grower = HashTableGrower<>,
     typename Allocator = HashTableAllocator>
-using PackedHashMap = HashMapTable<Key, HashMapCell<Key, Mapped, Hash, HashTableNoState, PackedPairNoInit<Key, Mapped>>, Hash, Grower, Allocator>;
+using PackedHashMap = HashMapTable<Key, PackedHashMapCell<Key, Mapped, Hash, HashTableNoState>, Hash, Grower, Allocator>;

From fc19e79f50686b7ed94165ac7827b088611a5bf7 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 18:14:40 +0200
Subject: [PATCH 629/689] Change coding style of declaring packed attribute in
 PackedHashMap

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Common/HashTable/PackedHashMap.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/HashTable/PackedHashMap.h b/src/Common/HashTable/PackedHashMap.h
index 6432c299b9d..0d25addb58e 100644
--- a/src/Common/HashTable/PackedHashMap.h
+++ b/src/Common/HashTable/PackedHashMap.h
@@ -9,7 +9,7 @@
 /// NOTE: makePairNoInit() is omitted for PackedPairNoInit since it is not
 /// required for PackedHashMap (see mergeBlockWithPipe() for details)
 template <typename First, typename Second>
-struct PackedPairNoInit
+struct __attribute__((packed)) PackedPairNoInit
 {
     First first;
     Second second;
@@ -28,7 +28,7 @@ struct PackedPairNoInit
         , second(std::forward<SecondValue>(second_))
     {
     }
-} __attribute__((packed));
+};
 
 /// The difference with ZeroTraits is that PackedZeroTraits accepts PackedPairNoInit instead of Key.
 namespace PackedZeroTraits

From 422cbe08fe83ca83f8ce24c48a811a1a0202abdb Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 12:46:26 +0200
Subject: [PATCH 630/689] Do not use PackedHashMap for non-POD for the purposes
 of layout

In clang-16 the behaviour for POD types had been changed in [1], this
does not allows us to use PackedHashMap for some types.

  [1]: https://github.com/llvm/llvm-project/commit/277123376ce08c98b07c154bf83e4092a5d4d3c6

Note, that I tried to come up with a more generic solution then
enumeratic types, but failed. Though now I think that this is good,
since this shows which types are not allowed for PackedHashMap

Another option is to use -fclang-abi-compat=13.0 but I doubt it is a
good idea.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../HashedDictionaryCollectionType.h          | 36 ++++++++++++++++++-
 src/Interpreters/examples/hash_map.cpp        | 30 ----------------
 2 files changed, 35 insertions(+), 31 deletions(-)

diff --git a/src/Dictionaries/HashedDictionaryCollectionType.h b/src/Dictionaries/HashedDictionaryCollectionType.h
index 1ab0ff8b0e3..23a08601fff 100644
--- a/src/Dictionaries/HashedDictionaryCollectionType.h
+++ b/src/Dictionaries/HashedDictionaryCollectionType.h
@@ -4,6 +4,7 @@
 #include <Common/HashTable/PackedHashMap.h>
 #include <Common/HashTable/HashMap.h>
 #include <Common/HashTable/HashSet.h>
+#include <Core/Types_fwd.h>
 #include <sparsehash/sparse_hash_map>
 #include <sparsehash/sparse_hash_set>
 #include <type_traits>
@@ -11,6 +12,33 @@
 namespace DB
 {
 
+/// Return true if the type is POD [1] for the purpose of layout (this is not
+/// the same as STL traits has).
+///
+///   [1]: https://stackoverflow.com/questions/4178175/what-are-aggregates-and-pods-and-how-why-are-they-special/4178176#4178176
+///
+/// The behaviour had been change in clang-16, see this for more details:
+/// - https://github.com/llvm/llvm-project/commit/a8b0c6fa28acced71db33e80bd0b51d00422035b
+/// - https://github.com/llvm/llvm-project/commit/277123376ce08c98b07c154bf83e4092a5d4d3c6
+/// - https://github.com/llvm/llvm-project/issues/62422
+/// - https://github.com/llvm/llvm-project/issues/62353
+/// - https://github.com/llvm/llvm-project/issues/62358
+template <typename V>
+constexpr bool isPodLayout()
+{
+    if constexpr (std::is_same_v<V, UUID>)
+        return false;
+    if constexpr (std::is_same_v<V, DateTime64>)
+        return false;
+    if constexpr (std::is_same_v<V, Decimal32> || std::is_same_v<V, Decimal64> || std::is_same_v<V, Decimal128> || std::is_same_v<V, Decimal256>)
+        return false;
+    if constexpr (std::is_same_v<V, StringRef>)
+        return false;
+    if constexpr (std::is_same_v<V, IPv6> || std::is_same_v<V, IPv4>)
+        return false;
+    return true;
+}
+
 /// HashMap with packed structure is better than google::sparse_hash_map if the
 /// <K, V> pair is small, for the sizeof(std::pair<K, V>) == 16, RSS for hash
 /// table with 1e9 elements will be:
@@ -37,7 +65,13 @@ namespace DB
 template <typename K, typename V>
 constexpr bool useSparseHashForHashedDictionary()
 {
-    return sizeof(PackedPairNoInit<K, V>) > 16;
+    if constexpr (!isPodLayout<K>())
+        return true;
+    if constexpr (!isPodLayout<V>())
+        return true;
+    /// NOTE: One should not use PackedPairNoInit<K, V> here since this will
+    /// create instantion of this type, and it could be illformed.
+    return sizeof(V) > 8;
 }
 
 /// Grower with custom fill limit/load factor (instead of default 50%).
diff --git a/src/Interpreters/examples/hash_map.cpp b/src/Interpreters/examples/hash_map.cpp
index c0016bf8cd8..f9fc36454f6 100644
--- a/src/Interpreters/examples/hash_map.cpp
+++ b/src/Interpreters/examples/hash_map.cpp
@@ -279,35 +279,5 @@ int main(int argc, char ** argv)
             << std::endl;
     }
 
-    if (argc < 3 || std::stol(argv[2]) == 7)
-    {
-        Stopwatch watch;
-
-        PackedHashMap<Key, Value> map;
-        PackedHashMap<Key, Value>::LookupResult it;
-        bool inserted;
-
-        for (size_t i = 0; i < n; ++i)
-        {
-            map.emplace(data[i], it, inserted);
-            if (inserted)
-            {
-                new (&it->getMapped()) Value;
-                std::swap(it->getMapped(), value);
-                INIT
-            }
-        }
-
-        watch.stop();
-        std::cerr << std::fixed << std::setprecision(2)
-            << "PackedHashMap. Size: " << map.size()
-            << ", elapsed: " << watch.elapsedSeconds()
-            << " (" << n / watch.elapsedSeconds() << " elem/sec.)"
-#ifdef DBMS_HASH_MAP_COUNT_COLLISIONS
-            << ", collisions: " << map.getCollisions()
-#endif
-            << std::endl;
-    }
-
     return 0;
 }

From 7eba6def94e23d8049197d431daa15bfee4f85e4 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 19:35:02 +0200
Subject: [PATCH 631/689] Add a comment for HashTableGrowerWithPrecalculation
 about load factor

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Common/HashTable/HashTable.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h
index a48cb8d49f2..2ff789caad2 100644
--- a/src/Common/HashTable/HashTable.h
+++ b/src/Common/HashTable/HashTable.h
@@ -270,6 +270,7 @@ struct HashTableGrower
 /** Determines the size of the hash table, and when and how much it should be resized.
   * This structure is aligned to cache line boundary and also occupies it all.
   * Precalculates some values to speed up lookups and insertion into the HashTable (and thus has bigger memory footprint than HashTableGrower).
+  * This grower assume 0.5 load factor
   */
 template <size_t initial_size_degree = 8>
 class alignas(64) HashTableGrowerWithPrecalculation

From 1ab130132cba6af42521ad35f3aaf97110e768be Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 19:37:59 +0200
Subject: [PATCH 632/689] Add more comments into
 HashedDictionaryCollectionType.h

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Dictionaries/HashedDictionaryCollectionType.h | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/Dictionaries/HashedDictionaryCollectionType.h b/src/Dictionaries/HashedDictionaryCollectionType.h
index 23a08601fff..cdf29a82b66 100644
--- a/src/Dictionaries/HashedDictionaryCollectionType.h
+++ b/src/Dictionaries/HashedDictionaryCollectionType.h
@@ -160,6 +160,17 @@ public:
 };
 static_assert(sizeof(HashTableGrowerWithMaxLoadFactor) == 64);
 
+/// Above goes various specialisations for the hash table that will be used for
+/// HASHED/SPARSE_HASHED dictionary, it could use one of the following depends
+/// on the layout of the dictionary and types of key/value (for more info see
+/// comments in this file):
+/// - HashMap
+/// - HashSet
+/// - HashMapWithSavedHash
+/// - HashSetWithSavedHash
+/// - PackedHashMap
+/// - google::sparse_hash_map
+
 ///
 /// Map (dictionary with attributes)
 ///

From 6f351851adf5c6bcd269e473339e0f99dd760f0a Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 19:41:47 +0200
Subject: [PATCH 633/689] Rename grower to
 HashTableGrowerWithPrecalculationAndMaxLoadFactor

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Dictionaries/HashedDictionary.cpp         |  2 +-
 .../HashedDictionaryCollectionType.h          | 24 +++++++++----------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp
index fa9a846195d..3903317cff2 100644
--- a/src/Dictionaries/HashedDictionary.cpp
+++ b/src/Dictionaries/HashedDictionary.cpp
@@ -630,7 +630,7 @@ void HashedDictionary<dictionary_key_type, sparse, sharded>::createAttributes()
     const auto size = dict_struct.attributes.size();
     attributes.reserve(size);
 
-    HashTableGrowerWithMaxLoadFactor grower(configuration.max_load_factor);
+    HashTableGrowerWithPrecalculationAndMaxLoadFactor grower(configuration.max_load_factor);
 
     for (const auto & dictionary_attribute : dict_struct.attributes)
     {
diff --git a/src/Dictionaries/HashedDictionaryCollectionType.h b/src/Dictionaries/HashedDictionaryCollectionType.h
index cdf29a82b66..a631813bdbc 100644
--- a/src/Dictionaries/HashedDictionaryCollectionType.h
+++ b/src/Dictionaries/HashedDictionaryCollectionType.h
@@ -85,7 +85,7 @@ constexpr bool useSparseHashForHashedDictionary()
 /// google::sparse_hash_map.
 ///
 /// Based on HashTableGrowerWithPrecalculation
-class alignas(64) HashTableGrowerWithMaxLoadFactor
+class alignas(64) HashTableGrowerWithPrecalculationAndMaxLoadFactor
 {
     static constexpr size_t initial_size_degree = 8;
     UInt8 size_degree = initial_size_degree;
@@ -103,8 +103,8 @@ public:
     /// If collision resolution chains are contiguous, we can implement erase operation by moving the elements.
     static constexpr auto performs_linear_probing_with_single_step = true;
 
-    HashTableGrowerWithMaxLoadFactor() = default;
-    explicit HashTableGrowerWithMaxLoadFactor(float max_load_factor_)
+    HashTableGrowerWithPrecalculationAndMaxLoadFactor() = default;
+    explicit HashTableGrowerWithPrecalculationAndMaxLoadFactor(float max_load_factor_)
         : max_load_factor(max_load_factor_)
     {
         increaseSizeDegree(0);
@@ -158,7 +158,7 @@ public:
         increaseSizeDegree(0);
     }
 };
-static_assert(sizeof(HashTableGrowerWithMaxLoadFactor) == 64);
+static_assert(sizeof(HashTableGrowerWithPrecalculationAndMaxLoadFactor) == 64);
 
 /// Above goes various specialisations for the hash table that will be used for
 /// HASHED/SPARSE_HASHED dictionary, it could use one of the following depends
@@ -185,8 +185,8 @@ struct HashedDictionaryMapType<dictionary_key_type, /* sparse= */ false, Key, Va
 {
     using Type = std::conditional_t<
         dictionary_key_type == DictionaryKeyType::Simple,
-        HashMap<UInt64, Value, DefaultHash<UInt64>, HashTableGrowerWithMaxLoadFactor>,
-        HashMapWithSavedHash<StringRef, Value, DefaultHash<StringRef>, HashTableGrowerWithMaxLoadFactor>>;
+        HashMap<UInt64, Value, DefaultHash<UInt64>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>,
+        HashMapWithSavedHash<StringRef, Value, DefaultHash<StringRef>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>>;
 };
 
 /// Implementations for SPARSE_HASHED layout.
@@ -220,8 +220,8 @@ struct HashedDictionarySparseMapType<dictionary_key_type, Key, Value, /* use_spa
 {
     using Type = std::conditional_t<
         dictionary_key_type == DictionaryKeyType::Simple,
-        PackedHashMap<UInt64, Value, DefaultHash<UInt64>, HashTableGrowerWithMaxLoadFactor>,
-        PackedHashMap<StringRef, Value, DefaultHash<StringRef>, HashTableGrowerWithMaxLoadFactor>>;
+        PackedHashMap<UInt64, Value, DefaultHash<UInt64>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>,
+        PackedHashMap<StringRef, Value, DefaultHash<StringRef>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>>;
 };
 template <DictionaryKeyType dictionary_key_type, typename Key, typename Value>
 struct HashedDictionaryMapType<dictionary_key_type, /* sparse= */ true, Key, Value>
@@ -244,8 +244,8 @@ struct HashedDictionarySetType<dictionary_key_type, /* sparse= */ false, Key>
 {
     using Type = std::conditional_t<
         dictionary_key_type == DictionaryKeyType::Simple,
-        HashSet<UInt64, DefaultHash<UInt64>, HashTableGrowerWithMaxLoadFactor>,
-        HashSetWithSavedHash<StringRef, DefaultHash<StringRef>, HashTableGrowerWithMaxLoadFactor>>;
+        HashSet<UInt64, DefaultHash<UInt64>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>,
+        HashSetWithSavedHash<StringRef, DefaultHash<StringRef>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>>;
 };
 
 /// Implementation for SPARSE_HASHED.
@@ -258,8 +258,8 @@ struct HashedDictionarySetType<dictionary_key_type, /* sparse= */ true, Key>
 {
     using Type = std::conditional_t<
         dictionary_key_type == DictionaryKeyType::Simple,
-        HashSet<UInt64, DefaultHash<UInt64>, HashTableGrowerWithMaxLoadFactor>,
-        HashSet<StringRef, DefaultHash<StringRef>, HashTableGrowerWithMaxLoadFactor>>;
+        HashSet<UInt64, DefaultHash<UInt64>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>,
+        HashSet<StringRef, DefaultHash<StringRef>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>>;
 };
 
 }

From 42eac6bfbc6ef80866a685b98c3ae89cb9e9e450 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 19:43:11 +0200
Subject: [PATCH 634/689] Wrap implementation helpers into HashedDictionaryImpl
 namespace

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Dictionaries/HashedDictionary.cpp               | 2 ++
 src/Dictionaries/HashedDictionary.h                 | 4 ++--
 src/Dictionaries/HashedDictionaryCollectionTraits.h | 5 +++++
 src/Dictionaries/HashedDictionaryCollectionType.h   | 5 +++++
 4 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp
index 3903317cff2..1ddcfae99b4 100644
--- a/src/Dictionaries/HashedDictionary.cpp
+++ b/src/Dictionaries/HashedDictionary.cpp
@@ -32,6 +32,8 @@ namespace CurrentMetrics
 namespace DB
 {
 
+using namespace HashedDictionaryImpl;
+
 namespace ErrorCodes
 {
     extern const int BAD_ARGUMENTS;
diff --git a/src/Dictionaries/HashedDictionary.h b/src/Dictionaries/HashedDictionary.h
index bff22706435..30eecb9ab09 100644
--- a/src/Dictionaries/HashedDictionary.h
+++ b/src/Dictionaries/HashedDictionary.h
@@ -134,7 +134,7 @@ public:
 
 private:
     template <typename Value>
-    using CollectionsHolder = std::vector<typename HashedDictionaryMapType<dictionary_key_type, sparse, KeyType, Value>::Type>;
+    using CollectionsHolder = std::vector<typename HashedDictionaryImpl::HashedDictionaryMapType<dictionary_key_type, sparse, KeyType, Value>::Type>;
 
     using NullableSet = HashSet<KeyType, DefaultHash<KeyType>>;
     using NullableSets = std::vector<NullableSet>;
@@ -232,7 +232,7 @@ private:
 
     BlockPtr update_field_loaded_block;
     std::vector<std::unique_ptr<Arena>> string_arenas;
-    std::vector<typename HashedDictionarySetType<dictionary_key_type, sparse, KeyType>::Type> no_attributes_containers;
+    std::vector<typename HashedDictionaryImpl::HashedDictionarySetType<dictionary_key_type, sparse, KeyType>::Type> no_attributes_containers;
     DictionaryHierarchicalParentToChildIndexPtr hierarchical_index;
 };
 
diff --git a/src/Dictionaries/HashedDictionaryCollectionTraits.h b/src/Dictionaries/HashedDictionaryCollectionTraits.h
index 20fc882a2fe..4989664890a 100644
--- a/src/Dictionaries/HashedDictionaryCollectionTraits.h
+++ b/src/Dictionaries/HashedDictionaryCollectionTraits.h
@@ -10,6 +10,9 @@
 namespace DB
 {
 
+namespace HashedDictionaryImpl
+{
+
 /// sparse_hash_map/sparse_hash_set
 template <typename C>
 concept IsGoogleSparseHashTable = std::is_same_v<C, google::sparse_hash_map<
@@ -100,3 +103,5 @@ template <typename C> void clearContainer(C & c) requires (IsBuiltinHashTable<C>
 // NOLINTEND(*)
 
 }
+
+}
diff --git a/src/Dictionaries/HashedDictionaryCollectionType.h b/src/Dictionaries/HashedDictionaryCollectionType.h
index a631813bdbc..d1772b99c03 100644
--- a/src/Dictionaries/HashedDictionaryCollectionType.h
+++ b/src/Dictionaries/HashedDictionaryCollectionType.h
@@ -12,6 +12,9 @@
 namespace DB
 {
 
+namespace HashedDictionaryImpl
+{
+
 /// Return true if the type is POD [1] for the purpose of layout (this is not
 /// the same as STL traits has).
 ///
@@ -263,3 +266,5 @@ struct HashedDictionarySetType<dictionary_key_type, /* sparse= */ true, Key>
 };
 
 }
+
+}

From 634f168a74b8451236407e0e861078bbd72723d7 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 19:54:20 +0200
Subject: [PATCH 635/689] Introduce max_size_degree for
 HashTableGrower{,WithPrecalculation}

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Common/HashTable/HashTable.h | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h
index 2ff789caad2..d4577d7814e 100644
--- a/src/Common/HashTable/HashTable.h
+++ b/src/Common/HashTable/HashTable.h
@@ -230,6 +230,8 @@ struct HashTableGrower
     /// If collision resolution chains are contiguous, we can implement erase operation by moving the elements.
     static constexpr auto performs_linear_probing_with_single_step = true;
 
+    static constexpr size_t max_size_degree = 23;
+
     /// The size of the hash table in the cells.
     size_t bufSize() const               { return 1ULL << size_degree; }
 
@@ -248,7 +250,7 @@ struct HashTableGrower
     /// Increase the size of the hash table.
     void increaseSize()
     {
-        size_degree += size_degree >= 23 ? 1 : 2;
+        size_degree += size_degree >= max_size_degree ? 1 : 2;
     }
 
     /// Set the buffer size by the number of elements in the hash table. Used when deserializing a hash table.
@@ -280,6 +282,7 @@ class alignas(64) HashTableGrowerWithPrecalculation
     UInt8 size_degree = initial_size_degree;
     size_t precalculated_mask = (1ULL << initial_size_degree) - 1;
     size_t precalculated_max_fill = 1ULL << (initial_size_degree - 1);
+    static constexpr size_t max_size_degree = 23;
 
 public:
     UInt8 sizeDegree() const { return size_degree; }
@@ -309,7 +312,7 @@ public:
     bool overflow(size_t elems) const { return elems > precalculated_max_fill; }
 
     /// Increase the size of the hash table.
-    void increaseSize() { increaseSizeDegree(size_degree >= 23 ? 1 : 2); }
+    void increaseSize() { increaseSizeDegree(size_degree >= max_size_degree ? 1 : 2); }
 
     /// Set the buffer size by the number of elements in the hash table. Used when deserializing a hash table.
     void set(size_t num_elems)

From 01bf041ccad7921b39013798a4b9261b1c37919e Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 19:57:43 +0200
Subject: [PATCH 636/689] Rewrite HashTableGrower{,WithPrecalculation}::set w/o
 ternary operators

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Common/HashTable/HashTable.h | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h
index d4577d7814e..ca3e88c93a2 100644
--- a/src/Common/HashTable/HashTable.h
+++ b/src/Common/HashTable/HashTable.h
@@ -256,11 +256,12 @@ struct HashTableGrower
     /// Set the buffer size by the number of elements in the hash table. Used when deserializing a hash table.
     void set(size_t num_elems)
     {
-        size_degree = num_elems <= 1
-             ? initial_size_degree
-             : ((initial_size_degree > static_cast<size_t>(log2(num_elems - 1)) + 2)
-                 ? initial_size_degree
-                 : (static_cast<size_t>(log2(num_elems - 1)) + 2));
+        if (num_elems <= 1)
+            size_degree = initial_size_degree;
+        else if (initial_size_degree > static_cast<size_t>(log2(num_elems - 1)) + 2)
+            size_degree = initial_size_degree;
+        else
+            size_degree = static_cast<size_t>(log2(num_elems - 1)) + 2;
     }
 
     void setBufSize(size_t buf_size_)
@@ -317,11 +318,12 @@ public:
     /// Set the buffer size by the number of elements in the hash table. Used when deserializing a hash table.
     void set(size_t num_elems)
     {
-        size_degree = num_elems <= 1
-             ? initial_size_degree
-             : ((initial_size_degree > static_cast<size_t>(log2(num_elems - 1)) + 2)
-                 ? initial_size_degree
-                 : (static_cast<size_t>(log2(num_elems - 1)) + 2));
+        if (num_elems <= 1)
+            size_degree = initial_size_degree;
+        else if (initial_size_degree > static_cast<size_t>(log2(num_elems - 1)) + 2)
+            size_degree = initial_size_degree;
+        else
+            size_degree = static_cast<size_t>(log2(num_elems - 1)) + 2;
         increaseSizeDegree(0);
     }
 

From c9cde110cdaebb27ddfaa8792988ac9b5e5162f2 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 19:59:43 +0200
Subject: [PATCH 637/689] Add initial degree as parameter for
 HashTableGrowerWithPrecalculationAndMaxLoadFactor

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../HashedDictionaryCollectionType.h          | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/Dictionaries/HashedDictionaryCollectionType.h b/src/Dictionaries/HashedDictionaryCollectionType.h
index d1772b99c03..745c1716964 100644
--- a/src/Dictionaries/HashedDictionaryCollectionType.h
+++ b/src/Dictionaries/HashedDictionaryCollectionType.h
@@ -88,9 +88,9 @@ constexpr bool useSparseHashForHashedDictionary()
 /// google::sparse_hash_map.
 ///
 /// Based on HashTableGrowerWithPrecalculation
+template <size_t initial_size_degree = 8>
 class alignas(64) HashTableGrowerWithPrecalculationAndMaxLoadFactor
 {
-    static constexpr size_t initial_size_degree = 8;
     UInt8 size_degree = initial_size_degree;
     size_t precalculated_mask = (1ULL << initial_size_degree) - 1;
     size_t precalculated_max_fill = 1ULL << (initial_size_degree - 1);
@@ -161,7 +161,7 @@ public:
         increaseSizeDegree(0);
     }
 };
-static_assert(sizeof(HashTableGrowerWithPrecalculationAndMaxLoadFactor) == 64);
+static_assert(sizeof(HashTableGrowerWithPrecalculationAndMaxLoadFactor<>) == 64);
 
 /// Above goes various specialisations for the hash table that will be used for
 /// HASHED/SPARSE_HASHED dictionary, it could use one of the following depends
@@ -188,8 +188,8 @@ struct HashedDictionaryMapType<dictionary_key_type, /* sparse= */ false, Key, Va
 {
     using Type = std::conditional_t<
         dictionary_key_type == DictionaryKeyType::Simple,
-        HashMap<UInt64, Value, DefaultHash<UInt64>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>,
-        HashMapWithSavedHash<StringRef, Value, DefaultHash<StringRef>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>>;
+        HashMap<UInt64, Value, DefaultHash<UInt64>, HashTableGrowerWithPrecalculationAndMaxLoadFactor<>>,
+        HashMapWithSavedHash<StringRef, Value, DefaultHash<StringRef>, HashTableGrowerWithPrecalculationAndMaxLoadFactor<>>>;
 };
 
 /// Implementations for SPARSE_HASHED layout.
@@ -223,8 +223,8 @@ struct HashedDictionarySparseMapType<dictionary_key_type, Key, Value, /* use_spa
 {
     using Type = std::conditional_t<
         dictionary_key_type == DictionaryKeyType::Simple,
-        PackedHashMap<UInt64, Value, DefaultHash<UInt64>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>,
-        PackedHashMap<StringRef, Value, DefaultHash<StringRef>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>>;
+        PackedHashMap<UInt64, Value, DefaultHash<UInt64>, HashTableGrowerWithPrecalculationAndMaxLoadFactor<>>,
+        PackedHashMap<StringRef, Value, DefaultHash<StringRef>, HashTableGrowerWithPrecalculationAndMaxLoadFactor<>>>;
 };
 template <DictionaryKeyType dictionary_key_type, typename Key, typename Value>
 struct HashedDictionaryMapType<dictionary_key_type, /* sparse= */ true, Key, Value>
@@ -247,8 +247,8 @@ struct HashedDictionarySetType<dictionary_key_type, /* sparse= */ false, Key>
 {
     using Type = std::conditional_t<
         dictionary_key_type == DictionaryKeyType::Simple,
-        HashSet<UInt64, DefaultHash<UInt64>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>,
-        HashSetWithSavedHash<StringRef, DefaultHash<StringRef>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>>;
+        HashSet<UInt64, DefaultHash<UInt64>, HashTableGrowerWithPrecalculationAndMaxLoadFactor<>>,
+        HashSetWithSavedHash<StringRef, DefaultHash<StringRef>, HashTableGrowerWithPrecalculationAndMaxLoadFactor<>>>;
 };
 
 /// Implementation for SPARSE_HASHED.
@@ -261,8 +261,8 @@ struct HashedDictionarySetType<dictionary_key_type, /* sparse= */ true, Key>
 {
     using Type = std::conditional_t<
         dictionary_key_type == DictionaryKeyType::Simple,
-        HashSet<UInt64, DefaultHash<UInt64>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>,
-        HashSet<StringRef, DefaultHash<StringRef>, HashTableGrowerWithPrecalculationAndMaxLoadFactor>>;
+        HashSet<UInt64, DefaultHash<UInt64>, HashTableGrowerWithPrecalculationAndMaxLoadFactor<>>,
+        HashSet<StringRef, DefaultHash<StringRef>, HashTableGrowerWithPrecalculationAndMaxLoadFactor<>>>;
 };
 
 }

From f8e7d2cb1f186e60d4537f0069969ae440ccb217 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 20:02:13 +0200
Subject: [PATCH 638/689] Remove part of the
 HashTableGrowerWithPrecalculationAndMaxLoadFactor comment

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Dictionaries/HashedDictionaryCollectionType.h | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/Dictionaries/HashedDictionaryCollectionType.h b/src/Dictionaries/HashedDictionaryCollectionType.h
index 745c1716964..0746d42bd1f 100644
--- a/src/Dictionaries/HashedDictionaryCollectionType.h
+++ b/src/Dictionaries/HashedDictionaryCollectionType.h
@@ -79,14 +79,6 @@ constexpr bool useSparseHashForHashedDictionary()
 
 /// Grower with custom fill limit/load factor (instead of default 50%).
 ///
-/// It turns out that HashMap can outperform google::sparse_hash_map in case of
-/// the structure size of not big, in terms of speed *and* memory. Even 99% of
-/// max load factor was faster then google::sparse_hash_map in my simple tests
-/// (1e9 UInt64 keys with UInt16 values, randomly distributed).
-///
-/// And not to mention very high allocator memory fragmentation in
-/// google::sparse_hash_map.
-///
 /// Based on HashTableGrowerWithPrecalculation
 template <size_t initial_size_degree = 8>
 class alignas(64) HashTableGrowerWithPrecalculationAndMaxLoadFactor

From 2b240d3721f66f1e3178be602837194e06d1bc44 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 20:10:06 +0200
Subject: [PATCH 639/689] Improve documentation for
 HASHED/SPARSE_HASHED/COMPLEX_KEY_HASHED/COMPLEX_KEY_SPARSE_HASHED

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docs/en/sql-reference/dictionaries/index.md | 27 ++++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index dfda1ff9c04..712a17308ba 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -267,14 +267,16 @@ or
 LAYOUT(HASHED())
 ```
 
-If `shards` greater then 1 (default is `1`) the dictionary will load data in parallel, useful if you have huge amount of elements in one dictionary.
-
 Configuration example:
 
 ``` xml
 <layout>
   <hashed>
+    <!-- If shards greater then 1 (default is `1`) the dictionary will load
+         data in parallel, useful if you have huge amount of elements in one
+         dictionary. -->
     <shards>10</shards>
+
     <!-- Size of the backlog for blocks in parallel queue.
 
          Since the bottleneck in parallel loading is rehash, and so to avoid
@@ -284,6 +286,7 @@ Configuration example:
          10000 is good balance between memory and speed.
          Even for 10e10 elements and can handle all the load without starvation. -->
     <shard_load_queue_backlog>10000</shard_load_queue_backlog>
+
     <!-- Maximum load factor of the hash table, with greater values, the memory
          is utilized more efficiently (less memory is wasted) but read/performance
          may deteriorate.
@@ -298,7 +301,7 @@ Configuration example:
 or
 
 ``` sql
-LAYOUT(HASHED(SHARDS 10 [SHARD_LOAD_QUEUE_BACKLOG 10000]))
+LAYOUT(HASHED([SHARDS 1] [SHARD_LOAD_QUEUE_BACKLOG 10000] [MAX_LOAD_FACTOR 0.5]))
 ```
 
 ### sparse_hashed
@@ -311,14 +314,18 @@ Configuration example:
 
 ``` xml
 <layout>
-  <sparse_hashed />
+  <sparse_hashed>
+    <!-- <shards>1</shards> -->
+    <!-- <shard_load_queue_backlog>10000</shard_load_queue_backlog> -->
+    <!-- <max_load_factor>0.5</max_load_factor> -->
+  </sparse_hashed>
 </layout>
 ```
 
 or
 
 ``` sql
-LAYOUT(SPARSE_HASHED())
+LAYOUT(SPARSE_HASHED([SHARDS 1] [SHARD_LOAD_QUEUE_BACKLOG 10000] [MAX_LOAD_FACTOR 0.5]))
 ```
 
 It is also possible to use `shards` for this type of dictionary, and again it is more important for `sparse_hashed` then for `hashed`, since `sparse_hashed` is slower.
@@ -332,7 +339,7 @@ Configuration example:
 ``` xml
 <layout>
   <complex_key_hashed>
-    <shards>1</shards>
+    <!-- <shards>1</shards> -->
     <!-- <shard_load_queue_backlog>10000</shard_load_queue_backlog> -->
     <!-- <max_load_factor>0.5</max_load_factor> -->
   </complex_key_hashed>
@@ -342,7 +349,7 @@ Configuration example:
 or
 
 ``` sql
-LAYOUT(COMPLEX_KEY_HASHED([SHARDS 1] [SHARD_LOAD_QUEUE_BACKLOG 10000]))
+LAYOUT(COMPLEX_KEY_HASHED([SHARDS 1] [SHARD_LOAD_QUEUE_BACKLOG 10000] [MAX_LOAD_FACTOR 0.5]))
 ```
 
 ### complex_key_sparse_hashed
@@ -354,7 +361,9 @@ Configuration example:
 ``` xml
 <layout>
   <complex_key_sparse_hashed>
-    <shards>1</shards>
+    <!-- <shards>1</shards> -->
+    <!-- <shard_load_queue_backlog>10000</shard_load_queue_backlog> -->
+    <!-- <max_load_factor>0.5</max_load_factor> -->
   </complex_key_sparse_hashed>
 </layout>
 ```
@@ -362,7 +371,7 @@ Configuration example:
 or
 
 ``` sql
-LAYOUT(COMPLEX_KEY_SPARSE_HASHED([SHARDS 1] [SHARD_LOAD_QUEUE_BACKLOG 10000]))
+LAYOUT(COMPLEX_KEY_SPARSE_HASHED([SHARDS 1] [SHARD_LOAD_QUEUE_BACKLOG 10000] [MAX_LOAD_FACTOR 0.5]))
 ```
 
 ### hashed_array

From e1e2a83a9e78b5e56476f38478895b8826e2b3dd Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 15 May 2023 20:28:28 +0200
Subject: [PATCH 640/689] Print type of the structure that will be used for
 HASHED/SPARSE_HASHED

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Dictionaries/HashedDictionary.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp
index 1ddcfae99b4..83d051df5e9 100644
--- a/src/Dictionaries/HashedDictionary.cpp
+++ b/src/Dictionaries/HashedDictionary.cpp
@@ -660,6 +660,11 @@ void HashedDictionary<dictionary_key_type, sparse, sharded>::createAttributes()
                     container.max_load_factor(configuration.max_load_factor);
                 attributes.emplace_back(std::move(attribute));
             }
+
+            if constexpr (IsBuiltinHashTable<typename CollectionsHolder<ValueType>::value_type>)
+                LOG_TRACE(log, "Using builtin hash table for {} attribute", dictionary_attribute.name);
+            else
+                LOG_TRACE(log, "Using sparsehash for {} attribute", dictionary_attribute.name);
         };
 
         callOnDictionaryAttributeType(dictionary_attribute.underlying_type, type_call);

From e37e8f83bb28dc752c0264cee52dcbb7c4f24352 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 16 May 2023 15:57:26 +0200
Subject: [PATCH 641/689] Fix flakiness of test_distributed_load_balancing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I saw the following in the logs for the failed test:

    2023.05.16 07:12:12.894051 [ 262 ] {74575ac0-b296-4fdc-bc8e-3476a305e6ea} <Warning> ConnectionPoolWithFailover: Connection failed at try №1, reason: Timeout exceeded while reading from socket (socket (172.16.3.2:9000), receive timeout 2000 ms)

And I think that the culprit is the
test_distributed_replica_max_ignored_errors for which it is normal,
however not for others, and this should not affect other tests.

So fix this by calling SYSTEM RELOAD CONFIG, which should reset error
count.

CI: https://s3.amazonaws.com/clickhouse-test-reports/49380/5abc1a1c68ee204c9024493be1d19835cf5630f7/integration_tests__release__[3_4].html
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../test_distributed_load_balancing/test.py   | 26 ++++++++-----------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/tests/integration/test_distributed_load_balancing/test.py b/tests/integration/test_distributed_load_balancing/test.py
index 1dba6a30bc4..271828f433e 100644
--- a/tests/integration/test_distributed_load_balancing/test.py
+++ b/tests/integration/test_distributed_load_balancing/test.py
@@ -29,23 +29,19 @@ nodes = len(cluster.instances)
 queries = nodes * 10
 
 
+# SYSTEM RELOAD CONFIG will reset some attributes of the nodes in cluster
+# - error_count
+# - last_used (round_robing)
+#
+# This is required to avoid interference results of one test to another
+@pytest.fixture(scope="function", autouse=True)
+def test_setup():
+    for n in list(cluster.instances.values()):
+        n.query("SYSTEM RELOAD CONFIG")
+
+
 def bootstrap():
     for n in list(cluster.instances.values()):
-        # At startup, server loads configuration files.
-        #
-        # However ConfigReloader does not know about already loaded files
-        # (files is empty()), hence it will always reload the configuration
-        # just after server starts (+ 2 seconds, reload timeout).
-        #
-        # And on configuration reload the clusters will be re-created, so some
-        # internal stuff will be reset:
-        # - error_count
-        # - last_used (round_robing)
-        #
-        # And if the reload will happen during round_robin test it will start
-        # querying from the beginning, so let's issue config reload just after
-        # start to avoid reload in the middle of the test execution.
-        n.query("SYSTEM RELOAD CONFIG")
         n.query("DROP TABLE IF EXISTS data")
         n.query("DROP TABLE IF EXISTS dist")
         n.query("CREATE TABLE data (key Int) Engine=Memory()")

From dc353faf44c466cd89391a50076979d6f933e1d1 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 16 May 2023 15:59:22 +0200
Subject: [PATCH 642/689] Simplify obtaining query shard in
 test_distributed_load_balancing

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../test_distributed_load_balancing/test.py   | 21 +++++++------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/tests/integration/test_distributed_load_balancing/test.py b/tests/integration/test_distributed_load_balancing/test.py
index 271828f433e..e879f09ccc1 100644
--- a/tests/integration/test_distributed_load_balancing/test.py
+++ b/tests/integration/test_distributed_load_balancing/test.py
@@ -109,19 +109,14 @@ def get_node(query_node, table="dist", *args, **kwargs):
 
     rows = query_node.query(
         """
-    SELECT c.host_name
-    FROM (
-        SELECT _shard_num
-        FROM cluster(shards_cluster, system.query_log)
-        WHERE
-            initial_query_id = '{query_id}' AND
-            is_initial_query = 0 AND
-            type = 'QueryFinish'
-        ORDER BY event_date DESC, event_time DESC
-        LIMIT 1
-    ) a
-    JOIN system.clusters c
-    ON a._shard_num = c.shard_num WHERE cluster = 'shards_cluster'
+    SELECT hostName()
+    FROM cluster(shards_cluster, system.query_log)
+    WHERE
+        initial_query_id = '{query_id}' AND
+        is_initial_query = 0 AND
+        type = 'QueryFinish'
+    ORDER BY event_date DESC, event_time DESC
+    LIMIT 1
     """.format(
             query_id=query_id
         )

From f47375d16c1c89c2ffea087617f1b4dd23734f00 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 19 May 2023 10:28:13 +0200
Subject: [PATCH 643/689] Support Tableau

---
 programs/server/config.xml                                | 8 ++++++--
 src/IO/Resource/DynamicResourceManager.cpp                | 4 ++--
 .../queries/0_stateless/02737_sql_auto_is_null.reference  | 1 +
 tests/queries/0_stateless/02737_sql_auto_is_null.sql      | 2 ++
 4 files changed, 11 insertions(+), 4 deletions(-)
 create mode 100644 tests/queries/0_stateless/02737_sql_auto_is_null.reference
 create mode 100644 tests/queries/0_stateless/02737_sql_auto_is_null.sql

diff --git a/programs/server/config.xml b/programs/server/config.xml
index 82dd697084c..c6ae47b3050 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -719,8 +719,12 @@
     <!-- Default profile of settings. -->
     <default_profile>default</default_profile>
 
-    <!-- Comma-separated list of prefixes for user-defined settings. -->
-    <custom_settings_prefixes></custom_settings_prefixes>
+    <!-- Comma-separated list of prefixes for user-defined settings.
+         The server will allow to set these settings, and retrieve them with the getSetting function.
+         They are also logged in the query_log, similarly to other settings, but have no special effect.
+         The "SQL_" prefix is introduced for compatibility with MySQL - these settings are being set be Tableau.
+    -->
+    <custom_settings_prefixes>SQL_</custom_settings_prefixes>
 
     <!-- System profile of settings. This settings are used by internal processes (Distributed DDL worker and so on). -->
     <!-- <system_profile>default</system_profile> -->
diff --git a/src/IO/Resource/DynamicResourceManager.cpp b/src/IO/Resource/DynamicResourceManager.cpp
index 49e12984e33..df0de6575f4 100644
--- a/src/IO/Resource/DynamicResourceManager.cpp
+++ b/src/IO/Resource/DynamicResourceManager.cpp
@@ -98,9 +98,9 @@ DynamicResourceManager::State::Resource::~Resource()
     if (attached_to != nullptr)
     {
         ISchedulerNode * root = nodes.find("/")->second.ptr.get();
-        attached_to->event_queue->enqueue([scheduler = attached_to, root]
+        attached_to->event_queue->enqueue([my_scheduler = attached_to, root]
         {
-            scheduler->removeChild(root);
+            my_scheduler->removeChild(root);
         });
     }
 }
diff --git a/tests/queries/0_stateless/02737_sql_auto_is_null.reference b/tests/queries/0_stateless/02737_sql_auto_is_null.reference
new file mode 100644
index 00000000000..573541ac970
--- /dev/null
+++ b/tests/queries/0_stateless/02737_sql_auto_is_null.reference
@@ -0,0 +1 @@
+0
diff --git a/tests/queries/0_stateless/02737_sql_auto_is_null.sql b/tests/queries/0_stateless/02737_sql_auto_is_null.sql
new file mode 100644
index 00000000000..22f1a9524ca
--- /dev/null
+++ b/tests/queries/0_stateless/02737_sql_auto_is_null.sql
@@ -0,0 +1,2 @@
+SET SQL_AUTO_IS_NULL = 0;
+SELECT getSetting('SQL_AUTO_IS_NULL');

From 4dbe5b83294b4a2c23e844de68de6530c653504c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 19 May 2023 11:13:28 +0200
Subject: [PATCH 644/689] Support them in tests

---
 tests/config/config.d/custom_settings_prefixes.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/config/config.d/custom_settings_prefixes.xml b/tests/config/config.d/custom_settings_prefixes.xml
index ec66c1a930e..9baaa4e09f3 100644
--- a/tests/config/config.d/custom_settings_prefixes.xml
+++ b/tests/config/config.d/custom_settings_prefixes.xml
@@ -1,4 +1,4 @@
 <clickhouse>
     <!-- Comma-separated list of prefixes for user-defined settings. -->
-    <custom_settings_prefixes>custom_</custom_settings_prefixes>
+    <custom_settings_prefixes>SQL_,custom_</custom_settings_prefixes>
 </clickhouse>

From 6e468b29e894a85b954626e24fba86a2d0a8f4fb Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 19 May 2023 10:15:06 +0000
Subject: [PATCH 645/689] Check return value of ftruncate

---
 src/Coordination/Changelog.cpp | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp
index ddedae4fa0f..7275a080252 100644
--- a/src/Coordination/Changelog.cpp
+++ b/src/Coordination/Changelog.cpp
@@ -279,7 +279,16 @@ private:
         flush();
 
         if (log_file_settings.max_size != 0)
-            ftruncate(file_buffer->getFD(), initial_file_size + file_buffer->count());
+        {
+            int res = -1;
+            do
+            {
+                res = ftruncate(file_buffer->getFD(), initial_file_size + file_buffer->count());
+            } while (res < 0 && errno == EINTR);
+
+            if (res != 0)
+                LOG_WARNING(log, "Could not ftruncate file. Error: {}, errno: {}", errnoToString(), errno);
+        }
 
         if (log_file_settings.compress_logs)
             compressed_buffer.reset();

From f5506210d64d39b2bddecdc32a724f78b85d47af Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 19 May 2023 12:43:55 +0200
Subject: [PATCH 646/689] Geo types are production ready

---
 docs/en/operations/settings/settings.md           | 11 -----------
 docs/en/sql-reference/data-types/index.md         |  4 ++--
 docs/ru/operations/settings/settings.md           | 10 ----------
 docs/ru/sql-reference/data-types/geo.md           |  9 ---------
 src/Core/Settings.h                               |  5 +++--
 src/IO/Resource/DynamicResourceManager.cpp        |  4 ++--
 src/Interpreters/InterpreterCreateQuery.cpp       | 15 ---------------
 .../parseColumnsListForTableFunction.cpp          | 12 ------------
 .../parseColumnsListForTableFunction.h            |  2 --
 tests/queries/0_stateless/01291_geo_types.sql     |  2 --
 .../0_stateless/02513_validate_data_types.sql     |  5 -----
 .../queries/0_stateless/02524_fuzz_and_fuss_2.sql |  2 +-
 12 files changed, 8 insertions(+), 73 deletions(-)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index cddde2090f8..dfc433717b7 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -3213,17 +3213,6 @@ Possible values:
 
 Default value: `0`.
 
-## allow_experimental_geo_types {#allow-experimental-geo-types}
-
-Allows working with experimental [geo data types](../../sql-reference/data-types/geo.md).
-
-Possible values:
-
-- 0 — Working with geo data types is disabled.
-- 1 — Working with geo data types is enabled.
-
-Default value: `0`.
-
 ## database_atomic_wait_for_drop_and_detach_synchronously {#database_atomic_wait_for_drop_and_detach_synchronously}
 
 Adds a modifier `SYNC` to all `DROP` and `DETACH` queries.
diff --git a/docs/en/sql-reference/data-types/index.md b/docs/en/sql-reference/data-types/index.md
index 2ad8ac4bb23..88663968e50 100644
--- a/docs/en/sql-reference/data-types/index.md
+++ b/docs/en/sql-reference/data-types/index.md
@@ -29,5 +29,5 @@ ClickHouse data types include:
 - **Tuples**: A [`Tuple` of elements](./tuple.md), each having an individual type.
 - **Nullable**: [`Nullable`](./nullable.md) allows you to store a value as `NULL` when a value is "missing" (instead of the column gettings its default value for the data type)
 - **IP addresses**: use [`IPv4`](./domains/ipv4.md) and [`IPv6`](./domains/ipv6.md) to efficiently store IP addresses
-- **Geo types**: for[ geographical data](./geo.md), including `Point`, `Ring`, `Polygon` and `MultiPolygon`
-- **Special data types**: including [`Expression`](./special-data-types/expression.md), [`Set`](./special-data-types/set.md), [`Nothing`](./special-data-types/nothing.md) and [`Interval`](./special-data-types/interval.md)
\ No newline at end of file
+- **Geo types**: for [geographical data](./geo.md), including `Point`, `Ring`, `Polygon` and `MultiPolygon`
+- **Special data types**: including [`Expression`](./special-data-types/expression.md), [`Set`](./special-data-types/set.md), [`Nothing`](./special-data-types/nothing.md) and [`Interval`](./special-data-types/interval.md)
diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md
index 066e71c25a5..fa3ea582c55 100644
--- a/docs/ru/operations/settings/settings.md
+++ b/docs/ru/operations/settings/settings.md
@@ -3185,16 +3185,6 @@ SELECT * FROM test2;
 
 Значение по умолчанию: `0`.
 
-## allow_experimental_geo_types {#allow-experimental-geo-types}
-
-Разрешает использование экспериментальных типов данных для работы с [географическими структурами](../../sql-reference/data-types/geo.md).
-
-Возможные значения:
--   0 — использование типов данных для работы с географическими структурами не поддерживается.
--   1 — использование типов данных для работы с географическими структурами поддерживается.
-
-Значение по умолчанию: `0`.
-
 ## database_atomic_wait_for_drop_and_detach_synchronously {#database_atomic_wait_for_drop_and_detach_synchronously}
 
 Добавляет модификатор `SYNC` ко всем запросам `DROP` и `DETACH`.
diff --git a/docs/ru/sql-reference/data-types/geo.md b/docs/ru/sql-reference/data-types/geo.md
index 24d981195f5..52c7dd97da0 100644
--- a/docs/ru/sql-reference/data-types/geo.md
+++ b/docs/ru/sql-reference/data-types/geo.md
@@ -8,13 +8,8 @@ sidebar_label: Географические структуры
 
 ClickHouse поддерживает типы данных для отображения географических объектов — точек (местоположений), территорий и т.п.
 
-:::danger "Предупреждение"
-    Сейчас использование типов данных для работы с географическими структурами является экспериментальной возможностью. Чтобы использовать эти типы данных, включите настройку `allow_experimental_geo_types = 1`.
-:::
-
 **См. также**
 - [Хранение географических структур данных](https://ru.wikipedia.org/wiki/GeoJSON).
-- Настройка [allow_experimental_geo_types](../../operations/settings/settings.md#allow-experimental-geo-types).
 
 ## Point {#point-data-type}
 
@@ -25,7 +20,6 @@ ClickHouse поддерживает типы данных для отображ
 Запрос:
 
 ```sql
-SET allow_experimental_geo_types = 1;
 CREATE TABLE geo_point (p Point) ENGINE = Memory();
 INSERT INTO geo_point VALUES((10, 10));
 SELECT p, toTypeName(p) FROM geo_point;
@@ -47,7 +41,6 @@ SELECT p, toTypeName(p) FROM geo_point;
 Запрос:
 
 ```sql
-SET allow_experimental_geo_types = 1;
 CREATE TABLE geo_ring (r Ring) ENGINE = Memory();
 INSERT INTO geo_ring VALUES([(0, 0), (10, 0), (10, 10), (0, 10)]);
 SELECT r, toTypeName(r) FROM geo_ring;
@@ -69,7 +62,6 @@ SELECT r, toTypeName(r) FROM geo_ring;
 Запись в этой таблице описывает многоугольник с одной дырой:
 
 ```sql
-SET allow_experimental_geo_types = 1;
 CREATE TABLE geo_polygon (pg Polygon) ENGINE = Memory();
 INSERT INTO geo_polygon VALUES([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]]);
 SELECT pg, toTypeName(pg) FROM geo_polygon;
@@ -92,7 +84,6 @@ SELECT pg, toTypeName(pg) FROM geo_polygon;
 Запись в этой таблице описывает элемент, состоящий из двух многоугольников — первый без дыр, а второй с одной дырой:
 
 ```sql
-SET allow_experimental_geo_types = 1;
 CREATE TABLE geo_multipolygon (mpg MultiPolygon) ENGINE = Memory();
 INSERT INTO geo_multipolygon VALUES([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]]);
 SELECT mpg, toTypeName(mpg) FROM geo_multipolygon;
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index c65958b86b7..3e1400c8528 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -19,7 +19,7 @@ class IColumn;
 
 /** List of settings: type, name, default value, description, flags
   *
-  * This looks rather unconvenient. It is done that way to avoid repeating settings in different places.
+  * This looks rather inconvenient. It is done that way to avoid repeating settings in different places.
   * Note: as an alternative, we could implement settings to be completely dynamic in form of map: String -> Field,
   *  but we are not going to do it, because settings is used everywhere as static struct fields.
   *
@@ -541,7 +541,6 @@ class IColumn;
     M(Seconds, lock_acquire_timeout, DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC, "How long locking request should wait before failing", 0) \
     M(Bool, materialize_ttl_after_modify, true, "Apply TTL for old data, after ALTER MODIFY TTL query", 0) \
     M(String, function_implementation, "", "Choose function implementation for specific target or variant (experimental). If empty enable all of them.", 0) \
-    M(Bool, allow_experimental_geo_types, true, "Allow geo data types such as Point, Ring, Polygon, MultiPolygon", 0) \
     M(Bool, data_type_default_nullable, false, "Data types without NULL or NOT NULL will make Nullable", 0) \
     M(Bool, cast_keep_nullable, false, "CAST operator keep Nullable for result data type", 0) \
     M(Bool, cast_ipv4_ipv6_default_on_conversion_error, false, "CAST operator into IPv4, CAST operator into IPV6 type, toIPv4, toIPv6 functions will return default value instead of throwing exception on conversion error.", 0) \
@@ -768,6 +767,8 @@ class IColumn;
     MAKE_OBSOLETE(M, Bool, allow_experimental_bigint_types, true) \
     MAKE_OBSOLETE(M, Bool, allow_experimental_window_functions, true) \
     MAKE_OBSOLETE(M, Bool, allow_experimental_lightweight_delete, true) \
+    MAKE_OBSOLETE(M, Bool, allow_experimental_geo_types, true) \
+    \
     MAKE_OBSOLETE(M, Milliseconds, async_insert_stale_timeout_ms, 0) \
     MAKE_OBSOLETE(M, HandleKafkaErrorMode, handle_kafka_error_mode, HandleKafkaErrorMode::DEFAULT) \
     MAKE_OBSOLETE(M, Bool, database_replicated_ddl_output, true) \
diff --git a/src/IO/Resource/DynamicResourceManager.cpp b/src/IO/Resource/DynamicResourceManager.cpp
index 49e12984e33..df0de6575f4 100644
--- a/src/IO/Resource/DynamicResourceManager.cpp
+++ b/src/IO/Resource/DynamicResourceManager.cpp
@@ -98,9 +98,9 @@ DynamicResourceManager::State::Resource::~Resource()
     if (attached_to != nullptr)
     {
         ISchedulerNode * root = nodes.find("/")->second.ptr.get();
-        attached_to->event_queue->enqueue([scheduler = attached_to, root]
+        attached_to->event_queue->enqueue([my_scheduler = attached_to, root]
         {
-            scheduler->removeChild(root);
+            my_scheduler->removeChild(root);
         });
     }
 }
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index a34bfa5a579..3e76ad23c30 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -848,21 +848,6 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat
         }
     }
 
-    if (!create.attach && !settings.allow_experimental_geo_types)
-    {
-        for (const auto & name_and_type_pair : properties.columns.getAllPhysical())
-        {
-            const auto & type = name_and_type_pair.type->getName();
-            if (type == "MultiPolygon" || type == "Polygon" || type == "Ring" || type == "Point")
-            {
-                throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot create table with column '{}' which type is '{}' "
-                                "because experimental geo types are not allowed. "
-                                "Set setting allow_experimental_geo_types = 1 in order to allow it",
-                                name_and_type_pair.name, type);
-            }
-        }
-    }
-
     if (!create.attach && !settings.allow_experimental_object_type)
     {
         for (const auto & [name, type] : properties.columns.getAllPhysical())
diff --git a/src/Interpreters/parseColumnsListForTableFunction.cpp b/src/Interpreters/parseColumnsListForTableFunction.cpp
index e7302b6324a..05d28bc53a8 100644
--- a/src/Interpreters/parseColumnsListForTableFunction.cpp
+++ b/src/Interpreters/parseColumnsListForTableFunction.cpp
@@ -35,18 +35,6 @@ void validateDataType(const DataTypePtr & type, const DataTypeValidationSettings
         }
     }
 
-    if (!settings.allow_experimental_geo_types)
-    {
-        const auto & type_name = type->getName();
-        if (type_name == "MultiPolygon" || type_name == "Polygon" || type_name == "Ring" || type_name == "Point")
-        {
-            throw Exception(
-                ErrorCodes::ILLEGAL_COLUMN,
-                "Cannot create column with type '{}' because experimental geo types are not allowed. Set setting "
-                "allow_experimental_geo_types = 1 in order to allow it", type_name);
-        }
-    }
-
     if (!settings.allow_experimental_object_type)
     {
         if (type->hasDynamicSubcolumns())
diff --git a/src/Interpreters/parseColumnsListForTableFunction.h b/src/Interpreters/parseColumnsListForTableFunction.h
index 14119ab55da..ef1bbe5498e 100644
--- a/src/Interpreters/parseColumnsListForTableFunction.h
+++ b/src/Interpreters/parseColumnsListForTableFunction.h
@@ -16,14 +16,12 @@ struct DataTypeValidationSettings
 
     explicit DataTypeValidationSettings(const Settings & settings)
         : allow_suspicious_low_cardinality_types(settings.allow_suspicious_low_cardinality_types)
-        , allow_experimental_geo_types(settings.allow_experimental_geo_types)
         , allow_experimental_object_type(settings.allow_experimental_object_type)
         , allow_suspicious_fixed_string_types(settings.allow_suspicious_fixed_string_types)
     {
     }
 
     bool allow_suspicious_low_cardinality_types = true;
-    bool allow_experimental_geo_types = true;
     bool allow_experimental_object_type = true;
     bool allow_suspicious_fixed_string_types = true;
 };
diff --git a/tests/queries/0_stateless/01291_geo_types.sql b/tests/queries/0_stateless/01291_geo_types.sql
index 6b686ddf520..4038c545679 100644
--- a/tests/queries/0_stateless/01291_geo_types.sql
+++ b/tests/queries/0_stateless/01291_geo_types.sql
@@ -1,7 +1,5 @@
 DROP TABLE IF EXISTS geo;
 
-SET allow_experimental_geo_types = 1;
-
 CREATE TABLE geo (a Point, b Ring, c Polygon, d MultiPolygon) ENGINE=Memory();
 
 INSERT INTO geo VALUES((0, 0), [(0, 0), (10, 0), (10, 10), (0, 10)], [[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]], [[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]]);
diff --git a/tests/queries/0_stateless/02513_validate_data_types.sql b/tests/queries/0_stateless/02513_validate_data_types.sql
index 1235d00ba79..5eb91ac7879 100644
--- a/tests/queries/0_stateless/02513_validate_data_types.sql
+++ b/tests/queries/0_stateless/02513_validate_data_types.sql
@@ -5,10 +5,6 @@ select CAST('{"x" : 1}', 'JSON'); -- {serverError ILLEGAL_COLUMN}
 desc file(nonexist.json, JSONAsObject); -- {serverError ILLEGAL_COLUMN}
 desc file(nonexist.json, JSONEachRow, 'x JSON'); -- {serverError ILLEGAL_COLUMN}
 
-set allow_experimental_geo_types=0;
-select CAST([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]], 'Polygon'); -- {serverError ILLEGAL_COLUMN}
-desc file(nonexist.json, JSONEachRow, 'pg Polygon'); -- {serverError ILLEGAL_COLUMN}
-
 set allow_suspicious_low_cardinality_types=0;
 select CAST(1000000, 'LowCardinality(UInt64)'); -- {serverError SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY}
 desc file(nonexist.json, JSONEachRow, 'lc LowCardinality(UInt64)'); -- {serverError SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY}
@@ -16,4 +12,3 @@ desc file(nonexist.json, JSONEachRow, 'lc LowCardinality(UInt64)'); -- {serverEr
 set allow_suspicious_fixed_string_types=0;
 select CAST('', 'FixedString(1000)'); -- {serverError ILLEGAL_COLUMN}
 desc file(nonexist.json, JSONEachRow, 'fs FixedString(1000)'); -- {serverError ILLEGAL_COLUMN}
-
diff --git a/tests/queries/0_stateless/02524_fuzz_and_fuss_2.sql b/tests/queries/0_stateless/02524_fuzz_and_fuss_2.sql
index f0d8d5202db..d78e8d34232 100644
--- a/tests/queries/0_stateless/02524_fuzz_and_fuss_2.sql
+++ b/tests/queries/0_stateless/02524_fuzz_and_fuss_2.sql
@@ -9,6 +9,6 @@ ENGINE = Memory;
 INSERT INTO data_a_02187
 SELECT *
 FROM system.one
-SETTINGS max_block_size = '1', min_insert_block_size_rows = '65536', min_insert_block_size_bytes = '0', max_insert_threads = '0', max_threads = '3', receive_timeout = '10', receive_data_timeout_ms = '10000', connections_with_failover_max_tries = '0', extremes = '1', use_uncompressed_cache = '0', optimize_move_to_prewhere = '1', optimize_move_to_prewhere_if_final = '0', replication_alter_partitions_sync = '2', totals_mode = 'before_having', allow_suspicious_low_cardinality_types = '1', compile_expressions = '1', min_count_to_compile_expression = '0', group_by_two_level_threshold = '100', distributed_aggregation_memory_efficient = '0', distributed_group_by_no_merge = '1', optimize_distributed_group_by_sharding_key = '1', optimize_skip_unused_shards = '1', optimize_skip_unused_shards_rewrite_in = '1', force_optimize_skip_unused_shards = '2', optimize_skip_unused_shards_nesting = '1', force_optimize_skip_unused_shards_nesting = '2', merge_tree_min_rows_for_concurrent_read = '10000', force_primary_key = '1', network_compression_method = 'ZSTD', network_zstd_compression_level = '7', log_queries = '0', log_queries_min_type = 'QUERY_FINISH', distributed_product_mode = 'local', insert_quorum = '2', insert_quorum_timeout = '0', insert_quorum_parallel = '0', select_sequential_consistency = '1', join_use_nulls = '1', any_join_distinct_right_table_keys = '1', preferred_max_column_in_block_size_bytes = '32', insert_distributed_sync = '1', insert_allow_materialized_columns = '1', use_index_for_in_with_subqueries = '1', joined_subquery_requires_alias = '0', empty_result_for_aggregation_by_empty_set = '1', allow_suspicious_codecs = '1', query_profiler_real_time_period_ns = '0', query_profiler_cpu_time_period_ns = '0', opentelemetry_start_trace_probability = '1', max_rows_to_read = '1000000', read_overflow_mode = 'break', max_rows_to_group_by = '10', group_by_overflow_mode = 'any', max_rows_to_sort = '100', sort_overflow_mode = 'break', max_result_rows = '10', max_execution_time = '3', max_execution_speed = '1', max_bytes_in_join = '100', join_algorithm = 'partial_merge', max_memory_usage = '1099511627776', log_query_threads = '1', send_logs_level = 'fatal', enable_optimize_predicate_expression = '1', prefer_localhost_replica = '1', optimize_read_in_order = '1', optimize_aggregation_in_order = '1', read_in_order_two_level_merge_threshold = '1', allow_introspection_functions = '1', check_query_single_value_result = '1', allow_experimental_live_view = '1', default_table_engine = 'Memory', mutations_sync = '2', convert_query_to_cnf = '0', optimize_arithmetic_operations_in_aggregate_functions = '1', optimize_duplicate_order_by_and_distinct = '0', optimize_multiif_to_if = '0', optimize_monotonous_functions_in_order_by = '1', optimize_functions_to_subcolumns = '1', optimize_using_constraints = '1', optimize_substitute_columns = '1', optimize_append_index = '1', transform_null_in = '1', allow_experimental_geo_types = '1', data_type_default_nullable = '1', cast_keep_nullable = '1', cast_ipv4_ipv6_default_on_conversion_error = '0', system_events_show_zero_values = '1', enable_global_with_statement = '1', optimize_on_insert = '0', optimize_rewrite_sum_if_to_count_if = '1', distributed_ddl_output_mode = 'throw', union_default_mode = 'ALL', optimize_aggregators_of_group_by_keys = '1', optimize_group_by_function_keys = '1', short_circuit_function_evaluation = 'enable', async_insert = '1', enable_filesystem_cache = '0', allow_deprecated_database_ordinary = '1', allow_deprecated_syntax_for_merge_tree = '1', allow_experimental_nlp_functions = '1', allow_experimental_object_type = '1', allow_experimental_map_type = '1', optimize_use_projections = '1', input_format_null_as_default = '1', input_format_ipv4_default_on_conversion_error = '0', input_format_ipv6_default_on_conversion_error = '0', output_format_json_named_tuples_as_objects = '1', output_format_write_statistics = '0', output_format_pretty_row_numbers = '1';
+SETTINGS max_block_size = '1', min_insert_block_size_rows = '65536', min_insert_block_size_bytes = '0', max_insert_threads = '0', max_threads = '3', receive_timeout = '10', receive_data_timeout_ms = '10000', connections_with_failover_max_tries = '0', extremes = '1', use_uncompressed_cache = '0', optimize_move_to_prewhere = '1', optimize_move_to_prewhere_if_final = '0', replication_alter_partitions_sync = '2', totals_mode = 'before_having', allow_suspicious_low_cardinality_types = '1', compile_expressions = '1', min_count_to_compile_expression = '0', group_by_two_level_threshold = '100', distributed_aggregation_memory_efficient = '0', distributed_group_by_no_merge = '1', optimize_distributed_group_by_sharding_key = '1', optimize_skip_unused_shards = '1', optimize_skip_unused_shards_rewrite_in = '1', force_optimize_skip_unused_shards = '2', optimize_skip_unused_shards_nesting = '1', force_optimize_skip_unused_shards_nesting = '2', merge_tree_min_rows_for_concurrent_read = '10000', force_primary_key = '1', network_compression_method = 'ZSTD', network_zstd_compression_level = '7', log_queries = '0', log_queries_min_type = 'QUERY_FINISH', distributed_product_mode = 'local', insert_quorum = '2', insert_quorum_timeout = '0', insert_quorum_parallel = '0', select_sequential_consistency = '1', join_use_nulls = '1', any_join_distinct_right_table_keys = '1', preferred_max_column_in_block_size_bytes = '32', insert_distributed_sync = '1', insert_allow_materialized_columns = '1', use_index_for_in_with_subqueries = '1', joined_subquery_requires_alias = '0', empty_result_for_aggregation_by_empty_set = '1', allow_suspicious_codecs = '1', query_profiler_real_time_period_ns = '0', query_profiler_cpu_time_period_ns = '0', opentelemetry_start_trace_probability = '1', max_rows_to_read = '1000000', read_overflow_mode = 'break', max_rows_to_group_by = '10', group_by_overflow_mode = 'any', max_rows_to_sort = '100', sort_overflow_mode = 'break', max_result_rows = '10', max_execution_time = '3', max_execution_speed = '1', max_bytes_in_join = '100', join_algorithm = 'partial_merge', max_memory_usage = '1099511627776', log_query_threads = '1', send_logs_level = 'fatal', enable_optimize_predicate_expression = '1', prefer_localhost_replica = '1', optimize_read_in_order = '1', optimize_aggregation_in_order = '1', read_in_order_two_level_merge_threshold = '1', allow_introspection_functions = '1', check_query_single_value_result = '1', allow_experimental_live_view = '1', default_table_engine = 'Memory', mutations_sync = '2', convert_query_to_cnf = '0', optimize_arithmetic_operations_in_aggregate_functions = '1', optimize_duplicate_order_by_and_distinct = '0', optimize_multiif_to_if = '0', optimize_monotonous_functions_in_order_by = '1', optimize_functions_to_subcolumns = '1', optimize_using_constraints = '1', optimize_substitute_columns = '1', optimize_append_index = '1', transform_null_in = '1', data_type_default_nullable = '1', cast_keep_nullable = '1', cast_ipv4_ipv6_default_on_conversion_error = '0', system_events_show_zero_values = '1', enable_global_with_statement = '1', optimize_on_insert = '0', optimize_rewrite_sum_if_to_count_if = '1', distributed_ddl_output_mode = 'throw', union_default_mode = 'ALL', optimize_aggregators_of_group_by_keys = '1', optimize_group_by_function_keys = '1', short_circuit_function_evaluation = 'enable', async_insert = '1', enable_filesystem_cache = '0', allow_deprecated_database_ordinary = '1', allow_deprecated_syntax_for_merge_tree = '1', allow_experimental_nlp_functions = '1', allow_experimental_object_type = '1', allow_experimental_map_type = '1', optimize_use_projections = '1', input_format_null_as_default = '1', input_format_ipv4_default_on_conversion_error = '0', input_format_ipv6_default_on_conversion_error = '0', output_format_json_named_tuples_as_objects = '1', output_format_write_statistics = '0', output_format_pretty_row_numbers = '1';
 
 DROP TABLE data_a_02187;

From e46476dba22af226d1ac47355e0adeaa3759d2db Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio2368@users.noreply.github.com>
Date: Fri, 19 May 2023 12:44:20 +0200
Subject: [PATCH 647/689] Update src/Coordination/Changelog.cpp

Co-authored-by: alesapin <alesapin@clickhouse.com>
---
 src/Coordination/Changelog.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp
index 7275a080252..e20fbf864eb 100644
--- a/src/Coordination/Changelog.cpp
+++ b/src/Coordination/Changelog.cpp
@@ -284,7 +284,8 @@ private:
             do
             {
                 res = ftruncate(file_buffer->getFD(), initial_file_size + file_buffer->count());
-            } while (res < 0 && errno == EINTR);
+            } 
+            while (res < 0 && errno == EINTR);
 
             if (res != 0)
                 LOG_WARNING(log, "Could not ftruncate file. Error: {}, errno: {}", errnoToString(), errno);

From 9c3b17fa18ad14ece86ec665fc0dd5d8396f432c Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio2368@users.noreply.github.com>
Date: Fri, 19 May 2023 13:00:51 +0200
Subject: [PATCH 648/689] Remove whitespace

---
 src/Coordination/Changelog.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp
index e20fbf864eb..894fd93cfa7 100644
--- a/src/Coordination/Changelog.cpp
+++ b/src/Coordination/Changelog.cpp
@@ -284,7 +284,7 @@ private:
             do
             {
                 res = ftruncate(file_buffer->getFD(), initial_file_size + file_buffer->count());
-            } 
+            }
             while (res < 0 && errno == EINTR);
 
             if (res != 0)

From 3107070e7699d14a6895dbdeb556b69986fc604b Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 19 May 2023 12:48:19 +0000
Subject: [PATCH 649/689] Avoid deadlock when starting table in attach thread

---
 .../ReplicatedMergeTreeRestartingThread.h     |  3 +-
 src/Storages/StorageReplicatedMergeTree.cpp   | 25 +++++---
 .../test_replicated_table_attach/__init__.py  |  0
 .../configs/config.xml                        |  3 +
 .../test_replicated_table_attach/test.py      | 58 +++++++++++++++++++
 5 files changed, 79 insertions(+), 10 deletions(-)
 create mode 100644 tests/integration/test_replicated_table_attach/__init__.py
 create mode 100644 tests/integration/test_replicated_table_attach/configs/config.xml
 create mode 100644 tests/integration/test_replicated_table_attach/test.py

diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h
index 8f5721e440d..5579fdf2033 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h
@@ -29,6 +29,7 @@ public:
 
     void shutdown(bool part_of_full_shutdown);
 
+    void run();
 private:
     StorageReplicatedMergeTree & storage;
     String log_name;
@@ -43,8 +44,6 @@ private:
     UInt32 consecutive_check_failures = 0;  /// How many consecutive checks have failed
     bool first_time = true;                 /// Activate replica for the first time.
 
-    void run();
-
     /// Restarts table if needed, returns false if it failed to restart replica.
     bool runImpl();
 
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index fcb7adbd69f..cdd43239bec 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -4416,8 +4416,23 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread)
 
         startBeingLeader();
 
-        /// In this thread replica will be activated.
-        restarting_thread.start();
+        if (from_attach_thread)
+        {
+            /// Try activating replica in current thread.
+            restarting_thread.run();
+        }
+        else
+        {
+            /// Activate replica in a seperate thread.
+            restarting_thread.start();
+
+            /// Wait while restarting_thread finishing initialization.
+            /// NOTE It does not mean that replication is actually started after receiving this event.
+            /// It only means that an attempt to startup replication was made.
+            /// Table may be still in readonly mode if this attempt failed for any reason.
+            startup_event.wait();
+        }
+
         /// And this is just a callback
         session_expired_callback_handler = EventNotifier::instance().subscribe(Coordination::Error::ZSESSIONEXPIRED, [this]()
         {
@@ -4425,12 +4440,6 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread)
             restarting_thread.start();
         });
 
-        /// Wait while restarting_thread finishing initialization.
-        /// NOTE It does not mean that replication is actually started after receiving this event.
-        /// It only means that an attempt to startup replication was made.
-        /// Table may be still in readonly mode if this attempt failed for any reason.
-        startup_event.wait();
-
         startBackgroundMovesIfNeeded();
 
         part_moves_between_shards_orchestrator.start();
diff --git a/tests/integration/test_replicated_table_attach/__init__.py b/tests/integration/test_replicated_table_attach/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_replicated_table_attach/configs/config.xml b/tests/integration/test_replicated_table_attach/configs/config.xml
new file mode 100644
index 00000000000..b7b5d996072
--- /dev/null
+++ b/tests/integration/test_replicated_table_attach/configs/config.xml
@@ -0,0 +1,3 @@
+<clickhouse>
+    <background_schedule_pool_size>1</background_schedule_pool_size>
+</clickhouse>
diff --git a/tests/integration/test_replicated_table_attach/test.py b/tests/integration/test_replicated_table_attach/test.py
new file mode 100644
index 00000000000..c09d67aab77
--- /dev/null
+++ b/tests/integration/test_replicated_table_attach/test.py
@@ -0,0 +1,58 @@
+import pytest
+
+from helpers.cluster import ClickHouseCluster
+from helpers.network import PartitionManager
+
+
+cluster = ClickHouseCluster(__file__)
+
+node = cluster.add_instance(
+    "node",
+    main_configs=["configs/config.xml"],
+    with_zookeeper=True,
+    stay_alive=True,
+)
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.start()
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def test_startup_with_small_bg_pool(started_cluster):
+    node.query(
+        "CREATE TABLE replicated_table (k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/replicated_table', 'r1') ORDER BY k"
+    )
+
+    node.query("INSERT INTO replicated_table VALUES(20, 30)")
+
+    def assert_values():
+        assert node.query("SELECT * FROM replicated_table") == "20\t30\n"
+
+    assert_values()
+    node.restart_clickhouse(stop_start_wait_sec=10)
+    assert_values()
+
+    node.query("DROP TABLE replicated_table SYNC")
+
+def test_startup_with_small_bg_pool_partitioned(started_cluster):
+    node.query(
+        "CREATE TABLE replicated_table_partitioned (k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/replicated_table_partitioned', 'r1') ORDER BY k"
+    )
+
+    node.query("INSERT INTO replicated_table_partitioned VALUES(20, 30)")
+
+    def assert_values():
+        assert node.query("SELECT * FROM replicated_table_partitioned") == "20\t30\n"
+
+    assert_values()
+    with PartitionManager() as pm:
+        pm.drop_instance_zk_connections(node)
+        node.restart_clickhouse(stop_start_wait_sec=20)
+        assert_values()
+
+    node.query("DROP TABLE replicated_table_partitioned SYNC")

From f947d1cc0ae147bc830c8200daf70295dc9aec38 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Fri, 19 May 2023 13:00:26 +0000
Subject: [PATCH 650/689] Automatic style fix

---
 tests/integration/test_replicated_table_attach/test.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/integration/test_replicated_table_attach/test.py b/tests/integration/test_replicated_table_attach/test.py
index c09d67aab77..11365ea9163 100644
--- a/tests/integration/test_replicated_table_attach/test.py
+++ b/tests/integration/test_replicated_table_attach/test.py
@@ -13,6 +13,7 @@ node = cluster.add_instance(
     stay_alive=True,
 )
 
+
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
@@ -39,6 +40,7 @@ def test_startup_with_small_bg_pool(started_cluster):
 
     node.query("DROP TABLE replicated_table SYNC")
 
+
 def test_startup_with_small_bg_pool_partitioned(started_cluster):
     node.query(
         "CREATE TABLE replicated_table_partitioned (k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/replicated_table_partitioned', 'r1') ORDER BY k"

From 55fc4adf055324cc9b359e57a8056025f744a317 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 19 May 2023 16:42:15 +0300
Subject: [PATCH 651/689] Update 02441_alter_delete_and_drop_column.sql

---
 tests/queries/0_stateless/02441_alter_delete_and_drop_column.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/02441_alter_delete_and_drop_column.sql b/tests/queries/0_stateless/02441_alter_delete_and_drop_column.sql
index b9b1b645e8e..9c4697362df 100644
--- a/tests/queries/0_stateless/02441_alter_delete_and_drop_column.sql
+++ b/tests/queries/0_stateless/02441_alter_delete_and_drop_column.sql
@@ -1,3 +1,4 @@
+-- Tags: no-replicated-database
 
 create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n;
 set insert_keeper_fault_injection_probability=0;

From acf71c5b9a976834202b856494f027421ee77422 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio2368@users.noreply.github.com>
Date: Fri, 19 May 2023 15:48:31 +0200
Subject: [PATCH 652/689] Fix typo

---
 src/Storages/StorageReplicatedMergeTree.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index cdd43239bec..6373c164c40 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -4423,7 +4423,7 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread)
         }
         else
         {
-            /// Activate replica in a seperate thread.
+            /// Activate replica in a separate thread.
             restarting_thread.start();
 
             /// Wait while restarting_thread finishing initialization.

From 5237dd02451cb23d4101c0cecfe8bf133292103e Mon Sep 17 00:00:00 2001
From: mateng915 <mateng915@outlook.com>
Date: Fri, 19 May 2023 22:06:43 +0800
Subject: [PATCH 653/689] New system table zookeeper connection (#45245)

* Feature: Support new system table to show which zookeeper node be connected

Description:
============
Currently we have no place to check which zk node be connected otherwise using
lsof command. It not convenient

Solution:
=========
Implemented a new system table, system.zookeeper_host when CK Server has zk
this table will show the zk node dir which connected by current CK server

Noted: This table can support multi-zookeeper cluster scenario.

* fixed review comments

* added test case

* update test cases

* remove unused code

* fixed review comments and removed unused code

* updated test cases for print host, port and is_expired

* modify the code comments

* fixed CI Failed

* fixed code style check failure

* updated test cases by added Tags

* update test reference

* update test cases

* added system.zookeeper_connection doc

* Update docs/en/operations/system-tables/zookeeper_connection.md

* Update docs/en/operations/system-tables/zookeeper_connection.md

* Update docs/en/operations/system-tables/zookeeper_connection.md

---------

Co-authored-by: Alexander Tokmakov <tavplubix@gmail.com>
---
 .../system-tables/zookeeper_connection.md     | 29 +++++++++++
 src/Common/ZooKeeper/IKeeper.h                |  2 +
 src/Common/ZooKeeper/TestKeeper.h             |  3 ++
 src/Common/ZooKeeper/ZooKeeper.cpp            | 20 +++++++
 src/Common/ZooKeeper/ZooKeeper.h              |  8 +++
 src/Common/ZooKeeper/ZooKeeperImpl.cpp        |  4 ++
 src/Common/ZooKeeper/ZooKeeperImpl.h          |  3 ++
 src/Interpreters/Context.cpp                  | 11 ++++
 src/Interpreters/Context.h                    |  8 ++-
 .../StorageSystemZooKeeperConnection.cpp      | 52 +++++++++++++++++++
 .../System/StorageSystemZooKeeperConnection.h | 28 ++++++++++
 src/Storages/System/attachSystemTables.cpp    |  4 ++
 ...2735_system_zookeeper_connection.reference |  1 +
 .../02735_system_zookeeper_connection.sql     | 13 +++++
 14 files changed, 185 insertions(+), 1 deletion(-)
 create mode 100644 docs/en/operations/system-tables/zookeeper_connection.md
 create mode 100644 src/Storages/System/StorageSystemZooKeeperConnection.cpp
 create mode 100644 src/Storages/System/StorageSystemZooKeeperConnection.h
 create mode 100644 tests/queries/0_stateless/02735_system_zookeeper_connection.reference
 create mode 100644 tests/queries/0_stateless/02735_system_zookeeper_connection.sql

diff --git a/docs/en/operations/system-tables/zookeeper_connection.md b/docs/en/operations/system-tables/zookeeper_connection.md
new file mode 100644
index 00000000000..9438cda1808
--- /dev/null
+++ b/docs/en/operations/system-tables/zookeeper_connection.md
@@ -0,0 +1,29 @@
+---
+slug: /en/operations/system-tables/zookeeper_connection
+---
+#zookeeper_connection
+
+This table does not exist if ZooKeeper is not configured. The 'system.zookeeper_connection' table shows current connections to ZooKeeper (including auxiliary ZooKeepers). Each row shows information about one connection.
+
+Columns:
+
+-   `name` ([String](../../sql-reference/data-types/string.md)) — ZooKeeper cluster's name.
+-   `host` ([String](../../sql-reference/data-types/string.md)) — The hostname/IP of the ZooKeeper node that ClickHouse connected to.
+-   `port` ([String](../../sql-reference/data-types/string.md)) — The port of the ZooKeeper node that ClickHouse connected to.
+-   `index` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The index of the ZooKeeper node that ClickHouse connected to. The index is from ZooKeeper config.
+-   `connected_time` ([String](../../sql-reference/data-types/string.md)) — When the connection was established
+-   `is_expired` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Is the current connection expired.
+-   `keeper_api_version` ([String](../../sql-reference/data-types/string.md)) — Keeper API version.
+-   `client_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Session id of the connection.
+
+Example:
+
+``` sql
+SELECT * FROM system.zookeeper_connection;
+```
+
+``` text
+┌─name──────────────┬─host─────────┬─port─┬─index─┬──────connected_time─┬─is_expired─┬─keeper_api_version─┬──────────client_id─┐
+│ default_zookeeper │ 127.0.0.1    │ 2181 │     0 │ 2023-05-19 14:30:16 │          0 │                  0 │ 216349144108826660 │
+└───────────────────┴──────────────┴──────┴───────┴─────────────────────┴────────────┴────────────────────┴────────────────────┘
+```
diff --git a/src/Common/ZooKeeper/IKeeper.h b/src/Common/ZooKeeper/IKeeper.h
index b09f096d761..86f9a388644 100644
--- a/src/Common/ZooKeeper/IKeeper.h
+++ b/src/Common/ZooKeeper/IKeeper.h
@@ -466,6 +466,8 @@ public:
     /// Useful to check owner of ephemeral node.
     virtual int64_t getSessionID() const = 0;
 
+    virtual String getConnectedAddress() const = 0;
+
     /// If the method will throw an exception, callbacks won't be called.
     ///
     /// After the method is executed successfully, you must wait for callbacks
diff --git a/src/Common/ZooKeeper/TestKeeper.h b/src/Common/ZooKeeper/TestKeeper.h
index 27405d8d571..11e56daf6b4 100644
--- a/src/Common/ZooKeeper/TestKeeper.h
+++ b/src/Common/ZooKeeper/TestKeeper.h
@@ -39,6 +39,7 @@ public:
 
     bool isExpired() const override { return expired; }
     int64_t getSessionID() const override { return 0; }
+    String getConnectedAddress() const override { return connected_zk_address; }
 
 
     void create(
@@ -126,6 +127,8 @@ private:
 
     zkutil::ZooKeeperArgs args;
 
+    String connected_zk_address;
+
     std::mutex push_request_mutex;
     std::atomic<bool> expired{false};
 
diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp
index 09047b5b232..c423e4fd498 100644
--- a/src/Common/ZooKeeper/ZooKeeper.cpp
+++ b/src/Common/ZooKeeper/ZooKeeper.cpp
@@ -111,6 +111,26 @@ void ZooKeeper::init(ZooKeeperArgs args_)
             LOG_TRACE(log, "Initialized, hosts: {}", fmt::join(args.hosts, ","));
         else
             LOG_TRACE(log, "Initialized, hosts: {}, chroot: {}", fmt::join(args.hosts, ","), args.chroot);
+
+        String address = impl->getConnectedAddress();
+
+        size_t colon_pos = address.find(':');
+        connected_zk_host = address.substr(0, colon_pos);
+        connected_zk_port = address.substr(colon_pos + 1);
+
+        connected_zk_index = 0;
+
+        if (args.hosts.size() > 1)
+        {
+            for (size_t i = 0; i < args.hosts.size(); i++)
+            {
+                if (args.hosts[i] == address)
+                {
+                    connected_zk_index = i;
+                    break;
+                }
+            }
+        }
     }
     else if (args.implementation == "testkeeper")
     {
diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h
index ca6a44c4cbc..9b85938c726 100644
--- a/src/Common/ZooKeeper/ZooKeeper.h
+++ b/src/Common/ZooKeeper/ZooKeeper.h
@@ -523,6 +523,10 @@ public:
 
     void setServerCompletelyStarted();
 
+    String getConnectedZooKeeperHost() const { return connected_zk_host; }
+    String getConnectedZooKeeperPort() const { return connected_zk_port; }
+    size_t getConnectedZooKeeperIndex() const { return connected_zk_index; }
+
 private:
     void init(ZooKeeperArgs args_);
 
@@ -586,6 +590,10 @@ private:
 
     ZooKeeperArgs args;
 
+    String connected_zk_host;
+    String connected_zk_port;
+    size_t connected_zk_index;
+
     std::mutex mutex;
 
     Poco::Logger * log = nullptr;
diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp
index 6c79fc4f178..34be8aa1332 100644
--- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp
+++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp
@@ -433,6 +433,8 @@ void ZooKeeper::connect(
                 }
 
                 connected = true;
+                connected_zk_address = node.address.toString();
+
                 break;
             }
             catch (...)
@@ -448,6 +450,8 @@ void ZooKeeper::connect(
     if (!connected)
     {
         WriteBufferFromOwnString message;
+        connected_zk_address = "";
+
         message << "All connection tries failed while connecting to ZooKeeper. nodes: ";
         bool first = true;
         for (const auto & node : nodes)
diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h
index c0c57d3f719..6715607ca88 100644
--- a/src/Common/ZooKeeper/ZooKeeperImpl.h
+++ b/src/Common/ZooKeeper/ZooKeeperImpl.h
@@ -125,6 +125,8 @@ public:
     /// Useful to check owner of ephemeral node.
     int64_t getSessionID() const override { return session_id; }
 
+    String getConnectedAddress() const override { return connected_zk_address; }
+
     void executeGenericRequest(
         const ZooKeeperRequestPtr & request,
         ResponseCallback callback);
@@ -201,6 +203,7 @@ public:
 
 private:
     ACLs default_acls;
+    String connected_zk_address;
 
     zkutil::ZooKeeperArgs args;
 
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index 0ef9ea53ee8..d9f450191bc 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -2778,6 +2778,17 @@ zkutil::ZooKeeperPtr Context::getAuxiliaryZooKeeper(const String & name) const
     return zookeeper->second;
 }
 
+
+std::map<String, zkutil::ZooKeeperPtr> Context::getAuxiliaryZooKeepers() const
+{
+    std::lock_guard lock(shared->auxiliary_zookeepers_mutex);
+
+    if (!shared->auxiliary_zookeepers.empty())
+        return shared->auxiliary_zookeepers;
+    else
+        return std::map<String, zkutil::ZooKeeperPtr>();
+}
+
 #if USE_ROCKSDB
 MergeTreeMetadataCachePtr Context::getMergeTreeMetadataCache() const
 {
diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h
index 87843a458e8..15f2ff625ef 100644
--- a/src/Interpreters/Context.h
+++ b/src/Interpreters/Context.h
@@ -32,7 +32,11 @@
 
 
 namespace Poco::Net { class IPAddress; }
-namespace zkutil { class ZooKeeper; }
+namespace zkutil
+{
+    class ZooKeeper;
+    using ZooKeeperPtr = std::shared_ptr<ZooKeeper>;
+}
 
 struct OvercommitTracker;
 
@@ -827,6 +831,8 @@ public:
     std::shared_ptr<zkutil::ZooKeeper> getZooKeeper() const;
     /// Same as above but return a zookeeper connection from auxiliary_zookeepers configuration entry.
     std::shared_ptr<zkutil::ZooKeeper> getAuxiliaryZooKeeper(const String & name) const;
+    /// return Auxiliary Zookeeper map
+    std::map<String, zkutil::ZooKeeperPtr> getAuxiliaryZooKeepers() const;
 
     /// Try to connect to Keeper using get(Auxiliary)ZooKeeper. Useful for
     /// internal Keeper start (check connection to some other node). Return true
diff --git a/src/Storages/System/StorageSystemZooKeeperConnection.cpp b/src/Storages/System/StorageSystemZooKeeperConnection.cpp
new file mode 100644
index 00000000000..f249097654e
--- /dev/null
+++ b/src/Storages/System/StorageSystemZooKeeperConnection.cpp
@@ -0,0 +1,52 @@
+#include <Interpreters/Context.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/DataTypeDateTime.h>
+#include <Common/ZooKeeper/ZooKeeper.h>
+#include <Storages/System/StorageSystemZooKeeperConnection.h>
+
+namespace DB
+{
+
+NamesAndTypesList StorageSystemZooKeeperConnection::getNamesAndTypes()
+{
+    return {
+        {"name", std::make_shared<DataTypeString>()},
+        {"host", std::make_shared<DataTypeString>()},
+        {"port", std::make_shared<DataTypeString>()},
+        {"index", std::make_shared<DataTypeUInt8>()},
+        {"connected_time", std::make_shared<DataTypeDateTime>()},
+        {"is_expired", std::make_shared<DataTypeUInt8>()},
+        {"keeper_api_version", std::make_shared<DataTypeUInt8>()},
+        {"client_id", std::make_shared<DataTypeInt64>()}
+    };
+}
+
+void StorageSystemZooKeeperConnection::fillData(MutableColumns & res_columns, ContextPtr context,
+    const SelectQueryInfo &) const
+{
+    res_columns[0]->insert("default_zookeeper");
+    res_columns[1]->insert(context->getZooKeeper()->getConnectedZooKeeperHost());
+    res_columns[2]->insert(context->getZooKeeper()->getConnectedZooKeeperPort());
+    res_columns[3]->insert(context->getZooKeeper()->getConnectedZooKeeperIndex());
+    res_columns[4]->insert(context->getZooKeeperSessionUptime());
+    res_columns[5]->insert(context->getZooKeeper()->expired());
+    res_columns[6]->insert(context->getZooKeeper()->getApiVersion());
+    res_columns[7]->insert(context->getZooKeeper()->getClientID());
+
+    for (const auto & elem : context->getAuxiliaryZooKeepers())
+    {
+        res_columns[0]->insert(elem.first);
+        res_columns[1]->insert(elem.second->getConnectedZooKeeperHost());
+        res_columns[1]->insert(elem.second->getConnectedZooKeeperHost());
+        res_columns[2]->insert(elem.second->getConnectedZooKeeperPort());
+        res_columns[3]->insert(elem.second->getConnectedZooKeeperIndex());
+        res_columns[4]->insert(elem.second->getSessionUptime());
+        res_columns[5]->insert(elem.second->expired());
+        res_columns[6]->insert(elem.second->getApiVersion());
+        res_columns[7]->insert(elem.second->getClientID());
+    }
+
+}
+
+}
diff --git a/src/Storages/System/StorageSystemZooKeeperConnection.h b/src/Storages/System/StorageSystemZooKeeperConnection.h
new file mode 100644
index 00000000000..dd4c293c112
--- /dev/null
+++ b/src/Storages/System/StorageSystemZooKeeperConnection.h
@@ -0,0 +1,28 @@
+#pragma once
+
+#include <Storages/System/IStorageSystemOneBlock.h>
+
+
+namespace DB
+{
+
+class Context;
+
+
+/** Implements `zookeeper_connection` system table, which allows you to get information about the connected zookeeper info.
+  */
+class StorageSystemZooKeeperConnection final : public IStorageSystemOneBlock<StorageSystemZooKeeperConnection>
+{
+public:
+    std::string getName() const override { return "SystemZooKeeperConnection"; }
+
+    static NamesAndTypesList getNamesAndTypes();
+
+protected:
+    using IStorageSystemOneBlock::IStorageSystemOneBlock;
+
+    void fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo & query_info) const override;
+};
+
+}
+
diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp
index d6982ba30d5..424c74662ec 100644
--- a/src/Storages/System/attachSystemTables.cpp
+++ b/src/Storages/System/attachSystemTables.cpp
@@ -80,6 +80,7 @@
 #include <Storages/System/StorageSystemCertificates.h>
 #include <Storages/System/StorageSystemSchemaInferenceCache.h>
 #include <Storages/System/StorageSystemDroppedTables.h>
+#include <Storages/System/StorageSystemZooKeeperConnection.h>
 
 #ifdef OS_LINUX
 #include <Storages/System/StorageSystemStackTrace.h>
@@ -186,7 +187,10 @@ void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, b
     attach<StorageSystemNamedCollections>(context, system_database, "named_collections");
 
     if (has_zookeeper)
+    {
         attach<StorageSystemZooKeeper>(context, system_database, "zookeeper");
+        attach<StorageSystemZooKeeperConnection>(context, system_database, "zookeeper_connection");
+    }
 
     if (context->getConfigRef().getInt("allow_experimental_transactions", 0))
         attach<StorageSystemTransactions>(context, system_database, "transactions");
diff --git a/tests/queries/0_stateless/02735_system_zookeeper_connection.reference b/tests/queries/0_stateless/02735_system_zookeeper_connection.reference
new file mode 100644
index 00000000000..c9cc8adede8
--- /dev/null
+++ b/tests/queries/0_stateless/02735_system_zookeeper_connection.reference
@@ -0,0 +1 @@
+[	:1]:9181	0
diff --git a/tests/queries/0_stateless/02735_system_zookeeper_connection.sql b/tests/queries/0_stateless/02735_system_zookeeper_connection.sql
new file mode 100644
index 00000000000..10f12177b2e
--- /dev/null
+++ b/tests/queries/0_stateless/02735_system_zookeeper_connection.sql
@@ -0,0 +1,13 @@
+-- Tags: no-fasttest
+
+DROP TABLE IF EXISTS test_zk_connection_table;
+
+CREATE TABLE test_zk_connection_table (
+    key UInt64
+)
+ENGINE ReplicatedMergeTree('/clickhouse/{database}/02731_zk_connection/{shard}', '{replica}')
+ORDER BY tuple();
+
+select host, port, is_expired from system.zookeeper_connection where name='default_zookeeper';
+
+DROP TABLE IF EXISTS test_zk_connection_table;

From b29edc473786a6c3cd5dcc37a53fa36c32283782 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Fri, 19 May 2023 16:38:14 +0200
Subject: [PATCH 654/689] Add method

---
 src/IO/S3/Client.cpp | 16 ++++++++++++----
 src/IO/S3/Client.h   |  9 +++++++--
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp
index 93ef30a927e..668b1a3959d 100644
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@@ -133,11 +133,12 @@ ProviderType deduceProviderType(const std::string & url)
 Client::Client(
     size_t max_redirects_,
     ServerSideEncryptionKMSConfig sse_kms_config_,
-    const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> & credentials_provider,
+    const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> & credentials_provider_,
     const Aws::Client::ClientConfiguration & client_configuration,
     Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads,
     bool use_virtual_addressing)
-    : Aws::S3::S3Client(credentials_provider, client_configuration, std::move(sign_payloads), use_virtual_addressing)
+    : Aws::S3::S3Client(credentials_provider_, client_configuration, std::move(sign_payloads), use_virtual_addressing)
+    , credentials_provider(credentials_provider_)
     , max_redirects(max_redirects_)
     , sse_kms_config(std::move(sse_kms_config_))
     , log(&Poco::Logger::get("S3Client"))
@@ -177,6 +178,7 @@ Client::Client(
 Client::Client(const Client & other)
     : Aws::S3::S3Client(other)
     , initial_endpoint(other.initial_endpoint)
+    , credentials_provider(other.credentials_provider)
     , explicit_region(other.explicit_region)
     , detect_region(other.detect_region)
     , provider_type(other.provider_type)
@@ -188,6 +190,11 @@ Client::Client(const Client & other)
     ClientCacheRegistry::instance().registerClient(cache);
 }
 
+Aws::Auth::AWSCredentials Client::getCredentials() const
+{
+    return credentials_provider->GetAWSCredentials();
+}
+
 bool Client::checkIfWrongRegionDefined(const std::string & bucket, const Aws::S3::S3Error & error, std::string & region) const
 {
     if (detect_region)
@@ -711,7 +718,8 @@ std::unique_ptr<S3::Client> ClientFactory::create( // NOLINT
     const String & server_side_encryption_customer_key_base64,
     ServerSideEncryptionKMSConfig sse_kms_config,
     HTTPHeaderEntries headers,
-    CredentialsConfiguration credentials_configuration)
+    CredentialsConfiguration credentials_configuration,
+    const String & session_token)
 {
     PocoHTTPClientConfiguration client_configuration = cfg_;
     client_configuration.updateSchemeAndRegion();
@@ -735,7 +743,7 @@ std::unique_ptr<S3::Client> ClientFactory::create( // NOLINT
     // These will be added after request signing
     client_configuration.extra_headers = std::move(headers);
 
-    Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key);
+    Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key, session_token);
     auto credentials_provider = std::make_shared<S3CredentialsProviderChain>(
             client_configuration,
             std::move(credentials),
diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h
index 36edb443681..e1b99c893a6 100644
--- a/src/IO/S3/Client.h
+++ b/src/IO/S3/Client.h
@@ -138,6 +138,9 @@ public:
 
     /// Returns the initial endpoint.
     const String & getInitialEndpoint() const { return initial_endpoint; }
+    const String & getRegion() const { return explicit_region; }
+
+    Aws::Auth::AWSCredentials getCredentials() const;
 
     /// Decorator for RetryStrategy needed for this client to work correctly.
     /// We want to manually handle permanent moves (status code 301) because:
@@ -207,7 +210,7 @@ private:
 
     Client(size_t max_redirects_,
            ServerSideEncryptionKMSConfig sse_kms_config_,
-           const std::shared_ptr<Aws::Auth::AWSCredentialsProvider>& credentials_provider,
+           const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> & credentials_provider_,
            const Aws::Client::ClientConfiguration& client_configuration,
            Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads,
            bool use_virtual_addressing);
@@ -247,6 +250,7 @@ private:
     void insertRegionOverride(const std::string & bucket, const std::string & region) const;
 
     String initial_endpoint;
+    std::shared_ptr<Aws::Auth::AWSCredentialsProvider> credentials_provider;
 
     std::string explicit_region;
     mutable bool detect_region = true;
@@ -282,7 +286,8 @@ public:
         const String & server_side_encryption_customer_key_base64,
         ServerSideEncryptionKMSConfig sse_kms_config,
         HTTPHeaderEntries headers,
-        CredentialsConfiguration credentials_configuration);
+        CredentialsConfiguration credentials_configuration,
+        const String & session_token = "");
 
     PocoHTTPClientConfiguration createClientConfiguration(
         const String & force_region,

From 10a382bd12de3b31c54655587e28cd06a1123360 Mon Sep 17 00:00:00 2001
From: DanRoscigno <dan@roscigno.com>
Date: Fri, 19 May 2023 10:49:43 -0400
Subject: [PATCH 655/689] add ALTER and DROP

---
 docs/en/operations/named-collections.md | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/docs/en/operations/named-collections.md b/docs/en/operations/named-collections.md
index 1866315529e..8ad403e448e 100644
--- a/docs/en/operations/named-collections.md
+++ b/docs/en/operations/named-collections.md
@@ -57,8 +57,7 @@ In the above example the `passowrd_sha256_hex` value is the hexadecimal represen
 
 ### XML example
 
-```xml
-$ cat /etc/clickhouse-server/config.d/named_collections.xml
+```xml title='/etc/clickhouse-server/config.d/named_collections.xml'
 <clickhouse>
      <named_collections>
         <name>
@@ -70,6 +69,27 @@ $ cat /etc/clickhouse-server/config.d/named_collections.xml
 </clickhouse>
 ```
 
+## Modifying named collections
+
+Named collections that are created with DDL queries can be altered or dropped with DDL.  Named collections created with XML files can be managed by editing or deleting the corresponding XML.
+
+### Alter a DDL named collection
+
+Change or add the keys `key1` and `key3` of the collection `collection2`:
+```sql
+ALTER NAMED COLLECTION collection2 SET key1=4, key3='value3'
+```
+
+Remove the key `key2` from `collection2`:
+```sql
+ALTER NAMED COLLECTION collection2 DELETE key2
+```
+
+### Drop the DDL named collection `collection2`:
+```sql
+DROP NAMED COLLECTION collection2
+```
+
 ## Named collections for accessing S3
 
 The description of parameters see [s3 Table Function](../sql-reference/table-functions/s3.md).
@@ -389,3 +409,4 @@ SELECT dictGet('dict', 'b', 1);
 │ a                       │
 └─────────────────────────┘
 ```
+

From ff63a73cf535fd9b18b20e6986748f8f5fc93a9f Mon Sep 17 00:00:00 2001
From: DanRoscigno <dan@roscigno.com>
Date: Fri, 19 May 2023 11:03:53 -0400
Subject: [PATCH 656/689] review

---
 docs/en/operations/named-collections.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/en/operations/named-collections.md b/docs/en/operations/named-collections.md
index 8ad403e448e..fc2d7c3540c 100644
--- a/docs/en/operations/named-collections.md
+++ b/docs/en/operations/named-collections.md
@@ -71,7 +71,7 @@ In the above example the `passowrd_sha256_hex` value is the hexadecimal represen
 
 ## Modifying named collections
 
-Named collections that are created with DDL queries can be altered or dropped with DDL.  Named collections created with XML files can be managed by editing or deleting the corresponding XML.
+Named collections that are created with DDL queries can be altered or dropped with DDL. Named collections created with XML files can be managed by editing or deleting the corresponding XML.
 
 ### Alter a DDL named collection
 
@@ -85,6 +85,11 @@ Remove the key `key2` from `collection2`:
 ALTER NAMED COLLECTION collection2 DELETE key2
 ```
 
+Change or add the key `key1` and delete the key `key3` of the collection `collection2`:
+```sql
+ALTER NAMED COLLECTION collection2 SET key1=4, DELETE key3
+```
+
 ### Drop the DDL named collection `collection2`:
 ```sql
 DROP NAMED COLLECTION collection2

From 4af818746461a59f73c3e36de291eefdbabf8e95 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 19 May 2023 15:06:02 +0000
Subject: [PATCH 657/689] Activate restarting thread in both cases

---
 .../MergeTree/ReplicatedMergeTreeRestartingThread.h       | 8 +++++++-
 src/Storages/StorageReplicatedMergeTree.cpp               | 6 +++---
 .../test_replicated_table_attach/configs/config.xml       | 3 +++
 tests/integration/test_replicated_table_attach/test.py    | 3 +++
 4 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h
index 5579fdf2033..9e99baab4c3 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h
@@ -23,7 +23,13 @@ class ReplicatedMergeTreeRestartingThread
 public:
     explicit ReplicatedMergeTreeRestartingThread(StorageReplicatedMergeTree & storage_);
 
-    void start() { task->activateAndSchedule(); }
+    void start(bool schedule = true)
+    {
+        if (schedule)
+            task->activateAndSchedule();
+        else
+            task->activate();
+    }
 
     void wakeup() { task->schedule(); }
 
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 6373c164c40..eccdded9a6f 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -4416,6 +4416,9 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread)
 
         startBeingLeader();
 
+        /// Activate replica in a separate thread if we are not calling from attach thread
+        restarting_thread.start(/*schedule=*/!from_attach_thread);
+
         if (from_attach_thread)
         {
             /// Try activating replica in current thread.
@@ -4423,9 +4426,6 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread)
         }
         else
         {
-            /// Activate replica in a separate thread.
-            restarting_thread.start();
-
             /// Wait while restarting_thread finishing initialization.
             /// NOTE It does not mean that replication is actually started after receiving this event.
             /// It only means that an attempt to startup replication was made.
diff --git a/tests/integration/test_replicated_table_attach/configs/config.xml b/tests/integration/test_replicated_table_attach/configs/config.xml
index b7b5d996072..fea3eab4126 100644
--- a/tests/integration/test_replicated_table_attach/configs/config.xml
+++ b/tests/integration/test_replicated_table_attach/configs/config.xml
@@ -1,3 +1,6 @@
 <clickhouse>
     <background_schedule_pool_size>1</background_schedule_pool_size>
+    <merge_tree>
+        <initialization_retry_period>5</initialization_retry_period>
+    </merge_tree>
 </clickhouse>
diff --git a/tests/integration/test_replicated_table_attach/test.py b/tests/integration/test_replicated_table_attach/test.py
index 11365ea9163..2d209ddaf79 100644
--- a/tests/integration/test_replicated_table_attach/test.py
+++ b/tests/integration/test_replicated_table_attach/test.py
@@ -57,4 +57,7 @@ def test_startup_with_small_bg_pool_partitioned(started_cluster):
         node.restart_clickhouse(stop_start_wait_sec=20)
         assert_values()
 
+    # check that we activate it in the end
+    node.query_with_retry("INSERT INTO replicated_table_partitioned VALUES(20, 30)")
+
     node.query("DROP TABLE replicated_table_partitioned SYNC")

From 90872c2671f468cef899834d8fb15af6e5896960 Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Fri, 19 May 2023 17:22:37 +0200
Subject: [PATCH 658/689] Update query_log.md

Fix links to data types
---
 docs/en/operations/system-tables/query_log.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/operations/system-tables/query_log.md b/docs/en/operations/system-tables/query_log.md
index 42247e6fba2..1bcecfeb161 100644
--- a/docs/en/operations/system-tables/query_log.md
+++ b/docs/en/operations/system-tables/query_log.md
@@ -97,8 +97,8 @@ Columns:
 - `forwarded_for` ([String](../../sql-reference/data-types/string.md)) — HTTP header `X-Forwarded-For` passed in the HTTP query.
 - `quota_key` ([String](../../sql-reference/data-types/string.md)) — The `quota key` specified in the [quotas](../../operations/quotas.md) setting (see `keyed`).
 - `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse revision.
-- `ProfileEvents` ([Map(String, UInt64)](../../sql-reference/data-types/array.md)) — ProfileEvents that measure different metrics. The description of them could be found in the table [system.events](../../operations/system-tables/events.md#system_tables-events)
-- `Settings` ([Map(String, String)](../../sql-reference/data-types/array.md)) — Settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` parameter to 1.
+- `ProfileEvents` ([Map(String, UInt64)](../../sql-reference/data-types/map.md)) — ProfileEvents that measure different metrics. The description of them could be found in the table [system.events](../../operations/system-tables/events.md#system_tables-events)
+- `Settings` ([Map(String, String)](../../sql-reference/data-types/map.md)) — Settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` parameter to 1.
 - `log_comment` ([String](../../sql-reference/data-types/string.md)) — Log comment. It can be set to arbitrary string no longer than [max_query_size](../../operations/settings/settings.md#settings-max_query_size). An empty string if it is not defined.
 - `thread_ids` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — Thread ids that are participating in query execution.
 - `used_aggregate_functions` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `aggregate functions`, which were used during query execution.

From 4260e86b1a1dd05ba9ca65cb6cba3c61a636bb06 Mon Sep 17 00:00:00 2001
From: DanRoscigno <dan@roscigno.com>
Date: Fri, 19 May 2023 13:00:18 -0400
Subject: [PATCH 659/689] fix typo

---
 docs/en/operations/named-collections.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/operations/named-collections.md b/docs/en/operations/named-collections.md
index fc2d7c3540c..7db8f2b642b 100644
--- a/docs/en/operations/named-collections.md
+++ b/docs/en/operations/named-collections.md
@@ -50,7 +50,7 @@ To manage named collections with DDL a user must have the `named_control_collect
 ```
 
 :::tip
-In the above example the `passowrd_sha256_hex` value is the hexadecimal representation of the SHA256 hash of the password.  This configuration for the user `default` has the attribute `replace=true` as in the default configuration has a plain text `password` set, and it is not possible to have both plaintext and sha256 hex passwords set for a user. 
+In the above example the `passowrd_sha256_hex` value is the hexadecimal representation of the SHA256 hash of the password.  This configuration for the user `default` has the attribute `replace=true` as in the default configuration has a plain text `password` set, and it is not possible to have both plain text and sha256 hex passwords set for a user. 
 :::
 
 ## Storing named collections in configuration files

From 35e77f8e2a8aaecc1d3328863ab38670908fab72 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 19 May 2023 17:53:22 +0000
Subject: [PATCH 660/689] fix comment

---
 src/Interpreters/Cluster.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp
index b696b539013..8c30dbe230f 100644
--- a/src/Interpreters/Cluster.cpp
+++ b/src/Interpreters/Cluster.cpp
@@ -487,8 +487,8 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config,
                     throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown element in config: {}", replica_key);
             }
 
-            addShard(settings, std::move(replica_addresses), false, current_shard_num,
-                     std::move(insert_paths), /* treat_local_as_remote */ weight, internal_replication);
+            addShard(settings, std::move(replica_addresses), /* treat_local_as_remote = */ false, current_shard_num,
+                     std::move(insert_paths), weight, internal_replication);
         }
         else
             throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown element in config: {}", key);

From 086888b2856b52edcbca2ff9c1d949ed1c48142a Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 19 May 2023 17:54:29 +0000
Subject: [PATCH 661/689] fix ConnectionPoolWithFailover::getPriority

---
 src/Client/ConnectionPoolWithFailover.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp
index e0c46804c60..129bc10bc27 100644
--- a/src/Client/ConnectionPoolWithFailover.cpp
+++ b/src/Client/ConnectionPoolWithFailover.cpp
@@ -73,9 +73,9 @@ IConnectionPool::Entry ConnectionPoolWithFailover::get(const ConnectionTimeouts
 
 Int64 ConnectionPoolWithFailover::getPriority() const
 {
-    return (*std::max_element(nested_pools.begin(), nested_pools.end(), [](const auto &a, const auto &b)
+    return (*std::max_element(nested_pools.begin(), nested_pools.end(), [](const auto & a, const auto & b)
     {
-        return a->getPriority() - b->getPriority();
+        return a->getPriority() < b->getPriority();
     }))->getPriority();
 }
 

From d69c35fcdd4b183f94acffbf8fc6e73988d39ac1 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 19 May 2023 17:57:00 +0000
Subject: [PATCH 662/689] fix PoolWithFailover `error_count` integer overflow

---
 src/Client/ConnectionPool.h       | 1 -
 src/Common/PoolWithFailoverBase.h | 8 ++++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/Client/ConnectionPool.h b/src/Client/ConnectionPool.h
index 9b1f7dc5b9d..aacd0a063c7 100644
--- a/src/Client/ConnectionPool.h
+++ b/src/Client/ConnectionPool.h
@@ -135,7 +135,6 @@ private:
     Protocol::Compression compression; /// Whether to compress data when interacting with the server.
     Protocol::Secure secure;           /// Whether to encrypt data when interacting with the server.
     Int64 priority;                    /// priority from <remote_servers>
-
 };
 
 /**
diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h
index 0e8fbb4e6d1..928d8b74a3f 100644
--- a/src/Common/PoolWithFailoverBase.h
+++ b/src/Common/PoolWithFailoverBase.h
@@ -101,7 +101,7 @@ public:
     struct ShuffledPool
     {
         NestedPool * pool{};
-        const PoolState * state{};
+        const PoolState * state{}; // WARNING: valid only during initial ordering, dangling
         size_t index = 0;
         size_t error_count = 0;
         size_t slowdown_count = 0;
@@ -115,7 +115,6 @@ public:
     /// this functor. The pools with lower result value will be tried first.
     using GetPriorityFunc = std::function<size_t(size_t index)>;
 
-
     /// Returns at least min_entries and at most max_entries connections (at most one connection per nested pool).
     /// The method will throw if it is unable to get min_entries alive connections or
     /// if fallback_to_stale_replicas is false and it is unable to get min_entries connections to up-to-date replicas.
@@ -175,10 +174,11 @@ PoolWithFailoverBase<TNestedPool>::getShuffledPools(
     }
 
     /// Sort the pools into order in which they will be tried (based on respective PoolStates).
+    /// Note that `error_count` and `slowdown_count` are used for ordering, but set to zero in the resulting ShuffledPool
     std::vector<ShuffledPool> shuffled_pools;
     shuffled_pools.reserve(nested_pools.size());
     for (size_t i = 0; i < nested_pools.size(); ++i)
-        shuffled_pools.push_back(ShuffledPool{nested_pools[i].get(), &pool_states[i], i, 0});
+        shuffled_pools.push_back(ShuffledPool{nested_pools[i].get(), &pool_states[i], i, /* error_count = */ 0, /* slowdown_count = */ 0});
     ::sort(
         shuffled_pools.begin(), shuffled_pools.end(),
         [](const ShuffledPool & lhs, const ShuffledPool & rhs)
@@ -371,7 +371,7 @@ PoolWithFailoverBase<TNestedPool>::updatePoolStates(size_t max_ignored_errors)
 
     /// distributed_replica_max_ignored_errors
     for (auto & state : result)
-        state.error_count = std::max<UInt64>(0, state.error_count - max_ignored_errors);
+        state.error_count = state.error_count > max_ignored_errors ? state.error_count - max_ignored_errors : 0;
 
     return result;
 }

From 052d8aca71e0639769620dae96ae060c1b5e6e70 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 19 May 2023 18:13:29 +0000
Subject: [PATCH 663/689] limit `max_tries` value by `max_error_cap` to avoid
 unlimited number of retries

---
 src/Common/PoolWithFailoverBase.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h
index 928d8b74a3f..646e10d6443 100644
--- a/src/Common/PoolWithFailoverBase.h
+++ b/src/Common/PoolWithFailoverBase.h
@@ -227,6 +227,10 @@ PoolWithFailoverBase<TNestedPool>::getMany(
 {
     std::vector<ShuffledPool> shuffled_pools = getShuffledPools(max_ignored_errors, get_priority);
 
+    /// Limit `max_tries` value by `max_error_cap` to avoid unlimited number of retries
+    if (max_tries > max_error_cap)
+        max_tries = max_error_cap;
+
     /// We will try to get a connection from each pool until a connection is produced or max_tries is reached.
     std::vector<TryResult> try_results(shuffled_pools.size());
     size_t entries_count = 0;

From 50919e0cc611f09fa7444eeb14845a2b451305d6 Mon Sep 17 00:00:00 2001
From: Dan Roscigno <dan@roscigno.com>
Date: Fri, 19 May 2023 14:24:50 -0400
Subject: [PATCH 664/689] Update src/Core/Settings.h

---
 src/Core/Settings.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index f1941c20baf..610c7135a75 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -156,7 +156,7 @@ class IColumn;
     M(ParallelReplicasCustomKeyFilterType, parallel_replicas_custom_key_filter_type, ParallelReplicasCustomKeyFilterType::DEFAULT, "Type of filter to use with custom key for parallel replicas. default - use modulo operation on the custom key, range - use range filter on custom key using all possible values for the value type of custom key.", 0) \
     \
     M(String, cluster_for_parallel_replicas, "default", "Cluster for a shard in which current server is located", 0) \
-    M(Bool, allow_experimental_parallel_reading_from_replicas, false, "If true, ClickHouse will send a SELECT query to all replicas of a table (up to `max_parallel_replicas`). It will work for any kind of MergeTree table.", 0) \
+    M(Bool, allow_experimental_parallel_reading_from_replicas, false, "If true, ClickHouse will send a SELECT query to all replicas of a table. It will work for any kind on MergeTree table.", 0) \
     M(Float, parallel_replicas_single_task_marks_count_multiplier, 2, "A multiplier which will be added during calculation for minimal number of marks to retrieve from coordinator. This will be applied only for remote replicas.", 0) \
     M(Bool, parallel_replicas_for_non_replicated_merge_tree, false, "If true, ClickHouse will use parallel replicas algorithm also for non-replicated MergeTree tables", 0) \
     \

From 1ded6d64097dccf0585e814f2c3d522d59912b73 Mon Sep 17 00:00:00 2001
From: Dan Roscigno <dan@roscigno.com>
Date: Fri, 19 May 2023 16:53:26 -0400
Subject: [PATCH 665/689] Update grant.md

---
 docs/en/sql-reference/statements/grant.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md
index b6d6f285f3d..e6073f3523a 100644
--- a/docs/en/sql-reference/statements/grant.md
+++ b/docs/en/sql-reference/statements/grant.md
@@ -9,7 +9,7 @@ sidebar_label: GRANT
 - Grants [privileges](#grant-privileges) to ClickHouse user accounts or roles.
 - Assigns roles to user accounts or to the other roles.
 
-To revoke privileges, use the [REVOKE](../../sql-reference/statements/revoke.md) statement. Also you can list granted privileges with the [SHOW GRANTS](../../sql-reference/statements/show.md#show-grants-statement) statement.
+To revoke privileges, use the [REVOKE](../../sql-reference/statements/revoke.md) statement. Also you can list granted privileges with the [SHOW GRANTS](../../sql-reference/statements/show.md#show-grants) statement.
 
 ## Granting Privilege Syntax
 

From a9e834aea4bdea39748dd521306f534d9fe59c35 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 20 May 2023 16:04:15 +0200
Subject: [PATCH 666/689] Fix 02735_system_zookeeper_connection for
 DatabaseReplicated

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/queries/0_stateless/02735_system_zookeeper_connection.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02735_system_zookeeper_connection.sql b/tests/queries/0_stateless/02735_system_zookeeper_connection.sql
index 10f12177b2e..b409913d183 100644
--- a/tests/queries/0_stateless/02735_system_zookeeper_connection.sql
+++ b/tests/queries/0_stateless/02735_system_zookeeper_connection.sql
@@ -1,4 +1,4 @@
--- Tags: no-fasttest
+-- Tags: no-fasttest, no-replicated-database
 
 DROP TABLE IF EXISTS test_zk_connection_table;
 

From 2444e93ad73d81ef45ca0f36e2a9478c0483d719 Mon Sep 17 00:00:00 2001
From: Yusuke Tanaka <yusuktan@maguro.dev>
Date: Sun, 21 May 2023 14:42:17 +0900
Subject: [PATCH 667/689] Fix broken link to async_insert_deduplicate in doc

---
 docs/en/operations/settings/settings.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index e150060befe..791186dd9c1 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -1714,7 +1714,7 @@ Default value: `100000`.
 
 ### async_insert_max_query_number {#async-insert-max-query-number}
 
-The maximum number of insert queries per block before being inserted. This setting takes effect only if [async_insert_deduplicate](#settings-async-insert-deduplicate) is enabled.
+The maximum number of insert queries per block before being inserted. This setting takes effect only if [async_insert_deduplicate](#async-insert-deduplicate) is enabled.
 
 Possible values:
 
@@ -1745,7 +1745,7 @@ Possible values:
 
 Default value: `0`.
 
-### async_insert_deduplicate {#settings-async-insert-deduplicate}
+### async_insert_deduplicate {#async-insert-deduplicate}
 
 Enables or disables insert deduplication of `ASYNC INSERT` (for Replicated\* tables).
 

From 312f751503a12fc9612f071f89a48266dfb42c37 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 21 May 2023 13:08:55 +0000
Subject: [PATCH 668/689] Uppercase remaining SQL keywords

---
 docs/en/sql-reference/dictionaries/index.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index eb45247e74a..4e9bb2936db 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -2206,7 +2206,7 @@ Regular expression tree dictionaries are a special type of dictionary which repr
 Regular expression tree dictionaries are defined in ClickHouse open-source using the YAMLRegExpTree source which is provided the path to a YAML file containing the regular expression tree.
 
 ```sql
-create dictionary regexp_dict
+CREATE DICTIONARY regexp_dict
 (
     regexp String,
     name String,
@@ -2322,7 +2322,7 @@ clickhouse client \
 You can see how to [Insert Local Files](https://clickhouse.com/docs/en/integrations/data-ingestion/insert-local-files) for more details. After we initialize the source table, we can create a RegexpTree by table source:
 
 ``` sql
-create dictionary regexp_dict
+CREATE DICTIONARY regexp_dict
 (
     regexp String,
     name String,

From 9d9d4e3d62ea042952bf808143898529dd86822d Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 21 May 2023 13:33:03 +0000
Subject: [PATCH 669/689] Some fixups

---
 docs/en/sql-reference/dictionaries/index.md | 37 ++++++++++-----------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index 4e9bb2936db..522fe132a66 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -2201,7 +2201,7 @@ Result:
 
 Regular expression tree dictionaries are a special type of dictionary which represent the mapping from key to attributes using a tree of regular expressions. There are some use cases, e.g. parsing of (user agent)[https://en.wikipedia.org/wiki/User_agent] strings, which can be expressed elegantly with regexp tree dictionaries.
 
-### Use Regular Expression Tree Dictionary in ClickHouse Open-Source Environment
+### Use Regular Expression Tree Dictionary in ClickHouse Open-Source
 
 Regular expression tree dictionaries are defined in ClickHouse open-source using the YAMLRegExpTree source which is provided the path to a YAML file containing the regular expression tree.
 
@@ -2238,15 +2238,14 @@ The dictionary source `YAMLRegExpTree` represents the structure of a regexp tree
       version: '10'
 ```
 
-This config consists of a list of Regular Expression Tree nodes. Each node has following structure:
+This config consists of a list of regular expression tree nodes. Each node has the following structure:
 
-- **regexp** means the regular expression of this node.
-- **user defined attributes** is a list of dictionary attributes defined in the dictionary structure. In this case, we have two attributes: `name` and `version`. The first node has both attributes. The second node only has `name` attribute, because the `version` is defined in the children nodes.
-  - The value of an attribute could contain a **back reference** which refers to a capture group of the matched regular expression. Reference number ranges from 1 to 9 and writes as `$1` or `\1`. During the query execution, the back reference in the value will be replaced by the matched capture group.
-- **children nodes** is the children of a regexp tree node, which has their own attributes and children nodes. String matching preceeds in a depth-first fasion. If a string matches any regexp node in the top layer, the dictionary checks if the string matches the children nodes of it. If it matches, we assign the attributes of the matching nodes. If two or more nodes define the same attribute, children nodes have more priority.
-  - the name of **children nodes** in YAML files can be arbitrary.
+- **regexp**: the regular expression of the node.
+- **attributes**: a list of user-defined dictionary attributes. In this example, there are two attributes: `name` and `version`. The first node defines both attributes. The second node only defines attribute `name`. Attribute `version` is provided by the child nodes of the second node.
+  - The value of an attribute may contain **back references**, referring to capture groups of the matched regular expression. In the example, the value of attribute `version` in the first node consists of a back-reference `\1` to capture group `(\d+[\.\d]*)` in the regular expression. Back-reference numbers range from 1 to 9 and are written as `$1` or `\1` (for number 1). The back reference is replaced by the matched capture group during query execution.
+- **child nodes**: a list of children of a regexp tree node, each of which has its own attributes and (potentially) children nodes. String matching proceeds in a depth-first fashion. If a string matches a regexp node, the dictionary checks if it also matches the nodes' child nodes. If that is the case, the attributes of the deepest matching node are assigned. Attributes of a child node overwrite equally named attributes of parent nodes. The name of child nodes in YAML files can be arbitrary, e.g. `versions` in above example.
 
-Due to the specialty of regexp tree dictionary, we only allow functions `dictGet`, `dictGetOrDefault` and `dictGetOrNull`.
+Regexp tree dictionaries only allow access using functions `dictGet`, `dictGetOrDefault` and `dictGetOrNull`.
 
 Example:
 
@@ -2262,16 +2261,16 @@ Result:
 └─────────────────────────────────────────────────────────────────┘
 ```
 
-In this case, we match the regular expression `\d+/tclwebkit(?:\d+[\.\d]*)` in the top layer's second node, so the dictionary continues to look into the children nodes and find it matches `3[12]/tclwebkit`. As a result, the value of `name` is `Android` defined in the first layer and the value of `version` is `12` defined the child node.
+In this case, we first match the regular expression `\d+/tclwebkit(?:\d+[\.\d]*)` in the top layer's second node. The dictionary then continues to look into the child nodes and find that the string also matches `3[12]/tclwebkit`. As a result, the value of attribute `name` is `Android` (defined in the first layer) and the value of `version` is `12` (defined the child node).
 
-With a powerful YAML configure file, we can use RepexpTree dictionary as a UA parser. We support [uap-core](https://github.com/ua-parser/uap-core) and demonstrate how to use it in the functional test [02504_regexp_dictionary_ua_parser](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh)
+With a powerful YAML configure file, we can use a regexp tree dictionaries as a user agent string parser. We support [uap-core](https://github.com/ua-parser/uap-core) and demonstrate how to use it in the functional test [02504_regexp_dictionary_ua_parser](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh)
 
 ### Use Regular Expression Tree Dictionary in ClickHouse Cloud
 
-We have shown how Regular Expression Tree work in the local environment, but we cannot use `YAMLRegExpTree` in the cloud. If we have a local YAML file, we can use this file to create Regular Expression Tree Dictionary in the local environment, then dump this dictionary to a csv file by the `dictionary` table function and [INTO OUTFILE](../statements/select/into-outfile.md) clause.
+Above used `YAMLRegExpTree` source works in ClickHouse Open Source but not in ClickHouse Cloud. To use regexp tree dictionaries in ClickHouse could, first create a regexp tree dictionary from a YAML file locally in ClickHouse Open Source, then dump this dictionary into a CSV file using the `dictionary` table function and the [INTO OUTFILE](../statements/select/into-outfile.md) clause.
 
 ```sql
-select * from dictionary(regexp_dict) into outfile('regexp_dict.csv')
+SELECT * FROM dictionary(regexp_dict) INTO OUTFILE('regexp_dict.csv')
 ```
 
 The content of csv file is:
@@ -2285,15 +2284,15 @@ The content of csv file is:
 6,2,"3[12]/tclwebkit","['version']","['10']"
 ```
 
-The schema of dumped file is always
+The schema of dumped file is:
 
-- `id UInt64` represents the id of the RegexpTree node.
-- `parent_id UInt64` represents the id of the parent of a node.
-- `regexp String` represents the regular expression string.
-- `keys Array(String)` represents the names of user defined attributes.
-- `values Array(String)` represents the values of user defined attributes.
+- `id UInt64`: the id of the RegexpTree node.
+- `parent_id UInt64`: the id of the parent of a node.
+- `regexp String`: the regular expression string.
+- `keys Array(String)`: the names of user-defined attributes.
+- `values Array(String)`: the values of user-defined attributes.
 
-On the cloud, we can create a table `regexp_dictionary_source_table` with the above table structure.
+To create the dictionary in ClickHouse Cloud, first create a table `regexp_dictionary_source_table` with below table structure:
 
 ```sql
 CREATE TABLE regexp_dictionary_source_table

From 491cf8b6e199757b35cef5273ea8c3cea76879b9 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 21 May 2023 13:43:05 +0000
Subject: [PATCH 670/689] Fix minor mistakes

---
 docs/en/sql-reference/dictionaries/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index 522fe132a66..f593cbe9052 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -2261,7 +2261,7 @@ Result:
 └─────────────────────────────────────────────────────────────────┘
 ```
 
-In this case, we first match the regular expression `\d+/tclwebkit(?:\d+[\.\d]*)` in the top layer's second node. The dictionary then continues to look into the child nodes and find that the string also matches `3[12]/tclwebkit`. As a result, the value of attribute `name` is `Android` (defined in the first layer) and the value of `version` is `12` (defined the child node).
+In this case, we first match the regular expression `\d+/tclwebkit(?:\d+[\.\d]*)` in the top layer's second node. The dictionary then continues to look into the child nodes and finds that the string also matches `3[12]/tclwebkit`. As a result, the value of attribute `name` is `Android` (defined in the first layer) and the value of attribute `version` is `12` (defined the child node).
 
 With a powerful YAML configure file, we can use a regexp tree dictionaries as a user agent string parser. We support [uap-core](https://github.com/ua-parser/uap-core) and demonstrate how to use it in the functional test [02504_regexp_dictionary_ua_parser](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh)
 

From b6cc504717dacf968dd8bff0d9f76e824fd3a246 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 20 May 2023 09:22:00 +0200
Subject: [PATCH 671/689] Remove Common/OpenTelemetryTraceContext.h from
 Context.h

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Interpreters/Context.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h
index 15f2ff625ef..1be662e0958 100644
--- a/src/Interpreters/Context.h
+++ b/src/Interpreters/Context.h
@@ -3,7 +3,6 @@
 #include <base/types.h>
 #include <Common/isLocalAddress.h>
 #include <Common/MultiVersion.h>
-#include <Common/OpenTelemetryTraceContext.h>
 #include <Common/RemoteHostFilter.h>
 #include <Common/ThreadPool_fwd.h>
 #include <Common/Throttler_fwd.h>

From 66cf16410daf872b241c5b9f65e2e95eac9f85a5 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Fri, 19 May 2023 10:37:22 +0200
Subject: [PATCH 672/689] Preserve initial_query_id for ON CLUSTER queries

v2: add proper escaping
v3: set distributed_ddl_output_mode=none for test to fix replicated database build
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Interpreters/DDLTask.cpp                  | 16 +++++++++++++
 src/Interpreters/DDLTask.h                    |  4 +++-
 src/Interpreters/DDLWorker.cpp                |  2 ++
 src/Interpreters/executeDDLQueryOnCluster.cpp |  1 +
 .../02761_ddl_initial_query_id.reference      |  5 ++++
 .../0_stateless/02761_ddl_initial_query_id.sh | 24 +++++++++++++++++++
 6 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/02761_ddl_initial_query_id.reference
 create mode 100755 tests/queries/0_stateless/02761_ddl_initial_query_id.sh

diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp
index 799f1b0b4f4..b24856a6146 100644
--- a/src/Interpreters/DDLTask.cpp
+++ b/src/Interpreters/DDLTask.cpp
@@ -104,6 +104,14 @@ String DDLLogEntry::toString() const
 
     if (version >= OPENTELEMETRY_ENABLED_VERSION)
         wb << "tracing: " << this->tracing_context;
+    /// NOTE: OPENTELEMETRY_ENABLED_VERSION has new line in TracingContext::serialize(), so no need to add one more
+
+    if (version >= PRESERVE_INITIAL_QUERY_ID_VERSION)
+    {
+        writeString("initial_query_id: ", wb);
+        writeEscapedString(initial_query_id, wb);
+        writeChar('\n', wb);
+    }
 
     return wb.str();
 }
@@ -150,6 +158,14 @@ void DDLLogEntry::parse(const String & data)
             rb >> "tracing: " >> this->tracing_context;
     }
 
+    if (version >= PRESERVE_INITIAL_QUERY_ID_VERSION)
+    {
+        checkString("initial_query_id: ", rb);
+        readEscapedString(initial_query_id, rb);
+        checkChar('\n', rb);
+    }
+
+
     assertEOF(rb);
 
     if (!host_id_strings.empty())
diff --git a/src/Interpreters/DDLTask.h b/src/Interpreters/DDLTask.h
index 2043de6701e..e92b1f9a885 100644
--- a/src/Interpreters/DDLTask.h
+++ b/src/Interpreters/DDLTask.h
@@ -71,10 +71,11 @@ struct DDLLogEntry
     static constexpr const UInt64 SETTINGS_IN_ZK_VERSION = 2;
     static constexpr const UInt64 NORMALIZE_CREATE_ON_INITIATOR_VERSION = 3;
     static constexpr const UInt64 OPENTELEMETRY_ENABLED_VERSION = 4;
+    static constexpr const UInt64 PRESERVE_INITIAL_QUERY_ID_VERSION = 5;
     /// Add new version here
 
     /// Remember to update the value below once new version is added
-    static constexpr const UInt64 DDL_ENTRY_FORMAT_MAX_VERSION = 4;
+    static constexpr const UInt64 DDL_ENTRY_FORMAT_MAX_VERSION = 5;
 
     UInt64 version = 1;
     String query;
@@ -82,6 +83,7 @@ struct DDLLogEntry
     String initiator; // optional
     std::optional<SettingsChanges> settings;
     OpenTelemetry::TracingContext tracing_context;
+    String initial_query_id;
 
     void setSettingsIfRequired(ContextPtr context);
     String toString() const;
diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp
index 7c021bd82e6..81c78000ac3 100644
--- a/src/Interpreters/DDLWorker.cpp
+++ b/src/Interpreters/DDLWorker.cpp
@@ -476,6 +476,8 @@ bool DDLWorker::tryExecuteQuery(DDLTaskBase & task, const ZooKeeperPtr & zookeep
             query_context->setSetting("implicit_transaction", Field{0});
         }
 
+        query_context->getClientInfo().initial_query_id = task.entry.initial_query_id;
+
         if (!task.is_initial_query)
             query_scope.emplace(query_context);
         executeQuery(istr, ostr, !task.is_initial_query, query_context, {});
diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp
index e4d880c1e0c..04c1d545207 100644
--- a/src/Interpreters/executeDDLQueryOnCluster.cpp
+++ b/src/Interpreters/executeDDLQueryOnCluster.cpp
@@ -184,6 +184,7 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, ContextPtr context,
     entry.initiator = ddl_worker.getCommonHostID();
     entry.setSettingsIfRequired(context);
     entry.tracing_context = OpenTelemetry::CurrentContext();
+    entry.initial_query_id = context->getClientInfo().initial_query_id;
     String node_path = ddl_worker.enqueueQuery(entry);
 
     return getDistributedDDLStatus(node_path, entry, context, /* hosts_to_wait */ nullptr);
diff --git a/tests/queries/0_stateless/02761_ddl_initial_query_id.reference b/tests/queries/0_stateless/02761_ddl_initial_query_id.reference
new file mode 100644
index 00000000000..5c6f448eed5
--- /dev/null
+++ b/tests/queries/0_stateless/02761_ddl_initial_query_id.reference
@@ -0,0 +1,5 @@
+default distributed_ddl_entry_format_version
+DROP TABLE IF EXISTS foo ON CLUSTER test_shard_localhost
+distributed_ddl_entry_format_version=PRESERVE_INITIAL_QUERY_ID_VERSION
+DROP TABLE IF EXISTS default.foo
+DROP TABLE IF EXISTS foo ON CLUSTER test_shard_localhost
diff --git a/tests/queries/0_stateless/02761_ddl_initial_query_id.sh b/tests/queries/0_stateless/02761_ddl_initial_query_id.sh
new file mode 100755
index 00000000000..e9a315b812b
--- /dev/null
+++ b/tests/queries/0_stateless/02761_ddl_initial_query_id.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+echo "default distributed_ddl_entry_format_version"
+query_id="$(random_str 10)"
+$CLICKHOUSE_CLIENT --query_id "$query_id" --distributed_ddl_output_mode=none -q "DROP TABLE IF EXISTS foo ON CLUSTER test_shard_localhost"
+$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS"
+$CLICKHOUSE_CLIENT -q "SELECT query FROM system.query_log WHERE initial_query_id = '$query_id' AND type != 'QueryStart'"
+
+echo "distributed_ddl_entry_format_version=PRESERVE_INITIAL_QUERY_ID_VERSION"
+PRESERVE_INITIAL_QUERY_ID_VERSION=5
+query_id="$(random_str 10)"
+# Check that serialization will not be broken with new lines in initial_query_id
+query_id+=$'\nfoo'
+$CLICKHOUSE_CLIENT --distributed_ddl_entry_format_version=$PRESERVE_INITIAL_QUERY_ID_VERSION --query_id "$query_id" --distributed_ddl_output_mode=none -q "DROP TABLE IF EXISTS foo ON CLUSTER test_shard_localhost"
+$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS"
+# - normalizeQuery() is required to strip out DDL comment
+# - replace() is required to avoid non deterministic behaviour of
+#   normalizeQuery() that replaces the identifier with "?" only if it has more
+#   then two numbers.
+$CLICKHOUSE_CLIENT -q "SELECT normalizeQuery(replace(query, currentDatabase(), 'default')) FROM system.query_log WHERE initial_query_id = '$query_id' AND type != 'QueryStart' ORDER BY event_time_microseconds"

From ef06bb8f1409961d789d071703dfe1d1b4aef871 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 21 May 2023 23:07:03 +0200
Subject: [PATCH 673/689] Fix crashing in case of Replicated database without
 arguments

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Interpreters/InterpreterCreateQuery.cpp                   | 2 +-
 .../0_stateless/02762_replicated_database_no_args.reference   | 0
 .../queries/0_stateless/02762_replicated_database_no_args.sql | 4 ++++
 3 files changed, 5 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/02762_replicated_database_no_args.reference
 create mode 100644 tests/queries/0_stateless/02762_replicated_database_no_args.sql

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 3e76ad23c30..ab9e1fb04d6 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -227,7 +227,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
         metadata_path = metadata_path / "metadata" / database_name_escaped;
     }
 
-    if (create.storage->engine->name == "Replicated" && !internal && !create.attach)
+    if (create.storage->engine->name == "Replicated" && !internal && !create.attach && create.storage->engine->arguments)
     {
         /// Fill in default parameters
         if (create.storage->engine->arguments->children.size() == 1)
diff --git a/tests/queries/0_stateless/02762_replicated_database_no_args.reference b/tests/queries/0_stateless/02762_replicated_database_no_args.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/02762_replicated_database_no_args.sql b/tests/queries/0_stateless/02762_replicated_database_no_args.sql
new file mode 100644
index 00000000000..1409a059b02
--- /dev/null
+++ b/tests/queries/0_stateless/02762_replicated_database_no_args.sql
@@ -0,0 +1,4 @@
+-- Tags: no-parallel
+
+set allow_experimental_database_replicated=1;
+create database replicated_db_no_args engine=Replicated; -- { serverError BAD_ARGUMENTS }

From 39ff030a6e9a646a90f0616c63b9b8064588436e Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Thu, 4 May 2023 10:17:04 +0800
Subject: [PATCH 674/689] grace hash join supports right/full join

---
 src/Interpreters/GraceHashJoin.cpp            |   79 +-
 src/Interpreters/GraceHashJoin.h              |    3 +-
 src/Interpreters/JoinUtils.cpp                |    2 +-
 .../Transforms/JoiningTransform.cpp           |   87 +-
 src/Processors/Transforms/JoiningTransform.h  |    6 +-
 .../02724_grace_hash_join.reference           | 1804 +++++++++++++++++
 .../0_stateless/02724_grace_hash_join.sql     |   44 +
 7 files changed, 1965 insertions(+), 60 deletions(-)
 create mode 100644 tests/queries/0_stateless/02724_grace_hash_join.reference
 create mode 100644 tests/queries/0_stateless/02724_grace_hash_join.sql

diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index 7795061072c..32801887f2b 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -302,10 +302,12 @@ void GraceHashJoin::initBuckets()
     current_bucket->startJoining();
 }
 
-bool GraceHashJoin::isSupported(const std::shared_ptr<TableJoin> & table_join)
+bool GraceHashJoin::isSupported(const std::shared_ptr<TableJoin> & table_join [[maybe_unused]])
 {
+
     bool is_asof = (table_join->strictness() == JoinStrictness::Asof);
-    return !is_asof && isInnerOrLeft(table_join->kind()) && table_join->oneDisjunct();
+    auto kind = table_join->kind();
+    return !is_asof && (isInner(kind) || isLeft(kind) || isRight(kind) || isFull(kind)) && table_join->oneDisjunct();
 }
 
 GraceHashJoin::~GraceHashJoin() = default;
@@ -325,7 +327,6 @@ bool GraceHashJoin::hasMemoryOverflow(size_t total_rows, size_t total_bytes) con
     /// One row can't be split, avoid loop
     if (total_rows < 2)
         return false;
-
     bool has_overflow = !table_join->sizeLimits().softCheck(total_rows, total_bytes);
 
     if (has_overflow)
@@ -471,23 +472,33 @@ bool GraceHashJoin::alwaysReturnsEmptySet() const
     return hash_join_is_empty;
 }
 
-IBlocksStreamPtr GraceHashJoin::getNonJoinedBlocks(const Block &, const Block &, UInt64) const
+IBlocksStreamPtr GraceHashJoin::getNonJoinedBlocks(
+    const Block & left_sample_block_ [[maybe_unused]], const Block & result_sample_block_ [[maybe_unused]], UInt64 max_block_size_ [[maybe_unused]]) const
 {
-    /// We do no support returning non joined blocks here.
-    /// TODO: They _should_ be reported by getDelayedBlocks instead
-    return nullptr;
+    return hash_join->getNonJoinedBlocks(left_sample_block_, result_sample_block_, max_block_size_);
 }
 
 class GraceHashJoin::DelayedBlocks : public IBlocksStream
 {
 public:
-    explicit DelayedBlocks(size_t current_bucket_, Buckets buckets_, InMemoryJoinPtr hash_join_, const Names & left_key_names_, const Names & right_key_names_)
+    explicit DelayedBlocks(
+        size_t current_bucket_,
+        Buckets buckets_,
+        InMemoryJoinPtr hash_join_,
+        const Names & left_key_names_,
+        const Names & right_key_names_,
+        const Block & left_sample_block_,
+        const Block & result_sample_block_,
+        size_t max_block_size_)
         : current_bucket(current_bucket_)
         , buckets(std::move(buckets_))
         , hash_join(std::move(hash_join_))
         , left_reader(buckets[current_bucket]->getLeftTableReader())
         , left_key_names(left_key_names_)
         , right_key_names(right_key_names_)
+        , left_sample_block(left_sample_block_)
+        , result_sample_block(result_sample_block_)
+        , max_block_size(max_block_size_)
     {
     }
 
@@ -499,10 +510,27 @@ public:
 
         do
         {
-            block = left_reader.read();
-            if (!block)
+            if (!is_left_reader_finished)
             {
-                return {};
+                block = left_reader.read();
+                if (!block)
+                {
+                    is_left_reader_finished = true;
+                    non_joined_blocks = hash_join->getNonJoinedBlocks(left_sample_block, result_sample_block, max_block_size);
+                }
+            }
+            if (is_left_reader_finished)
+            {
+                if (non_joined_blocks)
+                {
+                    block = non_joined_blocks->next();
+                    return block;
+                }
+                else
+                {
+                    // left/inner join.
+                    return {};
+                }
             }
 
             Blocks blocks = JoinCommon::scatterBlockByHash(left_key_names, block, num_buckets);
@@ -544,6 +572,11 @@ public:
 
     Names left_key_names;
     Names right_key_names;
+    Block left_sample_block;
+    Block result_sample_block;
+    bool is_left_reader_finished = false;
+    IBlocksStreamPtr non_joined_blocks;
+    size_t max_block_size;
 };
 
 IBlocksStreamPtr GraceHashJoin::getDelayedBlocks()
@@ -555,16 +588,6 @@ IBlocksStreamPtr GraceHashJoin::getDelayedBlocks()
 
     size_t bucket_idx = current_bucket->idx;
 
-    if (hash_join)
-    {
-        auto right_blocks = hash_join->releaseJoinedBlocks(/* restructure */ false);
-        for (auto & block : right_blocks)
-        {
-            Blocks blocks = JoinCommon::scatterBlockByHash(right_key_names, block, buckets.size());
-            flushBlocksToBuckets<JoinTableSide::Right>(blocks, buckets, bucket_idx);
-        }
-    }
-
     hash_join = makeInMemoryJoin();
 
     for (bucket_idx = bucket_idx + 1; bucket_idx < buckets.size(); ++bucket_idx)
@@ -589,8 +612,18 @@ IBlocksStreamPtr GraceHashJoin::getDelayedBlocks()
 
         LOG_TRACE(log, "Loaded bucket {} with {}(/{}) rows",
             bucket_idx, hash_join->getTotalRowCount(), num_rows);
-
-        return std::make_unique<DelayedBlocks>(current_bucket->idx, buckets, hash_join, left_key_names, right_key_names);
+        auto result_sample_block = left_sample_block;
+        ExtraBlockPtr tmp;
+        hash_join->joinBlock(result_sample_block, tmp);
+        return std::make_unique<DelayedBlocks>(
+            current_bucket->idx,
+            buckets,
+            hash_join,
+            left_key_names,
+            right_key_names,
+            left_sample_block,
+            result_sample_block,
+            max_block_size);
     }
 
     LOG_TRACE(log, "Finished loading all {} buckets", buckets.size());
diff --git a/src/Interpreters/GraceHashJoin.h b/src/Interpreters/GraceHashJoin.h
index b8d83f4cad0..eb39ee09208 100644
--- a/src/Interpreters/GraceHashJoin.h
+++ b/src/Interpreters/GraceHashJoin.h
@@ -13,7 +13,6 @@
 
 namespace DB
 {
-
 class TableJoin;
 class HashJoin;
 
@@ -79,7 +78,7 @@ public:
     bool supportTotals() const override { return false; }
 
     IBlocksStreamPtr
-    getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const override;
+    getNonJoinedBlocks(const Block & left_sample_block_, const Block & result_sample_block_, UInt64 max_block_size) const override;
 
     /// Open iterator over joined blocks.
     /// Must be called after all @joinBlock calls.
diff --git a/src/Interpreters/JoinUtils.cpp b/src/Interpreters/JoinUtils.cpp
index a05b58e14a1..59ee86ffa18 100644
--- a/src/Interpreters/JoinUtils.cpp
+++ b/src/Interpreters/JoinUtils.cpp
@@ -19,6 +19,7 @@
 
 #include <base/FnTraits.h>
 
+
 namespace DB
 {
 
@@ -864,7 +865,6 @@ Block NotJoinedBlocks::nextImpl()
         size_t rows_added = filler->fillColumns(columns_right);
         if (rows_added == 0)
             return {};
-
         addLeftColumns(result_block, rows_added);
         addRightColumns(result_block, columns_right);
     }
diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp
index bba8ec6fa16..4789e9c514a 100644
--- a/src/Processors/Transforms/JoiningTransform.cpp
+++ b/src/Processors/Transforms/JoiningTransform.cpp
@@ -350,6 +350,11 @@ IProcessor::Status DelayedJoinedBlocksWorkerTransform::prepare()
 
     if (!task)
     {
+        if (input.isFinished())
+        {
+            output.finish();
+            return Status::Finished;
+        }
         if (!input.hasData())
         {
             input.setNeeded();
@@ -362,10 +367,14 @@ IProcessor::Status DelayedJoinedBlocksWorkerTransform::prepare()
             output.pushException(data.exception);
             return Status::Finished;
         }
-
-        if (!data.chunk.hasChunkInfo())
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "DelayedJoinedBlocksWorkerTransform must have chunk info");
-        task = std::dynamic_pointer_cast<const DelayedBlocksTask>(data.chunk.getChunkInfo());
+        if (data.chunk.hasChunkInfo())
+            task = std::dynamic_pointer_cast<const DelayedBlocksTask>(data.chunk.getChunkInfo());
+        else
+        {
+            // Try to get one task from DelayedJoinedBlocksTransform again.
+            task = nullptr;
+            return Status::NeedData;
+        }
     }
     else
     {
@@ -386,7 +395,6 @@ void DelayedJoinedBlocksWorkerTransform::work()
 {
     if (!task)
         return;
-
     Block block = task->delayed_blocks->next();
 
     if (!block)
@@ -408,15 +416,19 @@ DelayedJoinedBlocksTransform::DelayedJoinedBlocksTransform(size_t num_streams, J
 
 void DelayedJoinedBlocksTransform::work()
 {
-    if (finished)
+    if (all_buckets_finished)
         return;
-
     delayed_blocks = join->getDelayedBlocks();
-    finished = finished || delayed_blocks == nullptr;
+    if (!delayed_blocks)
+    {
+        all_buckets_finished = true;
+    }
 }
 
 IProcessor::Status DelayedJoinedBlocksTransform::prepare()
 {
+
+    bool should_finished = false;
     for (auto & output : outputs)
     {
         if (output.isFinished())
@@ -424,41 +436,56 @@ IProcessor::Status DelayedJoinedBlocksTransform::prepare()
             /// If at least one output is finished, then we have read all data from buckets.
             /// Some workers can still be busy with joining the last chunk of data in memory,
             /// but after that they also will finish when they will try to get next chunk.
-            finished = true;
-            continue;
+            should_finished = true;
+            break;
         }
-        if (!output.canPush())
-            return Status::PortFull;
     }
-
-    if (finished)
+    if (should_finished)
     {
         for (auto & output : outputs)
         {
-            if (output.isFinished())
-                continue;
-            Chunk chunk;
-            chunk.setChunkInfo(std::make_shared<DelayedBlocksTask>());
-            output.push(std::move(chunk));
-            output.finish();
+            if (!output.isFinished())
+            {
+                output.finish();
+            }
         }
-
         return Status::Finished;
     }
 
-    if (delayed_blocks)
+    // No pending buckets.
+    if (all_buckets_finished && !delayed_blocks)
     {
         for (auto & output : outputs)
-        {
-            Chunk chunk;
-            chunk.setChunkInfo(std::make_shared<DelayedBlocksTask>(delayed_blocks));
-            output.push(std::move(chunk));
-        }
-        delayed_blocks = nullptr;
-        return Status::PortFull;
+            output.finish();
+        return Status::Finished;
     }
 
-    return Status::Ready;
+    // Try to read next bucket.
+    if (!delayed_blocks)
+        return Status::Ready;
+
+    // Put the ready task to the first available output port.
+    // Put empty tasks to other output ports. these ports will do nothing but require data again.
+    // Each bucket will handled in one thread, avoid lock contention in left file reader.
+    for (auto & output : outputs)
+    {
+        if (output.canPush())
+        {
+            if (delayed_blocks)
+            {
+                Chunk chunk;
+                chunk.setChunkInfo(std::make_shared<DelayedBlocksTask>(delayed_blocks));
+                output.push(std::move(chunk));
+                delayed_blocks = nullptr;
+            }
+            else
+            {
+                Chunk chunk;
+                output.push(std::move(chunk));
+            }
+        }
+    }
+    return Status::PortFull;
 }
 
 }
diff --git a/src/Processors/Transforms/JoiningTransform.h b/src/Processors/Transforms/JoiningTransform.h
index e7edff40c56..f189f53c004 100644
--- a/src/Processors/Transforms/JoiningTransform.h
+++ b/src/Processors/Transforms/JoiningTransform.h
@@ -125,7 +125,6 @@ public:
 
 using DelayedBlocksTaskPtr = std::shared_ptr<const DelayedBlocksTask>;
 
-
 /// Reads delayed joined blocks from Join
 class DelayedJoinedBlocksTransform : public IProcessor
 {
@@ -139,9 +138,8 @@ public:
 
 private:
     JoinPtr join;
-
-    IBlocksStreamPtr delayed_blocks = nullptr;
-    bool finished = false;
+    IBlocksStreamPtr delayed_blocks;
+    bool all_buckets_finished = false;
 };
 
 class DelayedJoinedBlocksWorkerTransform : public IProcessor
diff --git a/tests/queries/0_stateless/02724_grace_hash_join.reference b/tests/queries/0_stateless/02724_grace_hash_join.reference
new file mode 100644
index 00000000000..296ef623f04
--- /dev/null
+++ b/tests/queries/0_stateless/02724_grace_hash_join.reference
@@ -0,0 +1,1804 @@
+inner join 1
+51	11	21
+52	12	22
+53	13	23
+54	14	24
+55	15	25
+56	16	26
+57	17	27
+58	18	28
+59	19	29
+60	0	0
+61	1	1
+62	2	2
+63	3	3
+64	4	4
+65	5	5
+66	6	6
+67	7	7
+68	8	8
+69	9	9
+70	10	10
+71	11	11
+72	12	12
+73	13	13
+74	14	14
+75	15	15
+76	16	16
+77	17	17
+78	18	18
+79	19	19
+80	0	20
+81	1	21
+82	2	22
+83	3	23
+84	4	24
+85	5	25
+86	6	26
+87	7	27
+88	8	28
+89	9	29
+90	10	0
+91	11	1
+92	12	2
+93	13	3
+94	14	4
+95	15	5
+96	16	6
+97	17	7
+98	18	8
+99	19	9
+100	0	10
+101	1	11
+102	2	12
+103	3	13
+104	4	14
+105	5	15
+106	6	16
+107	7	17
+108	8	18
+109	9	19
+110	10	20
+111	11	21
+112	12	22
+113	13	23
+114	14	24
+115	15	25
+116	16	26
+117	17	27
+118	18	28
+119	19	29
+120	0	0
+121	1	1
+122	2	2
+123	3	3
+124	4	4
+125	5	5
+126	6	6
+127	7	7
+128	8	8
+129	9	9
+130	10	10
+131	11	11
+132	12	12
+133	13	13
+134	14	14
+135	15	15
+136	16	16
+137	17	17
+138	18	18
+139	19	19
+140	0	20
+141	1	21
+142	2	22
+143	3	23
+144	4	24
+145	5	25
+146	6	26
+147	7	27
+148	8	28
+149	9	29
+150	10	0
+151	11	1
+152	12	2
+153	13	3
+154	14	4
+155	15	5
+156	16	6
+157	17	7
+158	18	8
+159	19	9
+160	0	10
+161	1	11
+162	2	12
+163	3	13
+164	4	14
+165	5	15
+166	6	16
+167	7	17
+168	8	18
+169	9	19
+170	10	20
+171	11	21
+172	12	22
+173	13	23
+174	14	24
+175	15	25
+176	16	26
+177	17	27
+178	18	28
+179	19	29
+180	0	0
+181	1	1
+182	2	2
+183	3	3
+184	4	4
+185	5	5
+186	6	6
+187	7	7
+188	8	8
+189	9	9
+190	10	10
+191	11	11
+192	12	12
+193	13	13
+194	14	14
+195	15	15
+196	16	16
+197	17	17
+198	18	18
+199	19	19
+inner join 2
+51	21	11
+52	22	12
+53	23	13
+54	24	14
+55	25	15
+56	26	16
+57	27	17
+58	28	18
+59	29	19
+60	0	0
+61	1	1
+62	2	2
+63	3	3
+64	4	4
+65	5	5
+66	6	6
+67	7	7
+68	8	8
+69	9	9
+70	10	10
+71	11	11
+72	12	12
+73	13	13
+74	14	14
+75	15	15
+76	16	16
+77	17	17
+78	18	18
+79	19	19
+80	20	0
+81	21	1
+82	22	2
+83	23	3
+84	24	4
+85	25	5
+86	26	6
+87	27	7
+88	28	8
+89	29	9
+90	0	10
+91	1	11
+92	2	12
+93	3	13
+94	4	14
+95	5	15
+96	6	16
+97	7	17
+98	8	18
+99	9	19
+100	10	0
+101	11	1
+102	12	2
+103	13	3
+104	14	4
+105	15	5
+106	16	6
+107	17	7
+108	18	8
+109	19	9
+110	20	10
+111	21	11
+112	22	12
+113	23	13
+114	24	14
+115	25	15
+116	26	16
+117	27	17
+118	28	18
+119	29	19
+120	0	0
+121	1	1
+122	2	2
+123	3	3
+124	4	4
+125	5	5
+126	6	6
+127	7	7
+128	8	8
+129	9	9
+130	10	10
+131	11	11
+132	12	12
+133	13	13
+134	14	14
+135	15	15
+136	16	16
+137	17	17
+138	18	18
+139	19	19
+140	20	0
+141	21	1
+142	22	2
+143	23	3
+144	24	4
+145	25	5
+146	26	6
+147	27	7
+148	28	8
+149	29	9
+150	0	10
+151	1	11
+152	2	12
+153	3	13
+154	4	14
+155	5	15
+156	6	16
+157	7	17
+158	8	18
+159	9	19
+160	10	0
+161	11	1
+162	12	2
+163	13	3
+164	14	4
+165	15	5
+166	16	6
+167	17	7
+168	18	8
+169	19	9
+170	20	10
+171	21	11
+172	22	12
+173	23	13
+174	24	14
+175	25	15
+176	26	16
+177	27	17
+178	28	18
+179	29	19
+180	0	0
+181	1	1
+182	2	2
+183	3	3
+184	4	4
+185	5	5
+186	6	6
+187	7	7
+188	8	8
+189	9	9
+190	10	10
+191	11	11
+192	12	12
+193	13	13
+194	14	14
+195	15	15
+196	16	16
+197	17	17
+198	18	18
+199	19	19
+left join 1
+51	11	21
+52	12	22
+53	13	23
+54	14	24
+55	15	25
+56	16	26
+57	17	27
+58	18	28
+59	19	29
+60	0	0
+61	1	1
+62	2	2
+63	3	3
+64	4	4
+65	5	5
+66	6	6
+67	7	7
+68	8	8
+69	9	9
+70	10	10
+71	11	11
+72	12	12
+73	13	13
+74	14	14
+75	15	15
+76	16	16
+77	17	17
+78	18	18
+79	19	19
+80	0	20
+81	1	21
+82	2	22
+83	3	23
+84	4	24
+85	5	25
+86	6	26
+87	7	27
+88	8	28
+89	9	29
+90	10	0
+91	11	1
+92	12	2
+93	13	3
+94	14	4
+95	15	5
+96	16	6
+97	17	7
+98	18	8
+99	19	9
+100	0	10
+101	1	11
+102	2	12
+103	3	13
+104	4	14
+105	5	15
+106	6	16
+107	7	17
+108	8	18
+109	9	19
+110	10	20
+111	11	21
+112	12	22
+113	13	23
+114	14	24
+115	15	25
+116	16	26
+117	17	27
+118	18	28
+119	19	29
+120	0	0
+121	1	1
+122	2	2
+123	3	3
+124	4	4
+125	5	5
+126	6	6
+127	7	7
+128	8	8
+129	9	9
+130	10	10
+131	11	11
+132	12	12
+133	13	13
+134	14	14
+135	15	15
+136	16	16
+137	17	17
+138	18	18
+139	19	19
+140	0	20
+141	1	21
+142	2	22
+143	3	23
+144	4	24
+145	5	25
+146	6	26
+147	7	27
+148	8	28
+149	9	29
+150	10	0
+151	11	1
+152	12	2
+153	13	3
+154	14	4
+155	15	5
+156	16	6
+157	17	7
+158	18	8
+159	19	9
+160	0	10
+161	1	11
+162	2	12
+163	3	13
+164	4	14
+165	5	15
+166	6	16
+167	7	17
+168	8	18
+169	9	19
+170	10	20
+171	11	21
+172	12	22
+173	13	23
+174	14	24
+175	15	25
+176	16	26
+177	17	27
+178	18	28
+179	19	29
+180	0	0
+181	1	1
+182	2	2
+183	3	3
+184	4	4
+185	5	5
+186	6	6
+187	7	7
+188	8	8
+189	9	9
+190	10	10
+191	11	11
+192	12	12
+193	13	13
+194	14	14
+195	15	15
+196	16	16
+197	17	17
+198	18	18
+199	19	19
+left join 2
+0	0	\N
+1	1	\N
+2	2	\N
+3	3	\N
+4	4	\N
+5	5	\N
+6	6	\N
+7	7	\N
+8	8	\N
+9	9	\N
+10	10	\N
+11	11	\N
+12	12	\N
+13	13	\N
+14	14	\N
+15	15	\N
+16	16	\N
+17	17	\N
+18	18	\N
+19	19	\N
+20	20	\N
+21	21	\N
+22	22	\N
+23	23	\N
+24	24	\N
+25	25	\N
+26	26	\N
+27	27	\N
+28	28	\N
+29	29	\N
+30	0	\N
+31	1	\N
+32	2	\N
+33	3	\N
+34	4	\N
+35	5	\N
+36	6	\N
+37	7	\N
+38	8	\N
+39	9	\N
+40	10	\N
+41	11	\N
+42	12	\N
+43	13	\N
+44	14	\N
+45	15	\N
+46	16	\N
+47	17	\N
+48	18	\N
+49	19	\N
+50	20	\N
+51	21	11
+52	22	12
+53	23	13
+54	24	14
+55	25	15
+56	26	16
+57	27	17
+58	28	18
+59	29	19
+60	0	0
+61	1	1
+62	2	2
+63	3	3
+64	4	4
+65	5	5
+66	6	6
+67	7	7
+68	8	8
+69	9	9
+70	10	10
+71	11	11
+72	12	12
+73	13	13
+74	14	14
+75	15	15
+76	16	16
+77	17	17
+78	18	18
+79	19	19
+80	20	0
+81	21	1
+82	22	2
+83	23	3
+84	24	4
+85	25	5
+86	26	6
+87	27	7
+88	28	8
+89	29	9
+90	0	10
+91	1	11
+92	2	12
+93	3	13
+94	4	14
+95	5	15
+96	6	16
+97	7	17
+98	8	18
+99	9	19
+100	10	0
+101	11	1
+102	12	2
+103	13	3
+104	14	4
+105	15	5
+106	16	6
+107	17	7
+108	18	8
+109	19	9
+110	20	10
+111	21	11
+112	22	12
+113	23	13
+114	24	14
+115	25	15
+116	26	16
+117	27	17
+118	28	18
+119	29	19
+120	0	0
+121	1	1
+122	2	2
+123	3	3
+124	4	4
+125	5	5
+126	6	6
+127	7	7
+128	8	8
+129	9	9
+130	10	10
+131	11	11
+132	12	12
+133	13	13
+134	14	14
+135	15	15
+136	16	16
+137	17	17
+138	18	18
+139	19	19
+140	20	0
+141	21	1
+142	22	2
+143	23	3
+144	24	4
+145	25	5
+146	26	6
+147	27	7
+148	28	8
+149	29	9
+150	0	10
+151	1	11
+152	2	12
+153	3	13
+154	4	14
+155	5	15
+156	6	16
+157	7	17
+158	8	18
+159	9	19
+160	10	0
+161	11	1
+162	12	2
+163	13	3
+164	14	4
+165	15	5
+166	16	6
+167	17	7
+168	18	8
+169	19	9
+170	20	10
+171	21	11
+172	22	12
+173	23	13
+174	24	14
+175	25	15
+176	26	16
+177	27	17
+178	28	18
+179	29	19
+180	0	0
+181	1	1
+182	2	2
+183	3	3
+184	4	4
+185	5	5
+186	6	6
+187	7	7
+188	8	8
+189	9	9
+190	10	10
+191	11	11
+192	12	12
+193	13	13
+194	14	14
+195	15	15
+196	16	16
+197	17	17
+198	18	18
+199	19	19
+200	20	\N
+201	21	\N
+202	22	\N
+203	23	\N
+204	24	\N
+205	25	\N
+206	26	\N
+207	27	\N
+208	28	\N
+209	29	\N
+210	0	\N
+211	1	\N
+212	2	\N
+213	3	\N
+214	4	\N
+215	5	\N
+216	6	\N
+217	7	\N
+218	8	\N
+219	9	\N
+220	10	\N
+221	11	\N
+222	12	\N
+223	13	\N
+224	14	\N
+225	15	\N
+226	16	\N
+227	17	\N
+228	18	\N
+229	19	\N
+230	20	\N
+231	21	\N
+232	22	\N
+233	23	\N
+234	24	\N
+235	25	\N
+236	26	\N
+237	27	\N
+238	28	\N
+239	29	\N
+240	0	\N
+241	1	\N
+242	2	\N
+243	3	\N
+244	4	\N
+245	5	\N
+246	6	\N
+247	7	\N
+248	8	\N
+249	9	\N
+250	10	\N
+251	11	\N
+252	12	\N
+253	13	\N
+254	14	\N
+255	15	\N
+256	16	\N
+257	17	\N
+258	18	\N
+259	19	\N
+260	20	\N
+261	21	\N
+262	22	\N
+263	23	\N
+264	24	\N
+265	25	\N
+266	26	\N
+267	27	\N
+268	28	\N
+269	29	\N
+270	0	\N
+271	1	\N
+272	2	\N
+273	3	\N
+274	4	\N
+275	5	\N
+276	6	\N
+277	7	\N
+278	8	\N
+279	9	\N
+280	10	\N
+281	11	\N
+282	12	\N
+283	13	\N
+284	14	\N
+285	15	\N
+286	16	\N
+287	17	\N
+288	18	\N
+289	19	\N
+290	20	\N
+291	21	\N
+292	22	\N
+293	23	\N
+294	24	\N
+295	25	\N
+296	26	\N
+297	27	\N
+298	28	\N
+299	29	\N
+right join 1
+0	\N	0
+0	\N	0
+0	\N	0
+0	\N	0
+0	\N	0
+0	\N	1
+0	\N	1
+0	\N	1
+0	\N	1
+0	\N	1
+0	\N	2
+0	\N	2
+0	\N	2
+0	\N	2
+0	\N	2
+0	\N	3
+0	\N	3
+0	\N	3
+0	\N	3
+0	\N	3
+0	\N	4
+0	\N	4
+0	\N	4
+0	\N	4
+0	\N	4
+0	\N	5
+0	\N	5
+0	\N	5
+0	\N	5
+0	\N	5
+0	\N	6
+0	\N	6
+0	\N	6
+0	\N	6
+0	\N	6
+0	\N	7
+0	\N	7
+0	\N	7
+0	\N	7
+0	\N	7
+0	\N	8
+0	\N	8
+0	\N	8
+0	\N	8
+0	\N	8
+0	\N	9
+0	\N	9
+0	\N	9
+0	\N	9
+0	\N	9
+0	\N	10
+0	\N	10
+0	\N	10
+0	\N	10
+0	\N	10
+0	\N	11
+0	\N	11
+0	\N	11
+0	\N	11
+0	\N	11
+0	\N	12
+0	\N	12
+0	\N	12
+0	\N	12
+0	\N	12
+0	\N	13
+0	\N	13
+0	\N	13
+0	\N	13
+0	\N	13
+0	\N	14
+0	\N	14
+0	\N	14
+0	\N	14
+0	\N	14
+0	\N	15
+0	\N	15
+0	\N	15
+0	\N	15
+0	\N	15
+0	\N	16
+0	\N	16
+0	\N	16
+0	\N	16
+0	\N	16
+0	\N	17
+0	\N	17
+0	\N	17
+0	\N	17
+0	\N	17
+0	\N	18
+0	\N	18
+0	\N	18
+0	\N	18
+0	\N	18
+0	\N	19
+0	\N	19
+0	\N	19
+0	\N	19
+0	\N	19
+0	\N	20
+0	\N	20
+0	\N	20
+0	\N	20
+0	\N	20
+0	\N	20
+0	\N	21
+0	\N	21
+0	\N	21
+0	\N	21
+0	\N	21
+0	\N	22
+0	\N	22
+0	\N	22
+0	\N	22
+0	\N	22
+0	\N	23
+0	\N	23
+0	\N	23
+0	\N	23
+0	\N	23
+0	\N	24
+0	\N	24
+0	\N	24
+0	\N	24
+0	\N	24
+0	\N	25
+0	\N	25
+0	\N	25
+0	\N	25
+0	\N	25
+0	\N	26
+0	\N	26
+0	\N	26
+0	\N	26
+0	\N	26
+0	\N	27
+0	\N	27
+0	\N	27
+0	\N	27
+0	\N	27
+0	\N	28
+0	\N	28
+0	\N	28
+0	\N	28
+0	\N	28
+0	\N	29
+0	\N	29
+0	\N	29
+0	\N	29
+0	\N	29
+51	11	21
+52	12	22
+53	13	23
+54	14	24
+55	15	25
+56	16	26
+57	17	27
+58	18	28
+59	19	29
+60	0	0
+61	1	1
+62	2	2
+63	3	3
+64	4	4
+65	5	5
+66	6	6
+67	7	7
+68	8	8
+69	9	9
+70	10	10
+71	11	11
+72	12	12
+73	13	13
+74	14	14
+75	15	15
+76	16	16
+77	17	17
+78	18	18
+79	19	19
+80	0	20
+81	1	21
+82	2	22
+83	3	23
+84	4	24
+85	5	25
+86	6	26
+87	7	27
+88	8	28
+89	9	29
+90	10	0
+91	11	1
+92	12	2
+93	13	3
+94	14	4
+95	15	5
+96	16	6
+97	17	7
+98	18	8
+99	19	9
+100	0	10
+101	1	11
+102	2	12
+103	3	13
+104	4	14
+105	5	15
+106	6	16
+107	7	17
+108	8	18
+109	9	19
+110	10	20
+111	11	21
+112	12	22
+113	13	23
+114	14	24
+115	15	25
+116	16	26
+117	17	27
+118	18	28
+119	19	29
+120	0	0
+121	1	1
+122	2	2
+123	3	3
+124	4	4
+125	5	5
+126	6	6
+127	7	7
+128	8	8
+129	9	9
+130	10	10
+131	11	11
+132	12	12
+133	13	13
+134	14	14
+135	15	15
+136	16	16
+137	17	17
+138	18	18
+139	19	19
+140	0	20
+141	1	21
+142	2	22
+143	3	23
+144	4	24
+145	5	25
+146	6	26
+147	7	27
+148	8	28
+149	9	29
+150	10	0
+151	11	1
+152	12	2
+153	13	3
+154	14	4
+155	15	5
+156	16	6
+157	17	7
+158	18	8
+159	19	9
+160	0	10
+161	1	11
+162	2	12
+163	3	13
+164	4	14
+165	5	15
+166	6	16
+167	7	17
+168	8	18
+169	9	19
+170	10	20
+171	11	21
+172	12	22
+173	13	23
+174	14	24
+175	15	25
+176	16	26
+177	17	27
+178	18	28
+179	19	29
+180	0	0
+181	1	1
+182	2	2
+183	3	3
+184	4	4
+185	5	5
+186	6	6
+187	7	7
+188	8	8
+189	9	9
+190	10	10
+191	11	11
+192	12	12
+193	13	13
+194	14	14
+195	15	15
+196	16	16
+197	17	17
+198	18	18
+199	19	19
+right join 2
+51	21	11
+52	22	12
+53	23	13
+54	24	14
+55	25	15
+56	26	16
+57	27	17
+58	28	18
+59	29	19
+60	0	0
+61	1	1
+62	2	2
+63	3	3
+64	4	4
+65	5	5
+66	6	6
+67	7	7
+68	8	8
+69	9	9
+70	10	10
+71	11	11
+72	12	12
+73	13	13
+74	14	14
+75	15	15
+76	16	16
+77	17	17
+78	18	18
+79	19	19
+80	20	0
+81	21	1
+82	22	2
+83	23	3
+84	24	4
+85	25	5
+86	26	6
+87	27	7
+88	28	8
+89	29	9
+90	0	10
+91	1	11
+92	2	12
+93	3	13
+94	4	14
+95	5	15
+96	6	16
+97	7	17
+98	8	18
+99	9	19
+100	10	0
+101	11	1
+102	12	2
+103	13	3
+104	14	4
+105	15	5
+106	16	6
+107	17	7
+108	18	8
+109	19	9
+110	20	10
+111	21	11
+112	22	12
+113	23	13
+114	24	14
+115	25	15
+116	26	16
+117	27	17
+118	28	18
+119	29	19
+120	0	0
+121	1	1
+122	2	2
+123	3	3
+124	4	4
+125	5	5
+126	6	6
+127	7	7
+128	8	8
+129	9	9
+130	10	10
+131	11	11
+132	12	12
+133	13	13
+134	14	14
+135	15	15
+136	16	16
+137	17	17
+138	18	18
+139	19	19
+140	20	0
+141	21	1
+142	22	2
+143	23	3
+144	24	4
+145	25	5
+146	26	6
+147	27	7
+148	28	8
+149	29	9
+150	0	10
+151	1	11
+152	2	12
+153	3	13
+154	4	14
+155	5	15
+156	6	16
+157	7	17
+158	8	18
+159	9	19
+160	10	0
+161	11	1
+162	12	2
+163	13	3
+164	14	4
+165	15	5
+166	16	6
+167	17	7
+168	18	8
+169	19	9
+170	20	10
+171	21	11
+172	22	12
+173	23	13
+174	24	14
+175	25	15
+176	26	16
+177	27	17
+178	28	18
+179	29	19
+180	0	0
+181	1	1
+182	2	2
+183	3	3
+184	4	4
+185	5	5
+186	6	6
+187	7	7
+188	8	8
+189	9	9
+190	10	10
+191	11	11
+192	12	12
+193	13	13
+194	14	14
+195	15	15
+196	16	16
+197	17	17
+198	18	18
+199	19	19
+full join 1
+0	\N	0
+0	\N	0
+0	\N	0
+0	\N	0
+0	\N	0
+0	\N	1
+0	\N	1
+0	\N	1
+0	\N	1
+0	\N	1
+0	\N	2
+0	\N	2
+0	\N	2
+0	\N	2
+0	\N	2
+0	\N	3
+0	\N	3
+0	\N	3
+0	\N	3
+0	\N	3
+0	\N	4
+0	\N	4
+0	\N	4
+0	\N	4
+0	\N	4
+0	\N	5
+0	\N	5
+0	\N	5
+0	\N	5
+0	\N	5
+0	\N	6
+0	\N	6
+0	\N	6
+0	\N	6
+0	\N	6
+0	\N	7
+0	\N	7
+0	\N	7
+0	\N	7
+0	\N	7
+0	\N	8
+0	\N	8
+0	\N	8
+0	\N	8
+0	\N	8
+0	\N	9
+0	\N	9
+0	\N	9
+0	\N	9
+0	\N	9
+0	\N	10
+0	\N	10
+0	\N	10
+0	\N	10
+0	\N	10
+0	\N	11
+0	\N	11
+0	\N	11
+0	\N	11
+0	\N	11
+0	\N	12
+0	\N	12
+0	\N	12
+0	\N	12
+0	\N	12
+0	\N	13
+0	\N	13
+0	\N	13
+0	\N	13
+0	\N	13
+0	\N	14
+0	\N	14
+0	\N	14
+0	\N	14
+0	\N	14
+0	\N	15
+0	\N	15
+0	\N	15
+0	\N	15
+0	\N	15
+0	\N	16
+0	\N	16
+0	\N	16
+0	\N	16
+0	\N	16
+0	\N	17
+0	\N	17
+0	\N	17
+0	\N	17
+0	\N	17
+0	\N	18
+0	\N	18
+0	\N	18
+0	\N	18
+0	\N	18
+0	\N	19
+0	\N	19
+0	\N	19
+0	\N	19
+0	\N	19
+0	\N	20
+0	\N	20
+0	\N	20
+0	\N	20
+0	\N	20
+0	\N	20
+0	\N	21
+0	\N	21
+0	\N	21
+0	\N	21
+0	\N	21
+0	\N	22
+0	\N	22
+0	\N	22
+0	\N	22
+0	\N	22
+0	\N	23
+0	\N	23
+0	\N	23
+0	\N	23
+0	\N	23
+0	\N	24
+0	\N	24
+0	\N	24
+0	\N	24
+0	\N	24
+0	\N	25
+0	\N	25
+0	\N	25
+0	\N	25
+0	\N	25
+0	\N	26
+0	\N	26
+0	\N	26
+0	\N	26
+0	\N	26
+0	\N	27
+0	\N	27
+0	\N	27
+0	\N	27
+0	\N	27
+0	\N	28
+0	\N	28
+0	\N	28
+0	\N	28
+0	\N	28
+0	\N	29
+0	\N	29
+0	\N	29
+0	\N	29
+0	\N	29
+51	11	21
+52	12	22
+53	13	23
+54	14	24
+55	15	25
+56	16	26
+57	17	27
+58	18	28
+59	19	29
+60	0	0
+61	1	1
+62	2	2
+63	3	3
+64	4	4
+65	5	5
+66	6	6
+67	7	7
+68	8	8
+69	9	9
+70	10	10
+71	11	11
+72	12	12
+73	13	13
+74	14	14
+75	15	15
+76	16	16
+77	17	17
+78	18	18
+79	19	19
+80	0	20
+81	1	21
+82	2	22
+83	3	23
+84	4	24
+85	5	25
+86	6	26
+87	7	27
+88	8	28
+89	9	29
+90	10	0
+91	11	1
+92	12	2
+93	13	3
+94	14	4
+95	15	5
+96	16	6
+97	17	7
+98	18	8
+99	19	9
+100	0	10
+101	1	11
+102	2	12
+103	3	13
+104	4	14
+105	5	15
+106	6	16
+107	7	17
+108	8	18
+109	9	19
+110	10	20
+111	11	21
+112	12	22
+113	13	23
+114	14	24
+115	15	25
+116	16	26
+117	17	27
+118	18	28
+119	19	29
+120	0	0
+121	1	1
+122	2	2
+123	3	3
+124	4	4
+125	5	5
+126	6	6
+127	7	7
+128	8	8
+129	9	9
+130	10	10
+131	11	11
+132	12	12
+133	13	13
+134	14	14
+135	15	15
+136	16	16
+137	17	17
+138	18	18
+139	19	19
+140	0	20
+141	1	21
+142	2	22
+143	3	23
+144	4	24
+145	5	25
+146	6	26
+147	7	27
+148	8	28
+149	9	29
+150	10	0
+151	11	1
+152	12	2
+153	13	3
+154	14	4
+155	15	5
+156	16	6
+157	17	7
+158	18	8
+159	19	9
+160	0	10
+161	1	11
+162	2	12
+163	3	13
+164	4	14
+165	5	15
+166	6	16
+167	7	17
+168	8	18
+169	9	19
+170	10	20
+171	11	21
+172	12	22
+173	13	23
+174	14	24
+175	15	25
+176	16	26
+177	17	27
+178	18	28
+179	19	29
+180	0	0
+181	1	1
+182	2	2
+183	3	3
+184	4	4
+185	5	5
+186	6	6
+187	7	7
+188	8	8
+189	9	9
+190	10	10
+191	11	11
+192	12	12
+193	13	13
+194	14	14
+195	15	15
+196	16	16
+197	17	17
+198	18	18
+199	19	19
+full join 2
+0	0	\N
+1	1	\N
+2	2	\N
+3	3	\N
+4	4	\N
+5	5	\N
+6	6	\N
+7	7	\N
+8	8	\N
+9	9	\N
+10	10	\N
+11	11	\N
+12	12	\N
+13	13	\N
+14	14	\N
+15	15	\N
+16	16	\N
+17	17	\N
+18	18	\N
+19	19	\N
+20	20	\N
+21	21	\N
+22	22	\N
+23	23	\N
+24	24	\N
+25	25	\N
+26	26	\N
+27	27	\N
+28	28	\N
+29	29	\N
+30	0	\N
+31	1	\N
+32	2	\N
+33	3	\N
+34	4	\N
+35	5	\N
+36	6	\N
+37	7	\N
+38	8	\N
+39	9	\N
+40	10	\N
+41	11	\N
+42	12	\N
+43	13	\N
+44	14	\N
+45	15	\N
+46	16	\N
+47	17	\N
+48	18	\N
+49	19	\N
+50	20	\N
+51	21	11
+52	22	12
+53	23	13
+54	24	14
+55	25	15
+56	26	16
+57	27	17
+58	28	18
+59	29	19
+60	0	0
+61	1	1
+62	2	2
+63	3	3
+64	4	4
+65	5	5
+66	6	6
+67	7	7
+68	8	8
+69	9	9
+70	10	10
+71	11	11
+72	12	12
+73	13	13
+74	14	14
+75	15	15
+76	16	16
+77	17	17
+78	18	18
+79	19	19
+80	20	0
+81	21	1
+82	22	2
+83	23	3
+84	24	4
+85	25	5
+86	26	6
+87	27	7
+88	28	8
+89	29	9
+90	0	10
+91	1	11
+92	2	12
+93	3	13
+94	4	14
+95	5	15
+96	6	16
+97	7	17
+98	8	18
+99	9	19
+100	10	0
+101	11	1
+102	12	2
+103	13	3
+104	14	4
+105	15	5
+106	16	6
+107	17	7
+108	18	8
+109	19	9
+110	20	10
+111	21	11
+112	22	12
+113	23	13
+114	24	14
+115	25	15
+116	26	16
+117	27	17
+118	28	18
+119	29	19
+120	0	0
+121	1	1
+122	2	2
+123	3	3
+124	4	4
+125	5	5
+126	6	6
+127	7	7
+128	8	8
+129	9	9
+130	10	10
+131	11	11
+132	12	12
+133	13	13
+134	14	14
+135	15	15
+136	16	16
+137	17	17
+138	18	18
+139	19	19
+140	20	0
+141	21	1
+142	22	2
+143	23	3
+144	24	4
+145	25	5
+146	26	6
+147	27	7
+148	28	8
+149	29	9
+150	0	10
+151	1	11
+152	2	12
+153	3	13
+154	4	14
+155	5	15
+156	6	16
+157	7	17
+158	8	18
+159	9	19
+160	10	0
+161	11	1
+162	12	2
+163	13	3
+164	14	4
+165	15	5
+166	16	6
+167	17	7
+168	18	8
+169	19	9
+170	20	10
+171	21	11
+172	22	12
+173	23	13
+174	24	14
+175	25	15
+176	26	16
+177	27	17
+178	28	18
+179	29	19
+180	0	0
+181	1	1
+182	2	2
+183	3	3
+184	4	4
+185	5	5
+186	6	6
+187	7	7
+188	8	8
+189	9	9
+190	10	10
+191	11	11
+192	12	12
+193	13	13
+194	14	14
+195	15	15
+196	16	16
+197	17	17
+198	18	18
+199	19	19
+200	20	\N
+201	21	\N
+202	22	\N
+203	23	\N
+204	24	\N
+205	25	\N
+206	26	\N
+207	27	\N
+208	28	\N
+209	29	\N
+210	0	\N
+211	1	\N
+212	2	\N
+213	3	\N
+214	4	\N
+215	5	\N
+216	6	\N
+217	7	\N
+218	8	\N
+219	9	\N
+220	10	\N
+221	11	\N
+222	12	\N
+223	13	\N
+224	14	\N
+225	15	\N
+226	16	\N
+227	17	\N
+228	18	\N
+229	19	\N
+230	20	\N
+231	21	\N
+232	22	\N
+233	23	\N
+234	24	\N
+235	25	\N
+236	26	\N
+237	27	\N
+238	28	\N
+239	29	\N
+240	0	\N
+241	1	\N
+242	2	\N
+243	3	\N
+244	4	\N
+245	5	\N
+246	6	\N
+247	7	\N
+248	8	\N
+249	9	\N
+250	10	\N
+251	11	\N
+252	12	\N
+253	13	\N
+254	14	\N
+255	15	\N
+256	16	\N
+257	17	\N
+258	18	\N
+259	19	\N
+260	20	\N
+261	21	\N
+262	22	\N
+263	23	\N
+264	24	\N
+265	25	\N
+266	26	\N
+267	27	\N
+268	28	\N
+269	29	\N
+270	0	\N
+271	1	\N
+272	2	\N
+273	3	\N
+274	4	\N
+275	5	\N
+276	6	\N
+277	7	\N
+278	8	\N
+279	9	\N
+280	10	\N
+281	11	\N
+282	12	\N
+283	13	\N
+284	14	\N
+285	15	\N
+286	16	\N
+287	17	\N
+288	18	\N
+289	19	\N
+290	20	\N
+291	21	\N
+292	22	\N
+293	23	\N
+294	24	\N
+295	25	\N
+296	26	\N
+297	27	\N
+298	28	\N
+299	29	\N
diff --git a/tests/queries/0_stateless/02724_grace_hash_join.sql b/tests/queries/0_stateless/02724_grace_hash_join.sql
new file mode 100644
index 00000000000..370f6058bef
--- /dev/null
+++ b/tests/queries/0_stateless/02724_grace_hash_join.sql
@@ -0,0 +1,44 @@
+set join_algorithm = 'grace_hash';
+set max_rows_in_join = 100;
+
+drop table if exists test_t1;
+drop table if exists test_t2;
+
+CREATE TABLE test_t1
+(
+    `x` UInt64,
+    `y` Nullable(UInt64)
+)
+ENGINE = Memory;
+
+CREATE TABLE test_t2
+(
+    `x` UInt64,
+    `y` Nullable(UInt64)
+)
+ENGINE = Memory;
+
+insert into test_t1 select number as x, number % 20 as y from numbers(200) where number > 50 order by y;
+insert into test_t2 select number as x , number % 30 as y from numbers(300) order by y;
+
+select 'inner join 1';
+select l.x, l.y, r.y from test_t1 as l inner join test_t2 as r on l.x = r.x order by l.x, l.y, r.y;
+select 'inner join 2';
+select l.x, l.y, r.y from test_t2 as l inner join test_t1 as r on l.x = r.x order by l.x, l.y, r.y;
+select 'left join 1';
+select l.x, l.y, r.y from test_t1 as l left join test_t2 as r on l.x = r.x order by l.x, l.y, r.y;
+select 'left join 2';
+select l.x, l.y, r.y from test_t2 as l left join test_t1 as r on l.x = r.x order by l.x, l.y, r.y;
+select 'right join 1';
+select l.x, l.y, r.y from test_t1 as l right join test_t2 as r on l.x = r.x order by l.x, l.y, r.y;
+select 'right join 2';
+select l.x, l.y, r.y from test_t2 as l right join test_t1 as r on l.x = r.x order by l.x, l.y, r.y;
+select 'full join 1';
+select l.x, l.y, r.y from test_t1 as l full join test_t2 as r on l.x = r.x order by l.x, l.y, r.y;
+select 'full join 2';
+select l.x, l.y, r.y from test_t2 as l full join test_t1 as r on l.x = r.x order by l.x, l.y, r.y;
+
+
+
+
+

From 39db0f84d95ade3ab88c0c6bb8cd24b4ac2a58a0 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Thu, 4 May 2023 11:27:20 +0800
Subject: [PATCH 675/689] add comment

---
 src/Interpreters/GraceHashJoin.cpp | 11 ++++++++---
 src/Interpreters/JoinUtils.cpp     |  2 +-
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index 32801887f2b..95bb76b8373 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -302,7 +302,7 @@ void GraceHashJoin::initBuckets()
     current_bucket->startJoining();
 }
 
-bool GraceHashJoin::isSupported(const std::shared_ptr<TableJoin> & table_join [[maybe_unused]])
+bool GraceHashJoin::isSupported(const std::shared_ptr<TableJoin> & table_join)
 {
 
     bool is_asof = (table_join->strictness() == JoinStrictness::Asof);
@@ -472,8 +472,10 @@ bool GraceHashJoin::alwaysReturnsEmptySet() const
     return hash_join_is_empty;
 }
 
-IBlocksStreamPtr GraceHashJoin::getNonJoinedBlocks(
-    const Block & left_sample_block_ [[maybe_unused]], const Block & result_sample_block_ [[maybe_unused]], UInt64 max_block_size_ [[maybe_unused]]) const
+// This is only be called for bucket[0] at present. other buckets non-joined blocks are generated in
+// DelayedBlocks.
+IBlocksStreamPtr
+GraceHashJoin::getNonJoinedBlocks(const Block & left_sample_block_, const Block & result_sample_block_, UInt64 max_block_size_) const
 {
     return hash_join->getNonJoinedBlocks(left_sample_block_, result_sample_block_, max_block_size_);
 }
@@ -510,6 +512,8 @@ public:
 
         do
         {
+            // When left reader finish, return non-joined blocks.
+            // empty block means the end of this stream.
             if (!is_left_reader_finished)
             {
                 block = left_reader.read();
@@ -521,6 +525,7 @@ public:
             }
             if (is_left_reader_finished)
             {
+                // full/right join, non_joined_blocks != nullptr
                 if (non_joined_blocks)
                 {
                     block = non_joined_blocks->next();
diff --git a/src/Interpreters/JoinUtils.cpp b/src/Interpreters/JoinUtils.cpp
index 59ee86ffa18..a05b58e14a1 100644
--- a/src/Interpreters/JoinUtils.cpp
+++ b/src/Interpreters/JoinUtils.cpp
@@ -19,7 +19,6 @@
 
 #include <base/FnTraits.h>
 
-
 namespace DB
 {
 
@@ -865,6 +864,7 @@ Block NotJoinedBlocks::nextImpl()
         size_t rows_added = filler->fillColumns(columns_right);
         if (rows_added == 0)
             return {};
+
         addLeftColumns(result_block, rows_added);
         addRightColumns(result_block, columns_right);
     }

From 7772fed161b96cda58b837e589413b5fec01d49d Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Fri, 5 May 2023 15:09:39 +0800
Subject: [PATCH 676/689] update

1. fixed the memoery overflow problem when  handle all delayed buckets parallely
2. resue exists tests
---
 src/Interpreters/GraceHashJoin.cpp            |   87 +-
 src/Interpreters/GraceHashJoin.h              |    3 +
 .../Transforms/JoiningTransform.cpp           |   90 +-
 src/Processors/Transforms/JoiningTransform.h  |    6 +-
 .../01721_join_implicit_cast_long.sql.j2      |    1 -
 .../0_stateless/02273_full_sort_join.sql.j2   |    2 -
 .../02274_full_sort_join_nodistinct.sql.j2    |    1 -
 .../02275_full_sort_join_long.reference       |   24 +-
 .../02275_full_sort_join_long.sql.j2          |    9 +-
 .../02724_grace_hash_join.reference           | 1804 -----------------
 .../0_stateless/02724_grace_hash_join.sql     |   44 -
 11 files changed, 128 insertions(+), 1943 deletions(-)
 delete mode 100644 tests/queries/0_stateless/02724_grace_hash_join.reference
 delete mode 100644 tests/queries/0_stateless/02724_grace_hash_join.sql

diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index 95bb76b8373..407663ac310 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -12,6 +12,7 @@
 #include <Disks/TemporaryFileOnDisk.h>
 #include <Common/logger_useful.h>
 #include <Common/thread_local_rng.h>
+#include "GraceHashJoin.h"
 
 #include <base/FnTraits.h>
 #include <fmt/format.h>
@@ -472,12 +473,67 @@ bool GraceHashJoin::alwaysReturnsEmptySet() const
     return hash_join_is_empty;
 }
 
-// This is only be called for bucket[0] at present. other buckets non-joined blocks are generated in
+// Add lock for reading non-joined blocks, it's thread safe.
+class GraceHashJoin::NonJoinedBlocksStream : public IBlocksStream
+{
+public:
+    explicit NonJoinedBlocksStream(
+        InMemoryJoinPtr hash_join_,
+        const Block & left_sample_block_,
+        const Block & result_sample_block_,
+        size_t max_block_size_)
+        : hash_join(hash_join_)
+        , left_sample_block(left_sample_block_)
+        , result_sample_block(result_sample_block_)
+        , max_block_size(max_block_size_)
+    {
+    }
+
+    Block nextImpl() override
+    {
+        // initialize non_joined_blocks lazily.
+        if (!has_init_non_joined_blocks) [[unlikely]]
+        {
+            std::lock_guard lock(mutex);
+            if (!has_init_non_joined_blocks)
+            {
+                has_init_non_joined_blocks = true;
+                non_joined_blocks = hash_join->getNonJoinedBlocks(left_sample_block, result_sample_block, max_block_size);
+            }
+        }
+        // For right/full join, non_joined_blocks is not null.
+        // For inner/left join, non_joined_blocks is null, and return a empty block directly.
+        std::lock_guard lock(mutex);
+        if (non_joined_blocks)
+            return non_joined_blocks->next();
+        return {};
+    }
+
+private:
+    InMemoryJoinPtr hash_join;
+    Block left_sample_block;
+    Block result_sample_block;
+    size_t max_block_size;
+    bool has_init_non_joined_blocks = false;
+    IBlocksStreamPtr non_joined_blocks = nullptr;
+    std::mutex mutex;
+};
+
+// This is only be called for bucket[0] at present. other buckets' non-joined blocks are generated in
 // DelayedBlocks.
 IBlocksStreamPtr
 GraceHashJoin::getNonJoinedBlocks(const Block & left_sample_block_, const Block & result_sample_block_, UInt64 max_block_size_) const
 {
-    return hash_join->getNonJoinedBlocks(left_sample_block_, result_sample_block_, max_block_size_);
+    if (!has_initialized_non_joined_blocks) [[unlikely]]
+    {
+        std::lock_guard lock(hash_join_mutex);
+        if (!has_initialized_non_joined_blocks)
+        {
+            has_initialized_non_joined_blocks = true;
+            non_joined_blocks = std::make_shared<NonJoinedBlocksStream>(hash_join, left_sample_block_, result_sample_block_, max_block_size_);
+        }
+    }
+    return non_joined_blocks;
 }
 
 class GraceHashJoin::DelayedBlocks : public IBlocksStream
@@ -498,12 +554,14 @@ public:
         , left_reader(buckets[current_bucket]->getLeftTableReader())
         , left_key_names(left_key_names_)
         , right_key_names(right_key_names_)
-        , left_sample_block(left_sample_block_)
-        , result_sample_block(result_sample_block_)
-        , max_block_size(max_block_size_)
+        , non_joined_blocks_iter(hash_join, left_sample_block_, result_sample_block_, max_block_size_)
     {
     }
 
+    // One DelayedBlocks is shared among multiple DelayedJoinedBlocksWorkerTransforms, need locks for
+    // - reading from left_reader. left_reader.read() has had a lock inside.
+    // - reading non-joined blocks from hash_join. Since iterate on non-joined blocks will has state
+    //   changed inside.
     Block nextImpl() override
     {
         Block block;
@@ -520,24 +578,14 @@ public:
                 if (!block)
                 {
                     is_left_reader_finished = true;
-                    non_joined_blocks = hash_join->getNonJoinedBlocks(left_sample_block, result_sample_block, max_block_size);
                 }
             }
             if (is_left_reader_finished)
             {
-                // full/right join, non_joined_blocks != nullptr
-                if (non_joined_blocks)
-                {
-                    block = non_joined_blocks->next();
-                    return block;
-                }
-                else
-                {
-                    // left/inner join.
-                    return {};
-                }
+                return non_joined_blocks_iter.next();
             }
 
+            // block comes from left_reader, need to join with right table to get the result.
             Blocks blocks = JoinCommon::scatterBlockByHash(left_key_names, block, num_buckets);
             block = std::move(blocks[current_idx]);
 
@@ -577,11 +625,8 @@ public:
 
     Names left_key_names;
     Names right_key_names;
-    Block left_sample_block;
-    Block result_sample_block;
     bool is_left_reader_finished = false;
-    IBlocksStreamPtr non_joined_blocks;
-    size_t max_block_size;
+    NonJoinedBlocksStream non_joined_blocks_iter;
 };
 
 IBlocksStreamPtr GraceHashJoin::getDelayedBlocks()
diff --git a/src/Interpreters/GraceHashJoin.h b/src/Interpreters/GraceHashJoin.h
index eb39ee09208..ec0e5ac05aa 100644
--- a/src/Interpreters/GraceHashJoin.h
+++ b/src/Interpreters/GraceHashJoin.h
@@ -43,6 +43,7 @@ class HashJoin;
 class GraceHashJoin final : public IJoin
 {
     class FileBucket;
+    class NonJoinedBlocksStream;
     class DelayedBlocks;
     using InMemoryJoin = HashJoin;
 
@@ -144,6 +145,8 @@ private:
     InMemoryJoinPtr hash_join;
     Block hash_join_sample_block;
     mutable std::mutex hash_join_mutex;
+    mutable bool has_initialized_non_joined_blocks = false;
+    mutable std::shared_ptr<NonJoinedBlocksStream> non_joined_blocks = nullptr;
 };
 
 }
diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp
index 4789e9c514a..b638a5582c5 100644
--- a/src/Processors/Transforms/JoiningTransform.cpp
+++ b/src/Processors/Transforms/JoiningTransform.cpp
@@ -350,11 +350,6 @@ IProcessor::Status DelayedJoinedBlocksWorkerTransform::prepare()
 
     if (!task)
     {
-        if (input.isFinished())
-        {
-            output.finish();
-            return Status::Finished;
-        }
         if (!input.hasData())
         {
             input.setNeeded();
@@ -367,14 +362,10 @@ IProcessor::Status DelayedJoinedBlocksWorkerTransform::prepare()
             output.pushException(data.exception);
             return Status::Finished;
         }
-        if (data.chunk.hasChunkInfo())
-            task = std::dynamic_pointer_cast<const DelayedBlocksTask>(data.chunk.getChunkInfo());
-        else
-        {
-            // Try to get one task from DelayedJoinedBlocksTransform again.
-            task = nullptr;
-            return Status::NeedData;
-        }
+
+        if (!data.chunk.hasChunkInfo())
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "DelayedJoinedBlocksWorkerTransform must have chunk info");
+        task = std::dynamic_pointer_cast<const DelayedBlocksTask>(data.chunk.getChunkInfo());
     }
     else
     {
@@ -395,6 +386,7 @@ void DelayedJoinedBlocksWorkerTransform::work()
 {
     if (!task)
         return;
+
     Block block = task->delayed_blocks->next();
 
     if (!block)
@@ -416,19 +408,15 @@ DelayedJoinedBlocksTransform::DelayedJoinedBlocksTransform(size_t num_streams, J
 
 void DelayedJoinedBlocksTransform::work()
 {
-    if (all_buckets_finished)
+    if (finished)
         return;
+
     delayed_blocks = join->getDelayedBlocks();
-    if (!delayed_blocks)
-    {
-        all_buckets_finished = true;
-    }
+    finished = finished || delayed_blocks == nullptr;
 }
 
 IProcessor::Status DelayedJoinedBlocksTransform::prepare()
 {
-
-    bool should_finished = false;
     for (auto & output : outputs)
     {
         if (output.isFinished())
@@ -436,56 +424,44 @@ IProcessor::Status DelayedJoinedBlocksTransform::prepare()
             /// If at least one output is finished, then we have read all data from buckets.
             /// Some workers can still be busy with joining the last chunk of data in memory,
             /// but after that they also will finish when they will try to get next chunk.
-            should_finished = true;
-            break;
+            finished = true;
+            continue;
         }
-    }
-    if (should_finished)
-    {
-        for (auto & output : outputs)
-        {
-            if (!output.isFinished())
-            {
-                output.finish();
-            }
-        }
-        return Status::Finished;
+        if (!output.canPush())
+            return Status::PortFull;
     }
 
-    // No pending buckets.
-    if (all_buckets_finished && !delayed_blocks)
+    if (finished)
     {
+        // Since have memory limit, cannot handle all buckets parallelly by different
+        // DelayedJoinedBlocksWorkerTransform. So send the same task to all outputs.
+        // Wait for all DelayedJoinedBlocksWorkerTransform be idle before getting next bucket.
         for (auto & output : outputs)
+        {
+            if (output.isFinished())
+                continue;
+            Chunk chunk;
+            chunk.setChunkInfo(std::make_shared<DelayedBlocksTask>());
+            output.push(std::move(chunk));
             output.finish();
+        }
+
         return Status::Finished;
     }
 
-    // Try to read next bucket.
-    if (!delayed_blocks)
-        return Status::Ready;
-
-    // Put the ready task to the first available output port.
-    // Put empty tasks to other output ports. these ports will do nothing but require data again.
-    // Each bucket will handled in one thread, avoid lock contention in left file reader.
-    for (auto & output : outputs)
+    if (delayed_blocks)
     {
-        if (output.canPush())
+        for (auto & output : outputs)
         {
-            if (delayed_blocks)
-            {
-                Chunk chunk;
-                chunk.setChunkInfo(std::make_shared<DelayedBlocksTask>(delayed_blocks));
-                output.push(std::move(chunk));
-                delayed_blocks = nullptr;
-            }
-            else
-            {
-                Chunk chunk;
-                output.push(std::move(chunk));
-            }
+            Chunk chunk;
+            chunk.setChunkInfo(std::make_shared<DelayedBlocksTask>(delayed_blocks));
+            output.push(std::move(chunk));
         }
+        delayed_blocks = nullptr;
+        return Status::PortFull;
     }
-    return Status::PortFull;
+
+    return Status::Ready;
 }
 
 }
diff --git a/src/Processors/Transforms/JoiningTransform.h b/src/Processors/Transforms/JoiningTransform.h
index f189f53c004..e7edff40c56 100644
--- a/src/Processors/Transforms/JoiningTransform.h
+++ b/src/Processors/Transforms/JoiningTransform.h
@@ -125,6 +125,7 @@ public:
 
 using DelayedBlocksTaskPtr = std::shared_ptr<const DelayedBlocksTask>;
 
+
 /// Reads delayed joined blocks from Join
 class DelayedJoinedBlocksTransform : public IProcessor
 {
@@ -138,8 +139,9 @@ public:
 
 private:
     JoinPtr join;
-    IBlocksStreamPtr delayed_blocks;
-    bool all_buckets_finished = false;
+
+    IBlocksStreamPtr delayed_blocks = nullptr;
+    bool finished = false;
 };
 
 class DelayedJoinedBlocksWorkerTransform : public IProcessor
diff --git a/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 b/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2
index f5321939f28..38f71f4c5ec 100644
--- a/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2
+++ b/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2
@@ -10,7 +10,6 @@ INSERT INTO t1 SELECT number as a, 100 + number as b FROM system.numbers LIMIT 1
 INSERT INTO t2 SELECT number - 5 as a, 200 + number - 5 as b FROM system.numbers LIMIT 1, 10;
 
 {% macro is_implemented(join_algorithm) -%}
-{% if join_algorithm == 'grace_hash' %} -- { serverError NOT_IMPLEMENTED } {% endif %}
 {% endmacro -%}
 
 {% for join_algorithm in ['hash', 'partial_merge', 'auto', 'full_sorting_merge', 'grace_hash'] -%}
diff --git a/tests/queries/0_stateless/02273_full_sort_join.sql.j2 b/tests/queries/0_stateless/02273_full_sort_join.sql.j2
index 8b739330364..b371d9544da 100644
--- a/tests/queries/0_stateless/02273_full_sort_join.sql.j2
+++ b/tests/queries/0_stateless/02273_full_sort_join.sql.j2
@@ -26,9 +26,7 @@ INSERT INTO t2
         'val' || toString(number) as s
     FROM numbers_mt({{ table_size - 3 }});
 
-
 {% macro is_implemented(join_algorithm) -%}
-{% if join_algorithm == 'grace_hash' %} -- { serverError NOT_IMPLEMENTED } {% endif %}
 {% endmacro -%}
 
 {% for join_algorithm in ['default', 'full_sorting_merge', 'grace_hash'] -%}
diff --git a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2 b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2
index 613da65421e..6bf634ca1f5 100644
--- a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2
+++ b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2
@@ -16,7 +16,6 @@ INSERT INTO t2 VALUES (1, 'val11'), (1, 'val12'), (2, 'val22'), (2, 'val23'), (2
 INSERT INTO tn2 VALUES (1, 'val11'), (1, 'val12'), (NULL, 'val22'), (NULL, 'val23'), (NULL, 'val24'), (NULL, 'val25'), (NULL, 'val26'), (NULL, 'val27'), (NULL, 'val28'), (3, 'val3');
 
 {% macro is_implemented(join_algorithm) -%}
-{% if join_algorithm == 'grace_hash' %} -- { serverError NOT_IMPLEMENTED } {% endif %}
 {% endmacro -%}
 
 {% for join_algorithm in ['full_sorting_merge', 'grace_hash'] -%}
diff --git a/tests/queries/0_stateless/02275_full_sort_join_long.reference b/tests/queries/0_stateless/02275_full_sort_join_long.reference
index 9ec06aea3e6..73482358d12 100644
--- a/tests/queries/0_stateless/02275_full_sort_join_long.reference
+++ b/tests/queries/0_stateless/02275_full_sort_join_long.reference
@@ -41,16 +41,34 @@ ALL INNER
 ALL LEFT
 50195752660639	500353531835	10369589	10369589	1000342
 ALL RIGHT
-skipped
+500353531835	684008812186	1367170	1000342	1367170
 ALL INNER
 500353531835	500353531835	1000342	1000342	1000342
 ALL LEFT
 50195752660639	500353531835	10369589	10369589	1000342
 ALL RIGHT
-skipped
+500353531835	684008812186	1367170	1000342	1367170
 ALL INNER
 500353531835	500353531835	1000342	1000342	1000342
 ALL LEFT
 50195752660639	500353531835	10369589	10369589	1000342
 ALL RIGHT
-skipped
+500353531835	684008812186	1367170	1000342	1367170
+ANY INNER
+199622811843	199622811843	399458	399458	399458
+ANY LEFT
+50010619420459	315220291655	10000000	10000000	630753
+ANY RIGHT
+316611844056	500267124407	1000000	633172	1000000
+ANY INNER
+199622811843	199622811843	399458	399458	399458
+ANY LEFT
+50010619420459	315220291655	10000000	10000000	630753
+ANY RIGHT
+316611844056	500267124407	1000000	633172	1000000
+ANY INNER
+199622811843	199622811843	399458	399458	399458
+ANY LEFT
+50010619420459	315220291655	10000000	10000000	630753
+ANY RIGHT
+316611844056	500267124407	1000000	633172	1000000
diff --git a/tests/queries/0_stateless/02275_full_sort_join_long.sql.j2 b/tests/queries/0_stateless/02275_full_sort_join_long.sql.j2
index 98cc46c9cb4..0b28fd67050 100644
--- a/tests/queries/0_stateless/02275_full_sort_join_long.sql.j2
+++ b/tests/queries/0_stateless/02275_full_sort_join_long.sql.j2
@@ -22,11 +22,6 @@ INSERT INTO t2
     FROM numbers_mt({{ rtable_size }})
 ;
 
-{% macro is_implemented(join_algorithm) -%}
-{% if join_algorithm == 'grace_hash' %} -- { serverError NOT_IMPLEMENTED }
-SELECT 'skipped';
-{% endif -%}
-{% endmacro -%}
 
 {% for join_algorithm in ['full_sorting_merge', 'grace_hash'] -%}
 
@@ -40,7 +35,6 @@ SET join_algorithm = '{{ join_algorithm }}';
 
 SET max_block_size = {{ block_size }};
 
-{% if not (kind == 'ANY' and join_algorithm == 'grace_hash') -%}
 
 SELECT '{{ kind }} INNER';
 SELECT sum(t1.key), sum(t2.key), count(), countIf(t1.key != 0), countIf(t2.key != 0) FROM t1
@@ -58,9 +52,8 @@ SELECT '{{ kind }} RIGHT';
 SELECT sum(t1.key), sum(t2.key), count(), countIf(t1.key != 0), countIf(t2.key != 0) FROM t1
 {{ kind }} RIGHT JOIN t2
 ON t1.key == t2.key
-; {{ is_implemented(join_algorithm) }}
+;
 
-{% endif -%}
 
 {% endfor -%}
 {% endfor -%}
diff --git a/tests/queries/0_stateless/02724_grace_hash_join.reference b/tests/queries/0_stateless/02724_grace_hash_join.reference
deleted file mode 100644
index 296ef623f04..00000000000
--- a/tests/queries/0_stateless/02724_grace_hash_join.reference
+++ /dev/null
@@ -1,1804 +0,0 @@
-inner join 1
-51	11	21
-52	12	22
-53	13	23
-54	14	24
-55	15	25
-56	16	26
-57	17	27
-58	18	28
-59	19	29
-60	0	0
-61	1	1
-62	2	2
-63	3	3
-64	4	4
-65	5	5
-66	6	6
-67	7	7
-68	8	8
-69	9	9
-70	10	10
-71	11	11
-72	12	12
-73	13	13
-74	14	14
-75	15	15
-76	16	16
-77	17	17
-78	18	18
-79	19	19
-80	0	20
-81	1	21
-82	2	22
-83	3	23
-84	4	24
-85	5	25
-86	6	26
-87	7	27
-88	8	28
-89	9	29
-90	10	0
-91	11	1
-92	12	2
-93	13	3
-94	14	4
-95	15	5
-96	16	6
-97	17	7
-98	18	8
-99	19	9
-100	0	10
-101	1	11
-102	2	12
-103	3	13
-104	4	14
-105	5	15
-106	6	16
-107	7	17
-108	8	18
-109	9	19
-110	10	20
-111	11	21
-112	12	22
-113	13	23
-114	14	24
-115	15	25
-116	16	26
-117	17	27
-118	18	28
-119	19	29
-120	0	0
-121	1	1
-122	2	2
-123	3	3
-124	4	4
-125	5	5
-126	6	6
-127	7	7
-128	8	8
-129	9	9
-130	10	10
-131	11	11
-132	12	12
-133	13	13
-134	14	14
-135	15	15
-136	16	16
-137	17	17
-138	18	18
-139	19	19
-140	0	20
-141	1	21
-142	2	22
-143	3	23
-144	4	24
-145	5	25
-146	6	26
-147	7	27
-148	8	28
-149	9	29
-150	10	0
-151	11	1
-152	12	2
-153	13	3
-154	14	4
-155	15	5
-156	16	6
-157	17	7
-158	18	8
-159	19	9
-160	0	10
-161	1	11
-162	2	12
-163	3	13
-164	4	14
-165	5	15
-166	6	16
-167	7	17
-168	8	18
-169	9	19
-170	10	20
-171	11	21
-172	12	22
-173	13	23
-174	14	24
-175	15	25
-176	16	26
-177	17	27
-178	18	28
-179	19	29
-180	0	0
-181	1	1
-182	2	2
-183	3	3
-184	4	4
-185	5	5
-186	6	6
-187	7	7
-188	8	8
-189	9	9
-190	10	10
-191	11	11
-192	12	12
-193	13	13
-194	14	14
-195	15	15
-196	16	16
-197	17	17
-198	18	18
-199	19	19
-inner join 2
-51	21	11
-52	22	12
-53	23	13
-54	24	14
-55	25	15
-56	26	16
-57	27	17
-58	28	18
-59	29	19
-60	0	0
-61	1	1
-62	2	2
-63	3	3
-64	4	4
-65	5	5
-66	6	6
-67	7	7
-68	8	8
-69	9	9
-70	10	10
-71	11	11
-72	12	12
-73	13	13
-74	14	14
-75	15	15
-76	16	16
-77	17	17
-78	18	18
-79	19	19
-80	20	0
-81	21	1
-82	22	2
-83	23	3
-84	24	4
-85	25	5
-86	26	6
-87	27	7
-88	28	8
-89	29	9
-90	0	10
-91	1	11
-92	2	12
-93	3	13
-94	4	14
-95	5	15
-96	6	16
-97	7	17
-98	8	18
-99	9	19
-100	10	0
-101	11	1
-102	12	2
-103	13	3
-104	14	4
-105	15	5
-106	16	6
-107	17	7
-108	18	8
-109	19	9
-110	20	10
-111	21	11
-112	22	12
-113	23	13
-114	24	14
-115	25	15
-116	26	16
-117	27	17
-118	28	18
-119	29	19
-120	0	0
-121	1	1
-122	2	2
-123	3	3
-124	4	4
-125	5	5
-126	6	6
-127	7	7
-128	8	8
-129	9	9
-130	10	10
-131	11	11
-132	12	12
-133	13	13
-134	14	14
-135	15	15
-136	16	16
-137	17	17
-138	18	18
-139	19	19
-140	20	0
-141	21	1
-142	22	2
-143	23	3
-144	24	4
-145	25	5
-146	26	6
-147	27	7
-148	28	8
-149	29	9
-150	0	10
-151	1	11
-152	2	12
-153	3	13
-154	4	14
-155	5	15
-156	6	16
-157	7	17
-158	8	18
-159	9	19
-160	10	0
-161	11	1
-162	12	2
-163	13	3
-164	14	4
-165	15	5
-166	16	6
-167	17	7
-168	18	8
-169	19	9
-170	20	10
-171	21	11
-172	22	12
-173	23	13
-174	24	14
-175	25	15
-176	26	16
-177	27	17
-178	28	18
-179	29	19
-180	0	0
-181	1	1
-182	2	2
-183	3	3
-184	4	4
-185	5	5
-186	6	6
-187	7	7
-188	8	8
-189	9	9
-190	10	10
-191	11	11
-192	12	12
-193	13	13
-194	14	14
-195	15	15
-196	16	16
-197	17	17
-198	18	18
-199	19	19
-left join 1
-51	11	21
-52	12	22
-53	13	23
-54	14	24
-55	15	25
-56	16	26
-57	17	27
-58	18	28
-59	19	29
-60	0	0
-61	1	1
-62	2	2
-63	3	3
-64	4	4
-65	5	5
-66	6	6
-67	7	7
-68	8	8
-69	9	9
-70	10	10
-71	11	11
-72	12	12
-73	13	13
-74	14	14
-75	15	15
-76	16	16
-77	17	17
-78	18	18
-79	19	19
-80	0	20
-81	1	21
-82	2	22
-83	3	23
-84	4	24
-85	5	25
-86	6	26
-87	7	27
-88	8	28
-89	9	29
-90	10	0
-91	11	1
-92	12	2
-93	13	3
-94	14	4
-95	15	5
-96	16	6
-97	17	7
-98	18	8
-99	19	9
-100	0	10
-101	1	11
-102	2	12
-103	3	13
-104	4	14
-105	5	15
-106	6	16
-107	7	17
-108	8	18
-109	9	19
-110	10	20
-111	11	21
-112	12	22
-113	13	23
-114	14	24
-115	15	25
-116	16	26
-117	17	27
-118	18	28
-119	19	29
-120	0	0
-121	1	1
-122	2	2
-123	3	3
-124	4	4
-125	5	5
-126	6	6
-127	7	7
-128	8	8
-129	9	9
-130	10	10
-131	11	11
-132	12	12
-133	13	13
-134	14	14
-135	15	15
-136	16	16
-137	17	17
-138	18	18
-139	19	19
-140	0	20
-141	1	21
-142	2	22
-143	3	23
-144	4	24
-145	5	25
-146	6	26
-147	7	27
-148	8	28
-149	9	29
-150	10	0
-151	11	1
-152	12	2
-153	13	3
-154	14	4
-155	15	5
-156	16	6
-157	17	7
-158	18	8
-159	19	9
-160	0	10
-161	1	11
-162	2	12
-163	3	13
-164	4	14
-165	5	15
-166	6	16
-167	7	17
-168	8	18
-169	9	19
-170	10	20
-171	11	21
-172	12	22
-173	13	23
-174	14	24
-175	15	25
-176	16	26
-177	17	27
-178	18	28
-179	19	29
-180	0	0
-181	1	1
-182	2	2
-183	3	3
-184	4	4
-185	5	5
-186	6	6
-187	7	7
-188	8	8
-189	9	9
-190	10	10
-191	11	11
-192	12	12
-193	13	13
-194	14	14
-195	15	15
-196	16	16
-197	17	17
-198	18	18
-199	19	19
-left join 2
-0	0	\N
-1	1	\N
-2	2	\N
-3	3	\N
-4	4	\N
-5	5	\N
-6	6	\N
-7	7	\N
-8	8	\N
-9	9	\N
-10	10	\N
-11	11	\N
-12	12	\N
-13	13	\N
-14	14	\N
-15	15	\N
-16	16	\N
-17	17	\N
-18	18	\N
-19	19	\N
-20	20	\N
-21	21	\N
-22	22	\N
-23	23	\N
-24	24	\N
-25	25	\N
-26	26	\N
-27	27	\N
-28	28	\N
-29	29	\N
-30	0	\N
-31	1	\N
-32	2	\N
-33	3	\N
-34	4	\N
-35	5	\N
-36	6	\N
-37	7	\N
-38	8	\N
-39	9	\N
-40	10	\N
-41	11	\N
-42	12	\N
-43	13	\N
-44	14	\N
-45	15	\N
-46	16	\N
-47	17	\N
-48	18	\N
-49	19	\N
-50	20	\N
-51	21	11
-52	22	12
-53	23	13
-54	24	14
-55	25	15
-56	26	16
-57	27	17
-58	28	18
-59	29	19
-60	0	0
-61	1	1
-62	2	2
-63	3	3
-64	4	4
-65	5	5
-66	6	6
-67	7	7
-68	8	8
-69	9	9
-70	10	10
-71	11	11
-72	12	12
-73	13	13
-74	14	14
-75	15	15
-76	16	16
-77	17	17
-78	18	18
-79	19	19
-80	20	0
-81	21	1
-82	22	2
-83	23	3
-84	24	4
-85	25	5
-86	26	6
-87	27	7
-88	28	8
-89	29	9
-90	0	10
-91	1	11
-92	2	12
-93	3	13
-94	4	14
-95	5	15
-96	6	16
-97	7	17
-98	8	18
-99	9	19
-100	10	0
-101	11	1
-102	12	2
-103	13	3
-104	14	4
-105	15	5
-106	16	6
-107	17	7
-108	18	8
-109	19	9
-110	20	10
-111	21	11
-112	22	12
-113	23	13
-114	24	14
-115	25	15
-116	26	16
-117	27	17
-118	28	18
-119	29	19
-120	0	0
-121	1	1
-122	2	2
-123	3	3
-124	4	4
-125	5	5
-126	6	6
-127	7	7
-128	8	8
-129	9	9
-130	10	10
-131	11	11
-132	12	12
-133	13	13
-134	14	14
-135	15	15
-136	16	16
-137	17	17
-138	18	18
-139	19	19
-140	20	0
-141	21	1
-142	22	2
-143	23	3
-144	24	4
-145	25	5
-146	26	6
-147	27	7
-148	28	8
-149	29	9
-150	0	10
-151	1	11
-152	2	12
-153	3	13
-154	4	14
-155	5	15
-156	6	16
-157	7	17
-158	8	18
-159	9	19
-160	10	0
-161	11	1
-162	12	2
-163	13	3
-164	14	4
-165	15	5
-166	16	6
-167	17	7
-168	18	8
-169	19	9
-170	20	10
-171	21	11
-172	22	12
-173	23	13
-174	24	14
-175	25	15
-176	26	16
-177	27	17
-178	28	18
-179	29	19
-180	0	0
-181	1	1
-182	2	2
-183	3	3
-184	4	4
-185	5	5
-186	6	6
-187	7	7
-188	8	8
-189	9	9
-190	10	10
-191	11	11
-192	12	12
-193	13	13
-194	14	14
-195	15	15
-196	16	16
-197	17	17
-198	18	18
-199	19	19
-200	20	\N
-201	21	\N
-202	22	\N
-203	23	\N
-204	24	\N
-205	25	\N
-206	26	\N
-207	27	\N
-208	28	\N
-209	29	\N
-210	0	\N
-211	1	\N
-212	2	\N
-213	3	\N
-214	4	\N
-215	5	\N
-216	6	\N
-217	7	\N
-218	8	\N
-219	9	\N
-220	10	\N
-221	11	\N
-222	12	\N
-223	13	\N
-224	14	\N
-225	15	\N
-226	16	\N
-227	17	\N
-228	18	\N
-229	19	\N
-230	20	\N
-231	21	\N
-232	22	\N
-233	23	\N
-234	24	\N
-235	25	\N
-236	26	\N
-237	27	\N
-238	28	\N
-239	29	\N
-240	0	\N
-241	1	\N
-242	2	\N
-243	3	\N
-244	4	\N
-245	5	\N
-246	6	\N
-247	7	\N
-248	8	\N
-249	9	\N
-250	10	\N
-251	11	\N
-252	12	\N
-253	13	\N
-254	14	\N
-255	15	\N
-256	16	\N
-257	17	\N
-258	18	\N
-259	19	\N
-260	20	\N
-261	21	\N
-262	22	\N
-263	23	\N
-264	24	\N
-265	25	\N
-266	26	\N
-267	27	\N
-268	28	\N
-269	29	\N
-270	0	\N
-271	1	\N
-272	2	\N
-273	3	\N
-274	4	\N
-275	5	\N
-276	6	\N
-277	7	\N
-278	8	\N
-279	9	\N
-280	10	\N
-281	11	\N
-282	12	\N
-283	13	\N
-284	14	\N
-285	15	\N
-286	16	\N
-287	17	\N
-288	18	\N
-289	19	\N
-290	20	\N
-291	21	\N
-292	22	\N
-293	23	\N
-294	24	\N
-295	25	\N
-296	26	\N
-297	27	\N
-298	28	\N
-299	29	\N
-right join 1
-0	\N	0
-0	\N	0
-0	\N	0
-0	\N	0
-0	\N	0
-0	\N	1
-0	\N	1
-0	\N	1
-0	\N	1
-0	\N	1
-0	\N	2
-0	\N	2
-0	\N	2
-0	\N	2
-0	\N	2
-0	\N	3
-0	\N	3
-0	\N	3
-0	\N	3
-0	\N	3
-0	\N	4
-0	\N	4
-0	\N	4
-0	\N	4
-0	\N	4
-0	\N	5
-0	\N	5
-0	\N	5
-0	\N	5
-0	\N	5
-0	\N	6
-0	\N	6
-0	\N	6
-0	\N	6
-0	\N	6
-0	\N	7
-0	\N	7
-0	\N	7
-0	\N	7
-0	\N	7
-0	\N	8
-0	\N	8
-0	\N	8
-0	\N	8
-0	\N	8
-0	\N	9
-0	\N	9
-0	\N	9
-0	\N	9
-0	\N	9
-0	\N	10
-0	\N	10
-0	\N	10
-0	\N	10
-0	\N	10
-0	\N	11
-0	\N	11
-0	\N	11
-0	\N	11
-0	\N	11
-0	\N	12
-0	\N	12
-0	\N	12
-0	\N	12
-0	\N	12
-0	\N	13
-0	\N	13
-0	\N	13
-0	\N	13
-0	\N	13
-0	\N	14
-0	\N	14
-0	\N	14
-0	\N	14
-0	\N	14
-0	\N	15
-0	\N	15
-0	\N	15
-0	\N	15
-0	\N	15
-0	\N	16
-0	\N	16
-0	\N	16
-0	\N	16
-0	\N	16
-0	\N	17
-0	\N	17
-0	\N	17
-0	\N	17
-0	\N	17
-0	\N	18
-0	\N	18
-0	\N	18
-0	\N	18
-0	\N	18
-0	\N	19
-0	\N	19
-0	\N	19
-0	\N	19
-0	\N	19
-0	\N	20
-0	\N	20
-0	\N	20
-0	\N	20
-0	\N	20
-0	\N	20
-0	\N	21
-0	\N	21
-0	\N	21
-0	\N	21
-0	\N	21
-0	\N	22
-0	\N	22
-0	\N	22
-0	\N	22
-0	\N	22
-0	\N	23
-0	\N	23
-0	\N	23
-0	\N	23
-0	\N	23
-0	\N	24
-0	\N	24
-0	\N	24
-0	\N	24
-0	\N	24
-0	\N	25
-0	\N	25
-0	\N	25
-0	\N	25
-0	\N	25
-0	\N	26
-0	\N	26
-0	\N	26
-0	\N	26
-0	\N	26
-0	\N	27
-0	\N	27
-0	\N	27
-0	\N	27
-0	\N	27
-0	\N	28
-0	\N	28
-0	\N	28
-0	\N	28
-0	\N	28
-0	\N	29
-0	\N	29
-0	\N	29
-0	\N	29
-0	\N	29
-51	11	21
-52	12	22
-53	13	23
-54	14	24
-55	15	25
-56	16	26
-57	17	27
-58	18	28
-59	19	29
-60	0	0
-61	1	1
-62	2	2
-63	3	3
-64	4	4
-65	5	5
-66	6	6
-67	7	7
-68	8	8
-69	9	9
-70	10	10
-71	11	11
-72	12	12
-73	13	13
-74	14	14
-75	15	15
-76	16	16
-77	17	17
-78	18	18
-79	19	19
-80	0	20
-81	1	21
-82	2	22
-83	3	23
-84	4	24
-85	5	25
-86	6	26
-87	7	27
-88	8	28
-89	9	29
-90	10	0
-91	11	1
-92	12	2
-93	13	3
-94	14	4
-95	15	5
-96	16	6
-97	17	7
-98	18	8
-99	19	9
-100	0	10
-101	1	11
-102	2	12
-103	3	13
-104	4	14
-105	5	15
-106	6	16
-107	7	17
-108	8	18
-109	9	19
-110	10	20
-111	11	21
-112	12	22
-113	13	23
-114	14	24
-115	15	25
-116	16	26
-117	17	27
-118	18	28
-119	19	29
-120	0	0
-121	1	1
-122	2	2
-123	3	3
-124	4	4
-125	5	5
-126	6	6
-127	7	7
-128	8	8
-129	9	9
-130	10	10
-131	11	11
-132	12	12
-133	13	13
-134	14	14
-135	15	15
-136	16	16
-137	17	17
-138	18	18
-139	19	19
-140	0	20
-141	1	21
-142	2	22
-143	3	23
-144	4	24
-145	5	25
-146	6	26
-147	7	27
-148	8	28
-149	9	29
-150	10	0
-151	11	1
-152	12	2
-153	13	3
-154	14	4
-155	15	5
-156	16	6
-157	17	7
-158	18	8
-159	19	9
-160	0	10
-161	1	11
-162	2	12
-163	3	13
-164	4	14
-165	5	15
-166	6	16
-167	7	17
-168	8	18
-169	9	19
-170	10	20
-171	11	21
-172	12	22
-173	13	23
-174	14	24
-175	15	25
-176	16	26
-177	17	27
-178	18	28
-179	19	29
-180	0	0
-181	1	1
-182	2	2
-183	3	3
-184	4	4
-185	5	5
-186	6	6
-187	7	7
-188	8	8
-189	9	9
-190	10	10
-191	11	11
-192	12	12
-193	13	13
-194	14	14
-195	15	15
-196	16	16
-197	17	17
-198	18	18
-199	19	19
-right join 2
-51	21	11
-52	22	12
-53	23	13
-54	24	14
-55	25	15
-56	26	16
-57	27	17
-58	28	18
-59	29	19
-60	0	0
-61	1	1
-62	2	2
-63	3	3
-64	4	4
-65	5	5
-66	6	6
-67	7	7
-68	8	8
-69	9	9
-70	10	10
-71	11	11
-72	12	12
-73	13	13
-74	14	14
-75	15	15
-76	16	16
-77	17	17
-78	18	18
-79	19	19
-80	20	0
-81	21	1
-82	22	2
-83	23	3
-84	24	4
-85	25	5
-86	26	6
-87	27	7
-88	28	8
-89	29	9
-90	0	10
-91	1	11
-92	2	12
-93	3	13
-94	4	14
-95	5	15
-96	6	16
-97	7	17
-98	8	18
-99	9	19
-100	10	0
-101	11	1
-102	12	2
-103	13	3
-104	14	4
-105	15	5
-106	16	6
-107	17	7
-108	18	8
-109	19	9
-110	20	10
-111	21	11
-112	22	12
-113	23	13
-114	24	14
-115	25	15
-116	26	16
-117	27	17
-118	28	18
-119	29	19
-120	0	0
-121	1	1
-122	2	2
-123	3	3
-124	4	4
-125	5	5
-126	6	6
-127	7	7
-128	8	8
-129	9	9
-130	10	10
-131	11	11
-132	12	12
-133	13	13
-134	14	14
-135	15	15
-136	16	16
-137	17	17
-138	18	18
-139	19	19
-140	20	0
-141	21	1
-142	22	2
-143	23	3
-144	24	4
-145	25	5
-146	26	6
-147	27	7
-148	28	8
-149	29	9
-150	0	10
-151	1	11
-152	2	12
-153	3	13
-154	4	14
-155	5	15
-156	6	16
-157	7	17
-158	8	18
-159	9	19
-160	10	0
-161	11	1
-162	12	2
-163	13	3
-164	14	4
-165	15	5
-166	16	6
-167	17	7
-168	18	8
-169	19	9
-170	20	10
-171	21	11
-172	22	12
-173	23	13
-174	24	14
-175	25	15
-176	26	16
-177	27	17
-178	28	18
-179	29	19
-180	0	0
-181	1	1
-182	2	2
-183	3	3
-184	4	4
-185	5	5
-186	6	6
-187	7	7
-188	8	8
-189	9	9
-190	10	10
-191	11	11
-192	12	12
-193	13	13
-194	14	14
-195	15	15
-196	16	16
-197	17	17
-198	18	18
-199	19	19
-full join 1
-0	\N	0
-0	\N	0
-0	\N	0
-0	\N	0
-0	\N	0
-0	\N	1
-0	\N	1
-0	\N	1
-0	\N	1
-0	\N	1
-0	\N	2
-0	\N	2
-0	\N	2
-0	\N	2
-0	\N	2
-0	\N	3
-0	\N	3
-0	\N	3
-0	\N	3
-0	\N	3
-0	\N	4
-0	\N	4
-0	\N	4
-0	\N	4
-0	\N	4
-0	\N	5
-0	\N	5
-0	\N	5
-0	\N	5
-0	\N	5
-0	\N	6
-0	\N	6
-0	\N	6
-0	\N	6
-0	\N	6
-0	\N	7
-0	\N	7
-0	\N	7
-0	\N	7
-0	\N	7
-0	\N	8
-0	\N	8
-0	\N	8
-0	\N	8
-0	\N	8
-0	\N	9
-0	\N	9
-0	\N	9
-0	\N	9
-0	\N	9
-0	\N	10
-0	\N	10
-0	\N	10
-0	\N	10
-0	\N	10
-0	\N	11
-0	\N	11
-0	\N	11
-0	\N	11
-0	\N	11
-0	\N	12
-0	\N	12
-0	\N	12
-0	\N	12
-0	\N	12
-0	\N	13
-0	\N	13
-0	\N	13
-0	\N	13
-0	\N	13
-0	\N	14
-0	\N	14
-0	\N	14
-0	\N	14
-0	\N	14
-0	\N	15
-0	\N	15
-0	\N	15
-0	\N	15
-0	\N	15
-0	\N	16
-0	\N	16
-0	\N	16
-0	\N	16
-0	\N	16
-0	\N	17
-0	\N	17
-0	\N	17
-0	\N	17
-0	\N	17
-0	\N	18
-0	\N	18
-0	\N	18
-0	\N	18
-0	\N	18
-0	\N	19
-0	\N	19
-0	\N	19
-0	\N	19
-0	\N	19
-0	\N	20
-0	\N	20
-0	\N	20
-0	\N	20
-0	\N	20
-0	\N	20
-0	\N	21
-0	\N	21
-0	\N	21
-0	\N	21
-0	\N	21
-0	\N	22
-0	\N	22
-0	\N	22
-0	\N	22
-0	\N	22
-0	\N	23
-0	\N	23
-0	\N	23
-0	\N	23
-0	\N	23
-0	\N	24
-0	\N	24
-0	\N	24
-0	\N	24
-0	\N	24
-0	\N	25
-0	\N	25
-0	\N	25
-0	\N	25
-0	\N	25
-0	\N	26
-0	\N	26
-0	\N	26
-0	\N	26
-0	\N	26
-0	\N	27
-0	\N	27
-0	\N	27
-0	\N	27
-0	\N	27
-0	\N	28
-0	\N	28
-0	\N	28
-0	\N	28
-0	\N	28
-0	\N	29
-0	\N	29
-0	\N	29
-0	\N	29
-0	\N	29
-51	11	21
-52	12	22
-53	13	23
-54	14	24
-55	15	25
-56	16	26
-57	17	27
-58	18	28
-59	19	29
-60	0	0
-61	1	1
-62	2	2
-63	3	3
-64	4	4
-65	5	5
-66	6	6
-67	7	7
-68	8	8
-69	9	9
-70	10	10
-71	11	11
-72	12	12
-73	13	13
-74	14	14
-75	15	15
-76	16	16
-77	17	17
-78	18	18
-79	19	19
-80	0	20
-81	1	21
-82	2	22
-83	3	23
-84	4	24
-85	5	25
-86	6	26
-87	7	27
-88	8	28
-89	9	29
-90	10	0
-91	11	1
-92	12	2
-93	13	3
-94	14	4
-95	15	5
-96	16	6
-97	17	7
-98	18	8
-99	19	9
-100	0	10
-101	1	11
-102	2	12
-103	3	13
-104	4	14
-105	5	15
-106	6	16
-107	7	17
-108	8	18
-109	9	19
-110	10	20
-111	11	21
-112	12	22
-113	13	23
-114	14	24
-115	15	25
-116	16	26
-117	17	27
-118	18	28
-119	19	29
-120	0	0
-121	1	1
-122	2	2
-123	3	3
-124	4	4
-125	5	5
-126	6	6
-127	7	7
-128	8	8
-129	9	9
-130	10	10
-131	11	11
-132	12	12
-133	13	13
-134	14	14
-135	15	15
-136	16	16
-137	17	17
-138	18	18
-139	19	19
-140	0	20
-141	1	21
-142	2	22
-143	3	23
-144	4	24
-145	5	25
-146	6	26
-147	7	27
-148	8	28
-149	9	29
-150	10	0
-151	11	1
-152	12	2
-153	13	3
-154	14	4
-155	15	5
-156	16	6
-157	17	7
-158	18	8
-159	19	9
-160	0	10
-161	1	11
-162	2	12
-163	3	13
-164	4	14
-165	5	15
-166	6	16
-167	7	17
-168	8	18
-169	9	19
-170	10	20
-171	11	21
-172	12	22
-173	13	23
-174	14	24
-175	15	25
-176	16	26
-177	17	27
-178	18	28
-179	19	29
-180	0	0
-181	1	1
-182	2	2
-183	3	3
-184	4	4
-185	5	5
-186	6	6
-187	7	7
-188	8	8
-189	9	9
-190	10	10
-191	11	11
-192	12	12
-193	13	13
-194	14	14
-195	15	15
-196	16	16
-197	17	17
-198	18	18
-199	19	19
-full join 2
-0	0	\N
-1	1	\N
-2	2	\N
-3	3	\N
-4	4	\N
-5	5	\N
-6	6	\N
-7	7	\N
-8	8	\N
-9	9	\N
-10	10	\N
-11	11	\N
-12	12	\N
-13	13	\N
-14	14	\N
-15	15	\N
-16	16	\N
-17	17	\N
-18	18	\N
-19	19	\N
-20	20	\N
-21	21	\N
-22	22	\N
-23	23	\N
-24	24	\N
-25	25	\N
-26	26	\N
-27	27	\N
-28	28	\N
-29	29	\N
-30	0	\N
-31	1	\N
-32	2	\N
-33	3	\N
-34	4	\N
-35	5	\N
-36	6	\N
-37	7	\N
-38	8	\N
-39	9	\N
-40	10	\N
-41	11	\N
-42	12	\N
-43	13	\N
-44	14	\N
-45	15	\N
-46	16	\N
-47	17	\N
-48	18	\N
-49	19	\N
-50	20	\N
-51	21	11
-52	22	12
-53	23	13
-54	24	14
-55	25	15
-56	26	16
-57	27	17
-58	28	18
-59	29	19
-60	0	0
-61	1	1
-62	2	2
-63	3	3
-64	4	4
-65	5	5
-66	6	6
-67	7	7
-68	8	8
-69	9	9
-70	10	10
-71	11	11
-72	12	12
-73	13	13
-74	14	14
-75	15	15
-76	16	16
-77	17	17
-78	18	18
-79	19	19
-80	20	0
-81	21	1
-82	22	2
-83	23	3
-84	24	4
-85	25	5
-86	26	6
-87	27	7
-88	28	8
-89	29	9
-90	0	10
-91	1	11
-92	2	12
-93	3	13
-94	4	14
-95	5	15
-96	6	16
-97	7	17
-98	8	18
-99	9	19
-100	10	0
-101	11	1
-102	12	2
-103	13	3
-104	14	4
-105	15	5
-106	16	6
-107	17	7
-108	18	8
-109	19	9
-110	20	10
-111	21	11
-112	22	12
-113	23	13
-114	24	14
-115	25	15
-116	26	16
-117	27	17
-118	28	18
-119	29	19
-120	0	0
-121	1	1
-122	2	2
-123	3	3
-124	4	4
-125	5	5
-126	6	6
-127	7	7
-128	8	8
-129	9	9
-130	10	10
-131	11	11
-132	12	12
-133	13	13
-134	14	14
-135	15	15
-136	16	16
-137	17	17
-138	18	18
-139	19	19
-140	20	0
-141	21	1
-142	22	2
-143	23	3
-144	24	4
-145	25	5
-146	26	6
-147	27	7
-148	28	8
-149	29	9
-150	0	10
-151	1	11
-152	2	12
-153	3	13
-154	4	14
-155	5	15
-156	6	16
-157	7	17
-158	8	18
-159	9	19
-160	10	0
-161	11	1
-162	12	2
-163	13	3
-164	14	4
-165	15	5
-166	16	6
-167	17	7
-168	18	8
-169	19	9
-170	20	10
-171	21	11
-172	22	12
-173	23	13
-174	24	14
-175	25	15
-176	26	16
-177	27	17
-178	28	18
-179	29	19
-180	0	0
-181	1	1
-182	2	2
-183	3	3
-184	4	4
-185	5	5
-186	6	6
-187	7	7
-188	8	8
-189	9	9
-190	10	10
-191	11	11
-192	12	12
-193	13	13
-194	14	14
-195	15	15
-196	16	16
-197	17	17
-198	18	18
-199	19	19
-200	20	\N
-201	21	\N
-202	22	\N
-203	23	\N
-204	24	\N
-205	25	\N
-206	26	\N
-207	27	\N
-208	28	\N
-209	29	\N
-210	0	\N
-211	1	\N
-212	2	\N
-213	3	\N
-214	4	\N
-215	5	\N
-216	6	\N
-217	7	\N
-218	8	\N
-219	9	\N
-220	10	\N
-221	11	\N
-222	12	\N
-223	13	\N
-224	14	\N
-225	15	\N
-226	16	\N
-227	17	\N
-228	18	\N
-229	19	\N
-230	20	\N
-231	21	\N
-232	22	\N
-233	23	\N
-234	24	\N
-235	25	\N
-236	26	\N
-237	27	\N
-238	28	\N
-239	29	\N
-240	0	\N
-241	1	\N
-242	2	\N
-243	3	\N
-244	4	\N
-245	5	\N
-246	6	\N
-247	7	\N
-248	8	\N
-249	9	\N
-250	10	\N
-251	11	\N
-252	12	\N
-253	13	\N
-254	14	\N
-255	15	\N
-256	16	\N
-257	17	\N
-258	18	\N
-259	19	\N
-260	20	\N
-261	21	\N
-262	22	\N
-263	23	\N
-264	24	\N
-265	25	\N
-266	26	\N
-267	27	\N
-268	28	\N
-269	29	\N
-270	0	\N
-271	1	\N
-272	2	\N
-273	3	\N
-274	4	\N
-275	5	\N
-276	6	\N
-277	7	\N
-278	8	\N
-279	9	\N
-280	10	\N
-281	11	\N
-282	12	\N
-283	13	\N
-284	14	\N
-285	15	\N
-286	16	\N
-287	17	\N
-288	18	\N
-289	19	\N
-290	20	\N
-291	21	\N
-292	22	\N
-293	23	\N
-294	24	\N
-295	25	\N
-296	26	\N
-297	27	\N
-298	28	\N
-299	29	\N
diff --git a/tests/queries/0_stateless/02724_grace_hash_join.sql b/tests/queries/0_stateless/02724_grace_hash_join.sql
deleted file mode 100644
index 370f6058bef..00000000000
--- a/tests/queries/0_stateless/02724_grace_hash_join.sql
+++ /dev/null
@@ -1,44 +0,0 @@
-set join_algorithm = 'grace_hash';
-set max_rows_in_join = 100;
-
-drop table if exists test_t1;
-drop table if exists test_t2;
-
-CREATE TABLE test_t1
-(
-    `x` UInt64,
-    `y` Nullable(UInt64)
-)
-ENGINE = Memory;
-
-CREATE TABLE test_t2
-(
-    `x` UInt64,
-    `y` Nullable(UInt64)
-)
-ENGINE = Memory;
-
-insert into test_t1 select number as x, number % 20 as y from numbers(200) where number > 50 order by y;
-insert into test_t2 select number as x , number % 30 as y from numbers(300) order by y;
-
-select 'inner join 1';
-select l.x, l.y, r.y from test_t1 as l inner join test_t2 as r on l.x = r.x order by l.x, l.y, r.y;
-select 'inner join 2';
-select l.x, l.y, r.y from test_t2 as l inner join test_t1 as r on l.x = r.x order by l.x, l.y, r.y;
-select 'left join 1';
-select l.x, l.y, r.y from test_t1 as l left join test_t2 as r on l.x = r.x order by l.x, l.y, r.y;
-select 'left join 2';
-select l.x, l.y, r.y from test_t2 as l left join test_t1 as r on l.x = r.x order by l.x, l.y, r.y;
-select 'right join 1';
-select l.x, l.y, r.y from test_t1 as l right join test_t2 as r on l.x = r.x order by l.x, l.y, r.y;
-select 'right join 2';
-select l.x, l.y, r.y from test_t2 as l right join test_t1 as r on l.x = r.x order by l.x, l.y, r.y;
-select 'full join 1';
-select l.x, l.y, r.y from test_t1 as l full join test_t2 as r on l.x = r.x order by l.x, l.y, r.y;
-select 'full join 2';
-select l.x, l.y, r.y from test_t2 as l full join test_t1 as r on l.x = r.x order by l.x, l.y, r.y;
-
-
-
-
-

From d89beb1bf7ddfa782eb7420aa5e148de5237ffc6 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Sat, 6 May 2023 09:05:23 +0800
Subject: [PATCH 677/689] update tests

---
 src/Interpreters/GraceHashJoin.cpp            | 15 ++-----
 src/Interpreters/GraceHashJoin.h              |  2 -
 ...01721_join_implicit_cast_long.reference.j2 | 42 -------------------
 .../02273_full_sort_join.reference.j2         | 16 -------
 ...274_full_sort_join_nodistinct.reference.j2 | 32 --------------
 5 files changed, 3 insertions(+), 104 deletions(-)

diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index 407663ac310..ee369d4b195 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -524,16 +524,7 @@ private:
 IBlocksStreamPtr
 GraceHashJoin::getNonJoinedBlocks(const Block & left_sample_block_, const Block & result_sample_block_, UInt64 max_block_size_) const
 {
-    if (!has_initialized_non_joined_blocks) [[unlikely]]
-    {
-        std::lock_guard lock(hash_join_mutex);
-        if (!has_initialized_non_joined_blocks)
-        {
-            has_initialized_non_joined_blocks = true;
-            non_joined_blocks = std::make_shared<NonJoinedBlocksStream>(hash_join, left_sample_block_, result_sample_block_, max_block_size_);
-        }
-    }
-    return non_joined_blocks;
+    return std::make_shared<NonJoinedBlocksStream>(hash_join, left_sample_block_, result_sample_block_, max_block_size_);
 }
 
 class GraceHashJoin::DelayedBlocks : public IBlocksStream
@@ -580,7 +571,7 @@ public:
                     is_left_reader_finished = true;
                 }
             }
-            if (is_left_reader_finished)
+            if (is_left_reader_finished || !block)
             {
                 return non_joined_blocks_iter.next();
             }
@@ -625,7 +616,7 @@ public:
 
     Names left_key_names;
     Names right_key_names;
-    bool is_left_reader_finished = false;
+    std::atomic<bool> is_left_reader_finished = false;
     NonJoinedBlocksStream non_joined_blocks_iter;
 };
 
diff --git a/src/Interpreters/GraceHashJoin.h b/src/Interpreters/GraceHashJoin.h
index ec0e5ac05aa..bc5f894bc8c 100644
--- a/src/Interpreters/GraceHashJoin.h
+++ b/src/Interpreters/GraceHashJoin.h
@@ -145,8 +145,6 @@ private:
     InMemoryJoinPtr hash_join;
     Block hash_join_sample_block;
     mutable std::mutex hash_join_mutex;
-    mutable bool has_initialized_non_joined_blocks = false;
-    mutable std::shared_ptr<NonJoinedBlocksStream> non_joined_blocks = nullptr;
 };
 
 }
diff --git a/tests/queries/0_stateless/01721_join_implicit_cast_long.reference.j2 b/tests/queries/0_stateless/01721_join_implicit_cast_long.reference.j2
index e9f32087439..d40fc7ec864 100644
--- a/tests/queries/0_stateless/01721_join_implicit_cast_long.reference.j2
+++ b/tests/queries/0_stateless/01721_join_implicit_cast_long.reference.j2
@@ -1,7 +1,6 @@
 {% for join_algorithm in ['hash', 'partial_merge', 'auto', 'full_sorting_merge', 'grace_hash'] -%}
 === {{ join_algorithm }} ===
 = full =
-{% if join_algorithm not in ['grace_hash'] -%}
 -4	0	196
 -3	0	197
 -2	0	198
@@ -17,7 +16,6 @@
 8	108	\N
 9	109	\N
 10	110	\N
-{% endif -%}
 = left =
 1	101	201
 2	102	202
@@ -30,7 +28,6 @@
 9	109	\N
 10	110	\N
 = right =
-{% if join_algorithm not in ['grace_hash'] -%}
 -4	0	196
 -3	0	197
 -2	0	198
@@ -41,7 +38,6 @@
 3	103	203
 4	104	204
 5	105	205
-{% endif -%}
 = inner =
 1	101	201
 2	102	202
@@ -49,7 +45,6 @@
 4	104	204
 5	105	205
 = full =
-{% if join_algorithm not in ['grace_hash'] -%}
 0	0	-4
 0	0	-3
 0	0	-2
@@ -65,7 +60,6 @@
 8	8	0
 9	9	0
 10	10	0
-{% endif -%}
 = left =
 1	1	1
 2	2	2
@@ -78,7 +72,6 @@
 9	9	0
 10	10	0
 = right =
-{% if join_algorithm not in ['grace_hash'] -%}
 0	0	-4
 0	0	-3
 0	0	-2
@@ -89,7 +82,6 @@
 3	3	3
 4	4	4
 5	5	5
-{% endif -%}
 = inner =
 1	1	1
 2	2	2
@@ -98,7 +90,6 @@
 5	5	5
 = join on =
 = full =
-{% if join_algorithm not in ['grace_hash'] -%}
 0	0	-4	196
 0	0	-3	197
 0	0	-2	198
@@ -114,7 +105,6 @@
 8	108	0	\N
 9	109	0	\N
 10	110	0	\N
-{% endif -%}
 = left =
 1	101	1	201
 2	102	2	202
@@ -127,7 +117,6 @@
 9	109	0	\N
 10	110	0	\N
 = right =
-{% if join_algorithm not in ['grace_hash'] -%}
 0	0	-4	196
 0	0	-3	197
 0	0	-2	198
@@ -138,7 +127,6 @@
 3	103	3	203
 4	104	4	204
 5	105	5	205
-{% endif -%}
 = inner =
 1	101	1	201
 2	102	2	202
@@ -146,7 +134,6 @@
 4	104	4	204
 5	105	5	205
 = full =
-{% if join_algorithm not in ['grace_hash'] -%}
 0	0	-4	196
 0	0	-3	197
 0	0	-2	198
@@ -162,7 +149,6 @@
 8	108	0	\N
 9	109	0	\N
 10	110	0	\N
-{% endif -%}
 = left =
 1	101	1	201
 2	102	2	202
@@ -175,7 +161,6 @@
 9	109	0	\N
 10	110	0	\N
 = right =
-{% if join_algorithm not in ['grace_hash'] -%}
 0	0	-4	196
 0	0	-3	197
 0	0	-2	198
@@ -186,7 +171,6 @@
 3	103	3	203
 4	104	4	204
 5	105	5	205
-{% endif -%}
 = inner =
 1	101	1	201
 2	102	2	202
@@ -196,7 +180,6 @@
 = agg =
 1
 1
-{% if join_algorithm not in ['grace_hash'] -%}
 1
 1
 1
@@ -205,13 +188,11 @@
 1	55	1055
 0	0	-10	0	990
 1	55	15	1055	1015
-{% endif -%}
 = types =
 1
 1
 1
 1
-{% if join_algorithm not in ['grace_hash'] -%}
 1
 1
 1
@@ -219,11 +200,8 @@
 1
 1
 1
-{% endif -%}
-{% if join_algorithm not in ['full_sorting_merge'] -%}
 === join use nulls ===
 = full =
-{% if join_algorithm not in ['grace_hash'] -%}
 -4	\N	196
 -3	\N	197
 -2	\N	198
@@ -239,7 +217,6 @@
 8	108	\N
 9	109	\N
 10	110	\N
-{% endif -%}
 = left =
 1	101	201
 2	102	202
@@ -252,7 +229,6 @@
 9	109	\N
 10	110	\N
 = right =
-{% if join_algorithm not in ['grace_hash'] -%}
 -4	\N	196
 -3	\N	197
 -2	\N	198
@@ -263,7 +239,6 @@
 3	103	203
 4	104	204
 5	105	205
-{% endif -%}
 = inner =
 1	101	201
 2	102	202
@@ -271,7 +246,6 @@
 4	104	204
 5	105	205
 = full =
-{% if join_algorithm not in ['grace_hash'] -%}
 1	1	1
 2	2	2
 3	3	3
@@ -287,7 +261,6 @@
 \N	\N	-2
 \N	\N	-1
 \N	\N	0
-{% endif -%}
 = left =
 1	1	1
 2	2	2
@@ -300,7 +273,6 @@
 9	9	\N
 10	10	\N
 = right =
-{% if join_algorithm not in ['grace_hash'] -%}
 1	1	1
 2	2	2
 3	3	3
@@ -311,7 +283,6 @@
 \N	\N	-2
 \N	\N	-1
 \N	\N	0
-{% endif -%}
 = inner =
 1	1	1
 2	2	2
@@ -320,7 +291,6 @@
 5	5	5
 = join on =
 = full =
-{% if join_algorithm not in ['grace_hash'] -%}
 1	101	1	201
 2	102	2	202
 3	103	3	203
@@ -336,7 +306,6 @@
 \N	\N	-2	198
 \N	\N	-1	199
 \N	\N	0	200
-{% endif -%}
 = left =
 1	101	1	201
 2	102	2	202
@@ -349,7 +318,6 @@
 9	109	\N	\N
 10	110	\N	\N
 = right =
-{% if join_algorithm not in ['grace_hash'] -%}
 1	101	1	201
 2	102	2	202
 3	103	3	203
@@ -360,7 +328,6 @@
 \N	\N	-2	198
 \N	\N	-1	199
 \N	\N	0	200
-{% endif -%}
 = inner =
 1	101	1	201
 2	102	2	202
@@ -368,7 +335,6 @@
 4	104	4	204
 5	105	5	205
 = full =
-{% if join_algorithm not in ['grace_hash'] -%}
 1	101	1	201
 2	102	2	202
 3	103	3	203
@@ -384,7 +350,6 @@
 \N	\N	-2	198
 \N	\N	-1	199
 \N	\N	0	200
-{% endif -%}
 = left =
 1	101	1	201
 2	102	2	202
@@ -397,7 +362,6 @@
 9	109	\N	\N
 10	110	\N	\N
 = right =
-{% if join_algorithm not in ['grace_hash'] -%}
 1	101	1	201
 2	102	2	202
 3	103	3	203
@@ -408,7 +372,6 @@
 \N	\N	-2	198
 \N	\N	-1	199
 \N	\N	0	200
-{% endif -%}
 = inner =
 1	101	1	201
 2	102	2	202
@@ -418,7 +381,6 @@
 = agg =
 1
 1
-{% if join_algorithm not in ['grace_hash'] -%}
 1
 1
 1
@@ -427,13 +389,11 @@
 1	55	1055
 1	55	15	1055	1015
 \N	\N	-10	\N	990
-{% endif -%}
 = types =
 1
 1
 1
 1
-{% if join_algorithm not in ['grace_hash'] -%}
 1
 1
 1
@@ -441,6 +401,4 @@
 1
 1
 1
-{% endif -%}
-{% endif -%}
 {% endfor -%}
diff --git a/tests/queries/0_stateless/02273_full_sort_join.reference.j2 b/tests/queries/0_stateless/02273_full_sort_join.reference.j2
index 98bfd9d9b2b..7b994c2d11f 100644
--- a/tests/queries/0_stateless/02273_full_sort_join.reference.j2
+++ b/tests/queries/0_stateless/02273_full_sort_join.reference.j2
@@ -50,7 +50,6 @@ ALL LEFT | bs = {{ block_size }}
 14	14	val9	0
 14	14	val9	0
 ALL RIGHT | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 4	4	0	val10
 5	5	0	val6
 6	6	0	val8
@@ -64,7 +63,6 @@ ALL RIGHT | bs = {{ block_size }}
 13	13	0	val9
 14	14	0	val3
 14	14	0	val7
-{% endif -%}
 ALL INNER | bs = {{ block_size }} | copmosite key
 2	2	2	2	2	2	0	0
 2	2	2	2	2	2	0	0
@@ -85,7 +83,6 @@ ALL LEFT | bs = {{ block_size }} | copmosite key
 2	2	2	2	2	2	val12	0
 2	2	2	2	2	2	val9	0
 ALL RIGHT | bs = {{ block_size }} | copmosite key
-{% if join_algorithm != 'grace_hash' -%}
 0	\N	0	1	1	1	1	val2
 0	\N	0	1	1	1	1	val7
 0	\N	0	1	1	2	1	val5
@@ -99,7 +96,6 @@ ALL RIGHT | bs = {{ block_size }} | copmosite key
 0	\N	0	2	2	\N	1	val9
 2	2	2	2	2	2	0	val4
 2	2	2	2	2	2	0	val4
-{% endif -%}
 ANY INNER USING | bs = {{ block_size }}
 4	0	0
 5	0	0
@@ -137,7 +133,6 @@ ANY LEFT | bs = {{ block_size }}
 13	13	val13	0
 14	14	val9	0
 ANY RIGHT | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 4	4	0	val10
 5	5	0	val6
 6	6	0	val8
@@ -150,7 +145,6 @@ ANY RIGHT | bs = {{ block_size }}
 13	13	0	val9
 14	14	0	val3
 14	14	0	val7
-{% endif -%}
 ANY INNER | bs = {{ block_size }} | copmosite key
 2	2	2	2	2	2	0	0
 ANY LEFT | bs = {{ block_size }} | copmosite key
@@ -170,7 +164,6 @@ ANY LEFT | bs = {{ block_size }} | copmosite key
 2	2	2	2	2	2	val12	0
 2	2	2	2	2	2	val9	0
 ANY RIGHT | bs = {{ block_size }} | copmosite key
-{% if join_algorithm != 'grace_hash' -%}
 0	\N	0	1	1	1	1	val2
 0	\N	0	1	1	1	1	val7
 0	\N	0	1	1	2	1	val5
@@ -183,7 +176,6 @@ ANY RIGHT | bs = {{ block_size }} | copmosite key
 0	\N	0	2	1	\N	1	val3
 0	\N	0	2	2	\N	1	val9
 2	2	2	2	2	2	0	val4
-{% endif -%}
 {% endfor -%}
 ALL INNER | join_use_nulls = 1
 4	4	0	0
@@ -219,7 +211,6 @@ ALL LEFT | join_use_nulls = 1
 14	14	val9	0
 14	14	val9	0
 ALL RIGHT | join_use_nulls = 1
-{% if join_algorithm != 'grace_hash' -%}
 4	4	0	val10
 5	5	0	val6
 6	6	0	val8
@@ -233,7 +224,6 @@ ALL RIGHT | join_use_nulls = 1
 13	13	0	val9
 14	14	0	val3
 14	14	0	val7
-{% endif -%}
 ALL INNER | join_use_nulls = 1 | copmosite key
 2	2	2	2	2	2	0	0
 2	2	2	2	2	2	0	0
@@ -254,7 +244,6 @@ ALL LEFT | join_use_nulls = 1 | copmosite key
 2	2	2	2	2	2	val12	0
 2	2	2	2	2	2	val9	0
 ALL RIGHT | join_use_nulls = 1 | copmosite key
-{% if join_algorithm != 'grace_hash' -%}
 2	2	2	2	2	2	0	val4
 2	2	2	2	2	2	0	val4
 \N	\N	\N	1	1	1	\N	val2
@@ -268,7 +257,6 @@ ALL RIGHT | join_use_nulls = 1 | copmosite key
 \N	\N	\N	2	1	2	\N	val8
 \N	\N	\N	2	1	\N	\N	val3
 \N	\N	\N	2	2	\N	\N	val9
-{% endif -%}
 ANY INNER | join_use_nulls = 1
 4	4	0	0
 5	5	0	0
@@ -296,7 +284,6 @@ ANY LEFT | join_use_nulls = 1
 13	13	val13	0
 14	14	val9	0
 ANY RIGHT | join_use_nulls = 1
-{% if join_algorithm != 'grace_hash' -%}
 4	4	0	val10
 5	5	0	val6
 6	6	0	val8
@@ -309,7 +296,6 @@ ANY RIGHT | join_use_nulls = 1
 13	13	0	val9
 14	14	0	val3
 14	14	0	val7
-{% endif -%}
 ANY INNER | join_use_nulls = 1 | copmosite key
 2	2	2	2	2	2	0	0
 ANY LEFT | join_use_nulls = 1 | copmosite key
@@ -329,7 +315,6 @@ ANY LEFT | join_use_nulls = 1 | copmosite key
 2	2	2	2	2	2	val12	0
 2	2	2	2	2	2	val9	0
 ANY RIGHT | join_use_nulls = 1 | copmosite key
-{% if join_algorithm != 'grace_hash' -%}
 2	2	2	2	2	2	0	val4
 \N	\N	\N	1	1	1	\N	val2
 \N	\N	\N	1	1	1	\N	val7
@@ -342,5 +327,4 @@ ANY RIGHT | join_use_nulls = 1 | copmosite key
 \N	\N	\N	2	1	2	\N	val8
 \N	\N	\N	2	1	\N	\N	val3
 \N	\N	\N	2	2	\N	\N	val9
-{% endif -%}
 {% endfor -%}
diff --git a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2 b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2
index 2cc6c6e85d6..6b5b5bfd298 100644
--- a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2
+++ b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2
@@ -108,7 +108,6 @@ t1 ALL LEFT JOIN t2 | bs = {{ block_size }}
 2	2	val27	5
 3	3	val3	4
 t1 ALL RIGHT JOIN t2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 1	1	4	val11
 1	1	4	val12
 2	2	5	val22
@@ -161,7 +160,6 @@ t1 ALL RIGHT JOIN t2 | bs = {{ block_size }}
 2	2	5	val28
 2	2	5	val28
 3	3	4	val3
-{% endif -%}
 t1 ANY INNER JOIN t2 | bs = {{ block_size }}
 1	1	4	5
 2	2	5	5
@@ -177,7 +175,6 @@ t1 ANY LEFT JOIN t2 | bs = {{ block_size }}
 2	2	val27	5
 3	3	val3	4
 t1 ANY RIGHT JOIN t2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 1	1	4	val11
 1	1	4	val12
 2	2	5	val22
@@ -188,9 +185,7 @@ t1 ANY RIGHT JOIN t2 | bs = {{ block_size }}
 2	2	5	val27
 2	2	5	val28
 3	3	4	val3
-{% endif -%}
 t1 ALL FULL JOIN t2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 1	1	4	5
 1	1	4	5
 2	2	5	5
@@ -243,9 +238,7 @@ t1 ALL FULL JOIN t2 | bs = {{ block_size }}
 2	2	5	5
 2	2	5	5
 3	3	4	4
-{% endif -%}
 t1 ALL FULL JOIN USING t2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 1	4	5
 1	4	5
 2	5	5
@@ -298,7 +291,6 @@ t1 ALL FULL JOIN USING t2 | bs = {{ block_size }}
 2	5	5
 2	5	5
 3	4	4
-{% endif -%}
 t1 ALL INNER JOIN tn2 | bs = {{ block_size }}
 1	1	4	5
 1	1	4	5
@@ -315,7 +307,6 @@ t1 ALL LEFT JOIN tn2 | bs = {{ block_size }}
 2	\N	val27	0
 3	3	val3	4
 t1 ALL RIGHT JOIN tn2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 0	\N	0	val22
 0	\N	0	val23
 0	\N	0	val24
@@ -326,7 +317,6 @@ t1 ALL RIGHT JOIN tn2 | bs = {{ block_size }}
 1	1	4	val11
 1	1	4	val12
 3	3	4	val3
-{% endif -%}
 t1 ANY INNER JOIN tn2 | bs = {{ block_size }}
 1	1	4	5
 3	3	4	4
@@ -341,7 +331,6 @@ t1 ANY LEFT JOIN tn2 | bs = {{ block_size }}
 2	\N	val27	0
 3	3	val3	4
 t1 ANY RIGHT JOIN tn2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 0	\N	0	val22
 0	\N	0	val23
 0	\N	0	val24
@@ -352,9 +341,7 @@ t1 ANY RIGHT JOIN tn2 | bs = {{ block_size }}
 1	1	4	val11
 1	1	4	val12
 3	3	4	val3
-{% endif -%}
 t1 ALL FULL JOIN tn2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 0	\N	0	5
 0	\N	0	5
 0	\N	0	5
@@ -372,9 +359,7 @@ t1 ALL FULL JOIN tn2 | bs = {{ block_size }}
 2	\N	5	0
 2	\N	5	0
 3	3	4	4
-{% endif -%}
 t1 ALL FULL JOIN USING tn2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 1	4	5
 1	4	5
 2	5	0
@@ -392,7 +377,6 @@ t1 ALL FULL JOIN USING tn2 | bs = {{ block_size }}
 \N	0	5
 \N	0	5
 \N	0	5
-{% endif -%}
 tn1 ALL INNER JOIN t2 | bs = {{ block_size }}
 1	1	4	5
 1	1	4	5
@@ -409,7 +393,6 @@ tn1 ALL LEFT JOIN t2 | bs = {{ block_size }}
 \N	0	val26	0
 \N	0	val27	0
 tn1 ALL RIGHT JOIN t2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 1	1	4	val11
 1	1	4	val12
 3	3	4	val3
@@ -420,7 +403,6 @@ tn1 ALL RIGHT JOIN t2 | bs = {{ block_size }}
 \N	2	0	val26
 \N	2	0	val27
 \N	2	0	val28
-{% endif -%}
 tn1 ANY INNER JOIN t2 | bs = {{ block_size }}
 1	1	4	5
 3	3	4	4
@@ -435,7 +417,6 @@ tn1 ANY LEFT JOIN t2 | bs = {{ block_size }}
 \N	0	val26	0
 \N	0	val27	0
 tn1 ANY RIGHT JOIN t2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 1	1	4	val11
 1	1	4	val12
 3	3	4	val3
@@ -446,9 +427,7 @@ tn1 ANY RIGHT JOIN t2 | bs = {{ block_size }}
 \N	2	0	val26
 \N	2	0	val27
 \N	2	0	val28
-{% endif -%}
 tn1 ALL FULL JOIN t2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 1	1	4	5
 1	1	4	5
 3	3	4	4
@@ -466,9 +445,7 @@ tn1 ALL FULL JOIN t2 | bs = {{ block_size }}
 \N	2	0	5
 \N	2	0	5
 \N	2	0	5
-{% endif -%}
 tn1 ALL FULL JOIN USING t2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 1	4	5
 1	4	5
 2	0	5
@@ -486,7 +463,6 @@ tn1 ALL FULL JOIN USING t2 | bs = {{ block_size }}
 \N	5	0
 \N	5	0
 \N	5	0
-{% endif -%}
 tn1 ALL INNER JOIN tn2 | bs = {{ block_size }}
 1	1	4	5
 1	1	4	5
@@ -503,7 +479,6 @@ tn1 ALL LEFT JOIN tn2 | bs = {{ block_size }}
 \N	\N	val26	0
 \N	\N	val27	0
 tn1 ALL RIGHT JOIN tn2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 1	1	4	val11
 1	1	4	val12
 3	3	4	val3
@@ -514,7 +489,6 @@ tn1 ALL RIGHT JOIN tn2 | bs = {{ block_size }}
 \N	\N	0	val26
 \N	\N	0	val27
 \N	\N	0	val28
-{% endif -%}
 tn1 ANY INNER JOIN tn2 | bs = {{ block_size }}
 1	1	4	5
 3	3	4	4
@@ -529,7 +503,6 @@ tn1 ANY LEFT JOIN tn2 | bs = {{ block_size }}
 \N	\N	val26	0
 \N	\N	val27	0
 tn1 ANY RIGHT JOIN tn2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 1	1	4	val11
 1	1	4	val12
 3	3	4	val3
@@ -540,9 +513,7 @@ tn1 ANY RIGHT JOIN tn2 | bs = {{ block_size }}
 \N	\N	0	val26
 \N	\N	0	val27
 \N	\N	0	val28
-{% endif -%}
 tn1 ALL FULL JOIN tn2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 1	1	4	5
 1	1	4	5
 3	3	4	4
@@ -560,9 +531,7 @@ tn1 ALL FULL JOIN tn2 | bs = {{ block_size }}
 \N	\N	5	0
 \N	\N	5	0
 \N	\N	5	0
-{% endif -%}
 tn1 ALL FULL JOIN USING tn2 | bs = {{ block_size }}
-{% if join_algorithm != 'grace_hash' -%}
 1	4	5
 1	4	5
 3	4	4
@@ -580,6 +549,5 @@ tn1 ALL FULL JOIN USING tn2 | bs = {{ block_size }}
 \N	5	0
 \N	5	0
 \N	5	0
-{% endif -%}
 {% endfor -%}
 {% endfor -%}

From 5c44e6a562f23a388b9895088320286b1bf7b598 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Sat, 6 May 2023 16:26:40 +0800
Subject: [PATCH 678/689] triger ci

---
 src/Interpreters/GraceHashJoin.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index ee369d4b195..bdfd63602ec 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -501,7 +501,7 @@ public:
                 non_joined_blocks = hash_join->getNonJoinedBlocks(left_sample_block, result_sample_block, max_block_size);
             }
         }
-        // For right/full join, non_joined_blocks is not null.
+        // For right/full join, non_joined_blocks is not null
         // For inner/left join, non_joined_blocks is null, and return a empty block directly.
         std::lock_guard lock(mutex);
         if (non_joined_blocks)

From 89dd538beac9f1b690cf772764939318fc6cc47a Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Mon, 8 May 2023 09:29:29 +0800
Subject: [PATCH 679/689] update

---
 src/Interpreters/GraceHashJoin.cpp | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index bdfd63602ec..7d924bb3a02 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -629,6 +629,16 @@ IBlocksStreamPtr GraceHashJoin::getDelayedBlocks()
 
     size_t bucket_idx = current_bucket->idx;
 
+    if (hash_join)
+    {
+        auto right_blocks = hash_join->releaseJoinedBlocks(/* restructure */ false);
+        for (auto & block : right_blocks)
+        {
+            Blocks blocks = JoinCommon::scatterBlockByHash(right_key_names, block, buckets.size());
+            flushBlocksToBuckets<JoinTableSide::Right>(blocks, buckets, bucket_idx);
+        }
+    }
+
     hash_join = makeInMemoryJoin();
 
     for (bucket_idx = bucket_idx + 1; bucket_idx < buckets.size(); ++bucket_idx)

From 8efec9bccab4097574985030fa5aa112bc357242 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Mon, 8 May 2023 09:59:56 +0800
Subject: [PATCH 680/689] add locks for getNonJoinedBlocks

---
 src/Interpreters/GraceHashJoin.cpp | 13 ++++++++++++-
 src/Interpreters/GraceHashJoin.h   |  2 ++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index 7d924bb3a02..0e839e538fd 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -1,3 +1,4 @@
+#include <mutex>
 #include <Interpreters/GraceHashJoin.h>
 #include <Interpreters/HashJoin.h>
 #include <Interpreters/TableJoin.h>
@@ -524,7 +525,17 @@ private:
 IBlocksStreamPtr
 GraceHashJoin::getNonJoinedBlocks(const Block & left_sample_block_, const Block & result_sample_block_, UInt64 max_block_size_) const
 {
-    return std::make_shared<NonJoinedBlocksStream>(hash_join, left_sample_block_, result_sample_block_, max_block_size_);
+    std::lock_guard lock(hash_join_mutex);
+    if (!non_joined_block_stream)
+    {
+        if (last_hash_join_for_non_joined && last_hash_join_for_non_joined != hash_join)
+        {
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "getNonJoinedBlocks should be called once");
+        }
+        non_joined_block_stream = std::make_shared<NonJoinedBlocksStream>(hash_join, left_sample_block_, result_sample_block_, max_block_size_);
+        last_hash_join_for_non_joined = hash_join;
+    }
+    return non_joined_block_stream;
 }
 
 class GraceHashJoin::DelayedBlocks : public IBlocksStream
diff --git a/src/Interpreters/GraceHashJoin.h b/src/Interpreters/GraceHashJoin.h
index bc5f894bc8c..0ff3bc3f609 100644
--- a/src/Interpreters/GraceHashJoin.h
+++ b/src/Interpreters/GraceHashJoin.h
@@ -145,6 +145,8 @@ private:
     InMemoryJoinPtr hash_join;
     Block hash_join_sample_block;
     mutable std::mutex hash_join_mutex;
+    mutable IBlocksStreamPtr non_joined_block_stream = nullptr;
+    mutable InMemoryJoinPtr last_hash_join_for_non_joined = nullptr;
 };
 
 }

From d5efc0e6884b39ac77f31e1acff6a8eb370261ce Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Mon, 8 May 2023 11:35:48 +0800
Subject: [PATCH 681/689] update

---
 src/Interpreters/GraceHashJoin.cpp | 15 +++------------
 src/Interpreters/GraceHashJoin.h   |  2 --
 2 files changed, 3 insertions(+), 14 deletions(-)

diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index 0e839e538fd..06a4cfaa76a 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -522,20 +522,11 @@ private:
 
 // This is only be called for bucket[0] at present. other buckets' non-joined blocks are generated in
 // DelayedBlocks.
+// There is a finished counter in JoiningTransform, only the last JoiningTransform could call this function.
 IBlocksStreamPtr
 GraceHashJoin::getNonJoinedBlocks(const Block & left_sample_block_, const Block & result_sample_block_, UInt64 max_block_size_) const
 {
-    std::lock_guard lock(hash_join_mutex);
-    if (!non_joined_block_stream)
-    {
-        if (last_hash_join_for_non_joined && last_hash_join_for_non_joined != hash_join)
-        {
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "getNonJoinedBlocks should be called once");
-        }
-        non_joined_block_stream = std::make_shared<NonJoinedBlocksStream>(hash_join, left_sample_block_, result_sample_block_, max_block_size_);
-        last_hash_join_for_non_joined = hash_join;
-    }
-    return non_joined_block_stream;
+    return std::make_shared<NonJoinedBlocksStream>(hash_join, left_sample_block_, result_sample_block_, max_block_size_);
 }
 
 class GraceHashJoin::DelayedBlocks : public IBlocksStream
@@ -582,7 +573,7 @@ public:
                     is_left_reader_finished = true;
                 }
             }
-            if (is_left_reader_finished || !block)
+            if (is_left_reader_finished && !block)
             {
                 return non_joined_blocks_iter.next();
             }
diff --git a/src/Interpreters/GraceHashJoin.h b/src/Interpreters/GraceHashJoin.h
index 0ff3bc3f609..bc5f894bc8c 100644
--- a/src/Interpreters/GraceHashJoin.h
+++ b/src/Interpreters/GraceHashJoin.h
@@ -145,8 +145,6 @@ private:
     InMemoryJoinPtr hash_join;
     Block hash_join_sample_block;
     mutable std::mutex hash_join_mutex;
-    mutable IBlocksStreamPtr non_joined_block_stream = nullptr;
-    mutable InMemoryJoinPtr last_hash_join_for_non_joined = nullptr;
 };
 
 }

From 80af345ea6b8fa3e10ce84578fca2aaf76808ee1 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Mon, 8 May 2023 14:09:41 +0800
Subject: [PATCH 682/689] update

---
 src/Interpreters/GraceHashJoin.cpp                             | 3 ++-
 .../0_stateless/01721_join_implicit_cast_long.reference.j2     | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index 06a4cfaa76a..55ad0971a8f 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -571,9 +571,10 @@ public:
                 if (!block)
                 {
                     is_left_reader_finished = true;
+                    return non_joined_blocks_iter.next();
                 }
             }
-            if (is_left_reader_finished && !block)
+            else
             {
                 return non_joined_blocks_iter.next();
             }
diff --git a/tests/queries/0_stateless/01721_join_implicit_cast_long.reference.j2 b/tests/queries/0_stateless/01721_join_implicit_cast_long.reference.j2
index d40fc7ec864..ae43aa7195c 100644
--- a/tests/queries/0_stateless/01721_join_implicit_cast_long.reference.j2
+++ b/tests/queries/0_stateless/01721_join_implicit_cast_long.reference.j2
@@ -200,6 +200,7 @@
 1
 1
 1
+{% if join_algorithm not in ['full_sorting_merge'] -%}
 === join use nulls ===
 = full =
 -4	\N	196
@@ -401,4 +402,5 @@
 1
 1
 1
+{% endif -%}
 {% endfor -%}

From 29ade233979aab464dcc4d8141b2a51166a7bcb5 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Mon, 8 May 2023 15:18:32 +0800
Subject: [PATCH 683/689] fixed: return invalid mismatch rows on full/right
 join

---
 src/Interpreters/GraceHashJoin.cpp | 80 ++++++++++--------------------
 src/Interpreters/GraceHashJoin.h   |  1 -
 2 files changed, 25 insertions(+), 56 deletions(-)

diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index 55ad0971a8f..2e622a4c18e 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -1,4 +1,3 @@
-#include <mutex>
 #include <Interpreters/GraceHashJoin.h>
 #include <Interpreters/HashJoin.h>
 #include <Interpreters/TableJoin.h>
@@ -13,7 +12,6 @@
 #include <Disks/TemporaryFileOnDisk.h>
 #include <Common/logger_useful.h>
 #include <Common/thread_local_rng.h>
-#include "GraceHashJoin.h"
 
 #include <base/FnTraits.h>
 #include <fmt/format.h>
@@ -474,59 +472,13 @@ bool GraceHashJoin::alwaysReturnsEmptySet() const
     return hash_join_is_empty;
 }
 
-// Add lock for reading non-joined blocks, it's thread safe.
-class GraceHashJoin::NonJoinedBlocksStream : public IBlocksStream
-{
-public:
-    explicit NonJoinedBlocksStream(
-        InMemoryJoinPtr hash_join_,
-        const Block & left_sample_block_,
-        const Block & result_sample_block_,
-        size_t max_block_size_)
-        : hash_join(hash_join_)
-        , left_sample_block(left_sample_block_)
-        , result_sample_block(result_sample_block_)
-        , max_block_size(max_block_size_)
-    {
-    }
-
-    Block nextImpl() override
-    {
-        // initialize non_joined_blocks lazily.
-        if (!has_init_non_joined_blocks) [[unlikely]]
-        {
-            std::lock_guard lock(mutex);
-            if (!has_init_non_joined_blocks)
-            {
-                has_init_non_joined_blocks = true;
-                non_joined_blocks = hash_join->getNonJoinedBlocks(left_sample_block, result_sample_block, max_block_size);
-            }
-        }
-        // For right/full join, non_joined_blocks is not null
-        // For inner/left join, non_joined_blocks is null, and return a empty block directly.
-        std::lock_guard lock(mutex);
-        if (non_joined_blocks)
-            return non_joined_blocks->next();
-        return {};
-    }
-
-private:
-    InMemoryJoinPtr hash_join;
-    Block left_sample_block;
-    Block result_sample_block;
-    size_t max_block_size;
-    bool has_init_non_joined_blocks = false;
-    IBlocksStreamPtr non_joined_blocks = nullptr;
-    std::mutex mutex;
-};
-
 // This is only be called for bucket[0] at present. other buckets' non-joined blocks are generated in
 // DelayedBlocks.
 // There is a finished counter in JoiningTransform, only the last JoiningTransform could call this function.
 IBlocksStreamPtr
 GraceHashJoin::getNonJoinedBlocks(const Block & left_sample_block_, const Block & result_sample_block_, UInt64 max_block_size_) const
 {
-    return std::make_shared<NonJoinedBlocksStream>(hash_join, left_sample_block_, result_sample_block_, max_block_size_);
+    return hash_join->getNonJoinedBlocks(left_sample_block_, result_sample_block_, max_block_size_);
 }
 
 class GraceHashJoin::DelayedBlocks : public IBlocksStream
@@ -547,7 +499,9 @@ public:
         , left_reader(buckets[current_bucket]->getLeftTableReader())
         , left_key_names(left_key_names_)
         , right_key_names(right_key_names_)
-        , non_joined_blocks_iter(hash_join, left_sample_block_, result_sample_block_, max_block_size_)
+        , left_sample_block(left_sample_block_)
+        , result_sample_block(result_sample_block_)
+        , max_block_size(max_block_size_)
     {
     }
 
@@ -557,6 +511,12 @@ public:
     //   changed inside.
     Block nextImpl() override
     {
+        // there is data race case wihthout this lock:
+        //   1. thread 1 read the last block from left_reader, but not finish the join
+        //   2. thread 2 try to read from non-joined blocks. Since thread 1 has not finished,
+        //     the used flags in the hash_join is incomplete, then thread 2 return invalid mismatch rows.
+        std::lock_guard lock(non_joined_blocks_mutex);
+
         Block block;
         size_t num_buckets = buckets.size();
         size_t current_idx = buckets[current_bucket]->idx;
@@ -571,12 +531,18 @@ public:
                 if (!block)
                 {
                     is_left_reader_finished = true;
-                    return non_joined_blocks_iter.next();
+                    non_joined_blocks = hash_join->getNonJoinedBlocks(left_sample_block, result_sample_block, max_block_size);
                 }
             }
-            else
+
+            if (is_left_reader_finished)
             {
-                return non_joined_blocks_iter.next();
+                if (non_joined_blocks)
+                {
+                    return non_joined_blocks->next();
+                }
+                else
+                    return {};
             }
 
             // block comes from left_reader, need to join with right table to get the result.
@@ -619,8 +585,12 @@ public:
 
     Names left_key_names;
     Names right_key_names;
-    std::atomic<bool> is_left_reader_finished = false;
-    NonJoinedBlocksStream non_joined_blocks_iter;
+    Block left_sample_block;
+    Block result_sample_block;
+    size_t max_block_size = 0;
+    bool is_left_reader_finished = false;
+    IBlocksStreamPtr non_joined_blocks = nullptr;
+    std::mutex non_joined_blocks_mutex;
 };
 
 IBlocksStreamPtr GraceHashJoin::getDelayedBlocks()
diff --git a/src/Interpreters/GraceHashJoin.h b/src/Interpreters/GraceHashJoin.h
index bc5f894bc8c..eb39ee09208 100644
--- a/src/Interpreters/GraceHashJoin.h
+++ b/src/Interpreters/GraceHashJoin.h
@@ -43,7 +43,6 @@ class HashJoin;
 class GraceHashJoin final : public IJoin
 {
     class FileBucket;
-    class NonJoinedBlocksStream;
     class DelayedBlocks;
     using InMemoryJoin = HashJoin;
 

From 603c024eb0615392277297273d62e9b790237416 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Tue, 9 May 2023 11:17:09 +0800
Subject: [PATCH 684/689] ensure only the last processor could access
 non-joined blocks

---
 src/Interpreters/GraceHashJoin.cpp            | 67 +++---------------
 .../Transforms/JoiningTransform.cpp           | 69 +++++++++++++++++--
 src/Processors/Transforms/JoiningTransform.h  | 24 ++++++-
 src/QueryPipeline/QueryPipelineBuilder.cpp    |  2 +-
 ...274_full_sort_join_nodistinct.reference.j2 | 20 ++++++
 5 files changed, 115 insertions(+), 67 deletions(-)

diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index 2e622a4c18e..ad4457f924e 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -472,9 +472,9 @@ bool GraceHashJoin::alwaysReturnsEmptySet() const
     return hash_join_is_empty;
 }
 
-// This is only be called for bucket[0] at present. other buckets' non-joined blocks are generated in
-// DelayedBlocks.
-// There is a finished counter in JoiningTransform, only the last JoiningTransform could call this function.
+// There is a finished counter in JoiningTransform/DelayedJoinedBlocksWorkerTransform,
+// only the last processor could call this function to ensure all used flags have been inserted.
+// To support delayed stream mode, need to keep the hash join before next getDelayedBlocks call.
 IBlocksStreamPtr
 GraceHashJoin::getNonJoinedBlocks(const Block & left_sample_block_, const Block & result_sample_block_, UInt64 max_block_size_) const
 {
@@ -489,60 +489,30 @@ public:
         Buckets buckets_,
         InMemoryJoinPtr hash_join_,
         const Names & left_key_names_,
-        const Names & right_key_names_,
-        const Block & left_sample_block_,
-        const Block & result_sample_block_,
-        size_t max_block_size_)
+        const Names & right_key_names_)
         : current_bucket(current_bucket_)
         , buckets(std::move(buckets_))
         , hash_join(std::move(hash_join_))
         , left_reader(buckets[current_bucket]->getLeftTableReader())
         , left_key_names(left_key_names_)
         , right_key_names(right_key_names_)
-        , left_sample_block(left_sample_block_)
-        , result_sample_block(result_sample_block_)
-        , max_block_size(max_block_size_)
     {
     }
 
-    // One DelayedBlocks is shared among multiple DelayedJoinedBlocksWorkerTransforms, need locks for
-    // - reading from left_reader. left_reader.read() has had a lock inside.
-    // - reading non-joined blocks from hash_join. Since iterate on non-joined blocks will has state
-    //   changed inside.
     Block nextImpl() override
     {
-        // there is data race case wihthout this lock:
-        //   1. thread 1 read the last block from left_reader, but not finish the join
-        //   2. thread 2 try to read from non-joined blocks. Since thread 1 has not finished,
-        //     the used flags in the hash_join is incomplete, then thread 2 return invalid mismatch rows.
-        std::lock_guard lock(non_joined_blocks_mutex);
-
         Block block;
         size_t num_buckets = buckets.size();
         size_t current_idx = buckets[current_bucket]->idx;
 
         do
         {
-            // When left reader finish, return non-joined blocks.
-            // empty block means the end of this stream.
-            if (!is_left_reader_finished)
+            // One DelayedBlocks is shared among multiple DelayedJoinedBlocksWorkerTransform.
+            // There is a lock inside left_reader.read().
+            block = left_reader.read();
+            if (!block)
             {
-                block = left_reader.read();
-                if (!block)
-                {
-                    is_left_reader_finished = true;
-                    non_joined_blocks = hash_join->getNonJoinedBlocks(left_sample_block, result_sample_block, max_block_size);
-                }
-            }
-
-            if (is_left_reader_finished)
-            {
-                if (non_joined_blocks)
-                {
-                    return non_joined_blocks->next();
-                }
-                else
-                    return {};
+                return {};
             }
 
             // block comes from left_reader, need to join with right table to get the result.
@@ -585,12 +555,6 @@ public:
 
     Names left_key_names;
     Names right_key_names;
-    Block left_sample_block;
-    Block result_sample_block;
-    size_t max_block_size = 0;
-    bool is_left_reader_finished = false;
-    IBlocksStreamPtr non_joined_blocks = nullptr;
-    std::mutex non_joined_blocks_mutex;
 };
 
 IBlocksStreamPtr GraceHashJoin::getDelayedBlocks()
@@ -636,18 +600,7 @@ IBlocksStreamPtr GraceHashJoin::getDelayedBlocks()
 
         LOG_TRACE(log, "Loaded bucket {} with {}(/{}) rows",
             bucket_idx, hash_join->getTotalRowCount(), num_rows);
-        auto result_sample_block = left_sample_block;
-        ExtraBlockPtr tmp;
-        hash_join->joinBlock(result_sample_block, tmp);
-        return std::make_unique<DelayedBlocks>(
-            current_bucket->idx,
-            buckets,
-            hash_join,
-            left_key_names,
-            right_key_names,
-            left_sample_block,
-            result_sample_block,
-            max_block_size);
+        return std::make_unique<DelayedBlocks>(current_bucket->idx, buckets, hash_join, left_key_names, right_key_names);
     }
 
     LOG_TRACE(log, "Finished loading all {} buckets", buckets.size());
diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp
index b638a5582c5..d80d6b8e387 100644
--- a/src/Processors/Transforms/JoiningTransform.cpp
+++ b/src/Processors/Transforms/JoiningTransform.cpp
@@ -311,8 +311,16 @@ void FillingRightJoinSideTransform::work()
 }
 
 
-DelayedJoinedBlocksWorkerTransform::DelayedJoinedBlocksWorkerTransform(Block output_header)
-    : IProcessor(InputPorts{Block()}, OutputPorts{output_header})
+DelayedJoinedBlocksWorkerTransform::DelayedJoinedBlocksWorkerTransform(
+    Block left_header_,
+    Block output_header_,
+    size_t max_block_size_,
+    JoinPtr join_)
+    : IProcessor(InputPorts{Block()}, OutputPorts{output_header_})
+    , left_header(left_header_)
+    , output_header(output_header_)
+    , max_block_size(max_block_size_)
+    , join(join_)
 {
 }
 
@@ -366,6 +374,7 @@ IProcessor::Status DelayedJoinedBlocksWorkerTransform::prepare()
         if (!data.chunk.hasChunkInfo())
             throw Exception(ErrorCodes::LOGICAL_ERROR, "DelayedJoinedBlocksWorkerTransform must have chunk info");
         task = std::dynamic_pointer_cast<const DelayedBlocksTask>(data.chunk.getChunkInfo());
+
     }
     else
     {
@@ -386,12 +395,24 @@ void DelayedJoinedBlocksWorkerTransform::work()
 {
     if (!task)
         return;
+    Block block;
+    if (!left_delayed_stream_finished)
+    {
+        block = task->delayed_blocks->next();
 
-    Block block = task->delayed_blocks->next();
-
+        if (!block)
+        {
+            left_delayed_stream_finished = true;
+            block = nextNonJoinedBlock();
+        }
+    }
+    else
+    {
+        block = nextNonJoinedBlock();
+    }
     if (!block)
     {
-        task.reset();
+        resetTask();
         return;
     }
 
@@ -400,6 +421,38 @@ void DelayedJoinedBlocksWorkerTransform::work()
     output_chunk.setColumns(block.getColumns(), rows);
 }
 
+void DelayedJoinedBlocksWorkerTransform::resetTask()
+{
+    task.reset();
+    left_delayed_stream_finished = false;
+    setup_non_joined_stream = false;
+    non_joined_delay_stream = nullptr;
+}
+
+Block DelayedJoinedBlocksWorkerTransform::nextNonJoinedBlock()
+{
+    if (!setup_non_joined_stream)
+    {
+        setup_non_joined_stream = true;
+        // Before read from non-joined stream, all blocks in left file reader must have been joined.
+        // For example, in HashJoin, it may return invalid mismatch rows from non-joined stream before
+        // the all blocks in left file reader have been finished, since the used flags are incomplete.
+        // To make only one processor could read from non-joined stream seems be a easy way.
+        if (task && task->left_delayed_stream_finish_counter->isLast())
+        {
+            if (!non_joined_delay_stream)
+            {
+                non_joined_delay_stream = join->getNonJoinedBlocks(left_header, output_header, max_block_size);
+            }
+        }
+    }
+    if (non_joined_delay_stream)
+    {
+        return non_joined_delay_stream->next();
+    }
+    return {};
+}
+
 DelayedJoinedBlocksTransform::DelayedJoinedBlocksTransform(size_t num_streams, JoinPtr join_)
     : IProcessor(InputPorts{}, OutputPorts(num_streams, Block()))
     , join(std::move(join_))
@@ -451,10 +504,14 @@ IProcessor::Status DelayedJoinedBlocksTransform::prepare()
 
     if (delayed_blocks)
     {
+        // This counter is used to ensure that only the last DelayedJoinedBlocksWorkerTransform
+        // could read non-joined blocks from the join.
+        auto left_delayed_stream_finished_counter = std::make_shared<JoiningTransform::FinishCounter>(outputs.size());
         for (auto & output : outputs)
         {
             Chunk chunk;
-            chunk.setChunkInfo(std::make_shared<DelayedBlocksTask>(delayed_blocks));
+            auto task = std::make_shared<DelayedBlocksTask>(delayed_blocks, left_delayed_stream_finished_counter);
+            chunk.setChunkInfo(task);
             output.push(std::move(chunk));
         }
         delayed_blocks = nullptr;
diff --git a/src/Processors/Transforms/JoiningTransform.h b/src/Processors/Transforms/JoiningTransform.h
index e7edff40c56..e9bd79bf623 100644
--- a/src/Processors/Transforms/JoiningTransform.h
+++ b/src/Processors/Transforms/JoiningTransform.h
@@ -116,9 +116,14 @@ class DelayedBlocksTask : public ChunkInfo
 public:
 
     explicit DelayedBlocksTask() : finished(true) {}
-    explicit DelayedBlocksTask(IBlocksStreamPtr delayed_blocks_) : delayed_blocks(std::move(delayed_blocks_)) {}
+    explicit DelayedBlocksTask(IBlocksStreamPtr delayed_blocks_, JoiningTransform::FinishCounterPtr left_delayed_stream_finish_counter_)
+        : delayed_blocks(std::move(delayed_blocks_))
+        , left_delayed_stream_finish_counter(left_delayed_stream_finish_counter_)
+    {
+    }
 
     IBlocksStreamPtr delayed_blocks = nullptr;
+    JoiningTransform::FinishCounterPtr left_delayed_stream_finish_counter = nullptr;
 
     bool finished = false;
 };
@@ -147,7 +152,11 @@ private:
 class DelayedJoinedBlocksWorkerTransform : public IProcessor
 {
 public:
-    explicit DelayedJoinedBlocksWorkerTransform(Block output_header);
+    explicit DelayedJoinedBlocksWorkerTransform(
+        Block left_header_,
+        Block output_header_,
+        size_t max_block_size_,
+        JoinPtr join_);
 
     String getName() const override { return "DelayedJoinedBlocksWorkerTransform"; }
 
@@ -155,10 +164,19 @@ public:
     void work() override;
 
 private:
+    Block left_header;
+    Block output_header;
+    size_t max_block_size;
+    JoinPtr join;
     DelayedBlocksTaskPtr task;
     Chunk output_chunk;
 
-    bool finished = false;
+    bool left_delayed_stream_finished = false;
+    bool setup_non_joined_stream = false;
+    IBlocksStreamPtr non_joined_delay_stream = nullptr;
+
+    void resetTask();
+    Block nextNonJoinedBlock();
 };
 
 }
diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp
index a4edf107b2f..764997e7b7e 100644
--- a/src/QueryPipeline/QueryPipelineBuilder.cpp
+++ b/src/QueryPipeline/QueryPipelineBuilder.cpp
@@ -491,7 +491,7 @@ std::unique_ptr<QueryPipelineBuilder> QueryPipelineBuilder::joinPipelinesRightLe
         if (delayed_root)
         {
             // Process delayed joined blocks when all JoiningTransform are finished.
-            auto delayed = std::make_shared<DelayedJoinedBlocksWorkerTransform>(joined_header);
+            auto delayed = std::make_shared<DelayedJoinedBlocksWorkerTransform>(left_header, joined_header, max_block_size, join);
             if (delayed->getInputs().size() != 1 || delayed->getOutputs().size() != 1)
                 throw Exception(ErrorCodes::LOGICAL_ERROR, "DelayedJoinedBlocksWorkerTransform should have one input and one output");
 
diff --git a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2 b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2
index 6b5b5bfd298..2dc7fe33423 100644
--- a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2
+++ b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2
@@ -360,6 +360,7 @@ t1 ALL FULL JOIN tn2 | bs = {{ block_size }}
 2	\N	5	0
 3	3	4	4
 t1 ALL FULL JOIN USING tn2 | bs = {{ block_size }}
+{% if join_algorithm == 'full_sorting_merge' -%}
 1	4	5
 1	4	5
 2	5	0
@@ -377,6 +378,25 @@ t1 ALL FULL JOIN USING tn2 | bs = {{ block_size }}
 \N	0	5
 \N	0	5
 \N	0	5
+{% else -%}
+0	0	5
+0	0	5
+0	0	5
+0	0	5
+0	0	5
+0	0	5
+0	0	5
+1	4	5
+1	4	5
+2	5	0
+2	5	0
+2	5	0
+2	5	0
+2	5	0
+2	5	0
+2	5	0
+3	4	4
+{% endif -%}
 tn1 ALL INNER JOIN t2 | bs = {{ block_size }}
 1	1	4	5
 1	1	4	5

From 258239927109d808276f475d02c37a245c84f944 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Tue, 9 May 2023 17:42:07 +0800
Subject: [PATCH 685/689] fixed: a in-memory bucket contains rows of other
 buckets

---
 src/Interpreters/GraceHashJoin.cpp            | 22 ++++++++++---------
 .../Transforms/JoiningTransform.cpp           | 11 +++++-----
 src/Processors/Transforms/JoiningTransform.h  |  2 +-
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index ad4457f924e..129d58a5436 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -566,16 +566,6 @@ IBlocksStreamPtr GraceHashJoin::getDelayedBlocks()
 
     size_t bucket_idx = current_bucket->idx;
 
-    if (hash_join)
-    {
-        auto right_blocks = hash_join->releaseJoinedBlocks(/* restructure */ false);
-        for (auto & block : right_blocks)
-        {
-            Blocks blocks = JoinCommon::scatterBlockByHash(right_key_names, block, buckets.size());
-            flushBlocksToBuckets<JoinTableSide::Right>(blocks, buckets, bucket_idx);
-        }
-    }
-
     hash_join = makeInMemoryJoin();
 
     for (bucket_idx = bucket_idx + 1; bucket_idx < buckets.size(); ++bucket_idx)
@@ -639,6 +629,18 @@ void GraceHashJoin::addJoinedBlockImpl(Block block)
         if (!hash_join)
             hash_join = makeInMemoryJoin();
 
+        // buckets size has been changed in other threads. Need to scatter current_block again.
+        // rehash could only happen under hash_join_mutex's scope.
+        auto current_buckets = getCurrentBuckets();
+        if (buckets_snapshot.size() != current_buckets.size())
+        {
+            LOG_TRACE(log, "mismatch buckets size. previous:{}, current:{}", buckets_snapshot.size(), getCurrentBuckets().size());
+            Blocks blocks = JoinCommon::scatterBlockByHash(right_key_names, current_block, current_buckets.size());
+            flushBlocksToBuckets<JoinTableSide::Right>(blocks, current_buckets, bucket_index);
+            current_block = std::move(blocks[bucket_index]);
+            if (!current_block.rows())
+                return;
+        }
         hash_join->addJoinedBlock(current_block, /* check_limits = */ false);
 
         if (!hasMemoryOverflow(hash_join))
diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp
index d80d6b8e387..d4ea601209f 100644
--- a/src/Processors/Transforms/JoiningTransform.cpp
+++ b/src/Processors/Transforms/JoiningTransform.cpp
@@ -189,7 +189,6 @@ void JoiningTransform::transform(Chunk & chunk)
     }
     else
         block = readExecute(chunk);
-
     auto num_rows = block.rows();
     chunk.setColumns(block.getColumns(), num_rows);
 }
@@ -426,7 +425,7 @@ void DelayedJoinedBlocksWorkerTransform::resetTask()
     task.reset();
     left_delayed_stream_finished = false;
     setup_non_joined_stream = false;
-    non_joined_delay_stream = nullptr;
+    non_joined_delayed_stream = nullptr;
 }
 
 Block DelayedJoinedBlocksWorkerTransform::nextNonJoinedBlock()
@@ -440,15 +439,15 @@ Block DelayedJoinedBlocksWorkerTransform::nextNonJoinedBlock()
         // To make only one processor could read from non-joined stream seems be a easy way.
         if (task && task->left_delayed_stream_finish_counter->isLast())
         {
-            if (!non_joined_delay_stream)
+            if (!non_joined_delayed_stream)
             {
-                non_joined_delay_stream = join->getNonJoinedBlocks(left_header, output_header, max_block_size);
+                non_joined_delayed_stream = join->getNonJoinedBlocks(left_header, output_header, max_block_size);
             }
         }
     }
-    if (non_joined_delay_stream)
+    if (non_joined_delayed_stream)
     {
-        return non_joined_delay_stream->next();
+        return non_joined_delayed_stream->next();
     }
     return {};
 }
diff --git a/src/Processors/Transforms/JoiningTransform.h b/src/Processors/Transforms/JoiningTransform.h
index e9bd79bf623..17222f0e514 100644
--- a/src/Processors/Transforms/JoiningTransform.h
+++ b/src/Processors/Transforms/JoiningTransform.h
@@ -173,7 +173,7 @@ private:
 
     bool left_delayed_stream_finished = false;
     bool setup_non_joined_stream = false;
-    IBlocksStreamPtr non_joined_delay_stream = nullptr;
+    IBlocksStreamPtr non_joined_delayed_stream = nullptr;
 
     void resetTask();
     Block nextNonJoinedBlock();

From 983514f9461862b6d999455863f3178dd09ab2a1 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Wed, 10 May 2023 16:00:41 +0800
Subject: [PATCH 686/689] update tests for using clause

---
 ...274_full_sort_join_nodistinct.reference.j2 | 22 +++----------------
 .../02274_full_sort_join_nodistinct.sql.j2    |  3 ++-
 2 files changed, 5 insertions(+), 20 deletions(-)

diff --git a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2 b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2
index 2dc7fe33423..2b8ca580f50 100644
--- a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2
+++ b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2
@@ -359,8 +359,8 @@ t1 ALL FULL JOIN tn2 | bs = {{ block_size }}
 2	\N	5	0
 2	\N	5	0
 3	3	4	4
+{% if join_algorithm != 'grace_hash' -%}
 t1 ALL FULL JOIN USING tn2 | bs = {{ block_size }}
-{% if join_algorithm == 'full_sorting_merge' -%}
 1	4	5
 1	4	5
 2	5	0
@@ -378,24 +378,6 @@ t1 ALL FULL JOIN USING tn2 | bs = {{ block_size }}
 \N	0	5
 \N	0	5
 \N	0	5
-{% else -%}
-0	0	5
-0	0	5
-0	0	5
-0	0	5
-0	0	5
-0	0	5
-0	0	5
-1	4	5
-1	4	5
-2	5	0
-2	5	0
-2	5	0
-2	5	0
-2	5	0
-2	5	0
-2	5	0
-3	4	4
 {% endif -%}
 tn1 ALL INNER JOIN t2 | bs = {{ block_size }}
 1	1	4	5
@@ -551,6 +533,7 @@ tn1 ALL FULL JOIN tn2 | bs = {{ block_size }}
 \N	\N	5	0
 \N	\N	5	0
 \N	\N	5	0
+{% if join_algorithm != 'grace_hash' -%}
 tn1 ALL FULL JOIN USING tn2 | bs = {{ block_size }}
 1	4	5
 1	4	5
@@ -569,5 +552,6 @@ tn1 ALL FULL JOIN USING tn2 | bs = {{ block_size }}
 \N	5	0
 \N	5	0
 \N	5	0
+{% endif -%}
 {% endfor -%}
 {% endfor -%}
diff --git a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2 b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2
index 6bf634ca1f5..a1f7b133cf6 100644
--- a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2
+++ b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2
@@ -46,9 +46,10 @@ SELECT t1.key, t2.key, length(t1.s), t2.s FROM {{ t1 }} AS t1 {{ kind }} RIGHT J
 SELECT '{{ t1 }} ALL FULL JOIN {{ t2 }} | bs = {{ block_size }}';
 SELECT t1.key, t2.key, length(t1.s), length(t2.s) FROM {{ t1 }} AS t1 {{ kind }} FULL JOIN {{ t2 }} AS t2 ON t1.key == t2.key ORDER BY t1.key, t2.key, length(t1.s), length(t2.s); {{ is_implemented(join_algorithm) }}
 
+{% if join_algorithm == 'full_sorting_merge' or t2 != 'tn2' -%}
 SELECT '{{ t1 }} ALL FULL JOIN USING {{ t2 }} | bs = {{ block_size }}';
 SELECT key, length(t1.s), length(t2.s) FROM {{ t1 }} AS t1 ALL FULL JOIN {{ t2 }} AS t2 USING (key) ORDER BY key, length(t1.s), length(t2.s); {{ is_implemented(join_algorithm) }}
-
+{% endif -%}
 {% endfor -%}
 {% endfor -%}
 SET max_bytes_in_join = 0;

From 35d83887057e245dc038e918913be45fa3ca66ae Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Fri, 12 May 2023 09:38:21 +0800
Subject: [PATCH 687/689] update

---
 src/Interpreters/GraceHashJoin.cpp             | 13 +++++++++----
 src/Processors/Transforms/JoiningTransform.cpp |  2 +-
 src/Processors/Transforms/JoiningTransform.h   |  1 +
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index 129d58a5436..4a4c69ff473 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -471,10 +471,15 @@ bool GraceHashJoin::alwaysReturnsEmptySet() const
 
     return hash_join_is_empty;
 }
-
-// There is a finished counter in JoiningTransform/DelayedJoinedBlocksWorkerTransform,
-// only the last processor could call this function to ensure all used flags have been inserted.
-// To support delayed stream mode, need to keep the hash join before next getDelayedBlocks call.
+/// Each bucket are handled by the following steps
+/// 1. build hash_join by the right side blocks.
+/// 2. join left side with the hash_join,
+/// 3. read right non-joined blocks from hash_join.
+/// buckets are handled one by one, each hash_join will not be release before the right non-joined blocks are emitted.
+///
+/// There is a finished counter in JoiningTransform/DelayedJoinedBlocksWorkerTransform,
+/// only one processor could take the non-joined blocks from right stream, and ensure all rows from
+/// left stream have been emitted before this.
 IBlocksStreamPtr
 GraceHashJoin::getNonJoinedBlocks(const Block & left_sample_block_, const Block & result_sample_block_, UInt64 max_block_size_) const
 {
diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp
index d4ea601209f..256ef66a27d 100644
--- a/src/Processors/Transforms/JoiningTransform.cpp
+++ b/src/Processors/Transforms/JoiningTransform.cpp
@@ -504,7 +504,7 @@ IProcessor::Status DelayedJoinedBlocksTransform::prepare()
     if (delayed_blocks)
     {
         // This counter is used to ensure that only the last DelayedJoinedBlocksWorkerTransform
-        // could read non-joined blocks from the join.
+        // could read right non-joined blocks from the join.
         auto left_delayed_stream_finished_counter = std::make_shared<JoiningTransform::FinishCounter>(outputs.size());
         for (auto & output : outputs)
         {
diff --git a/src/Processors/Transforms/JoiningTransform.h b/src/Processors/Transforms/JoiningTransform.h
index 17222f0e514..3577906b26a 100644
--- a/src/Processors/Transforms/JoiningTransform.h
+++ b/src/Processors/Transforms/JoiningTransform.h
@@ -171,6 +171,7 @@ private:
     DelayedBlocksTaskPtr task;
     Chunk output_chunk;
 
+    /// All joined and non-joined rows from left stream are emitted, only right non-joined rows are left
     bool left_delayed_stream_finished = false;
     bool setup_non_joined_stream = false;
     IBlocksStreamPtr non_joined_delayed_stream = nullptr;

From cc93f9ab52669dda67dce644241c4f7e88b94216 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Wed, 17 May 2023 09:06:33 +0800
Subject: [PATCH 688/689] update tests for flaky test

---
 docs/en/operations/settings/settings.md                         | 2 ++
 tests/queries/0_stateless/02273_full_sort_join.reference.j2     | 2 +-
 tests/queries/0_stateless/02273_full_sort_join.sql.j2           | 2 +-
 .../0_stateless/02274_full_sort_join_nodistinct.reference.j2    | 2 +-
 .../queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2  | 2 +-
 5 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 791186dd9c1..2239084a429 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -452,6 +452,8 @@ Possible values:
 
  The first phase of a grace join reads the right table and splits it into N buckets depending on the hash value of key columns (initially, N is `grace_hash_join_initial_buckets`). This is done in a way to ensure that each bucket can be processed independently. Rows from the first bucket are added to an in-memory hash table while the others are saved to disk. If the hash table grows beyond the memory limit (e.g., as set by [`max_bytes_in_join`](/docs/en/operations/settings/query-complexity.md/#settings-max_bytes_in_join)), the number of buckets is increased and the assigned bucket for each row. Any rows which don’t belong to the current bucket are flushed and reassigned.
 
+ Supports `INNER/LEFT/RIGHT/FULL ALL/ANY JOIN`.
+
 - hash
 
  [Hash join algorithm](https://en.wikipedia.org/wiki/Hash_join) is used. The most generic implementation that supports all combinations of kind and strictness and multiple join keys that are combined with `OR` in the `JOIN ON` section.
diff --git a/tests/queries/0_stateless/02273_full_sort_join.reference.j2 b/tests/queries/0_stateless/02273_full_sort_join.reference.j2
index 7b994c2d11f..0af4158e971 100644
--- a/tests/queries/0_stateless/02273_full_sort_join.reference.j2
+++ b/tests/queries/0_stateless/02273_full_sort_join.reference.j2
@@ -1,7 +1,7 @@
 {% set table_size = 15 -%}
 {% for join_algorithm in ['default', 'full_sorting_merge', 'grace_hash'] -%}
 -- {{ join_algorithm }} --
-{% for block_size in range(1, table_size + 1) -%}
+{% for block_size in range(1, table_size + 1, 4) -%}
 ALL INNER USING | bs = {{ block_size }}
 4	0	0
 5	0	0
diff --git a/tests/queries/0_stateless/02273_full_sort_join.sql.j2 b/tests/queries/0_stateless/02273_full_sort_join.sql.j2
index b371d9544da..6500306356c 100644
--- a/tests/queries/0_stateless/02273_full_sort_join.sql.j2
+++ b/tests/queries/0_stateless/02273_full_sort_join.sql.j2
@@ -36,7 +36,7 @@ SET max_bytes_in_join = '{% if join_algorithm == 'grace_hash' %}10K{% else %}0{%
 SELECT '-- {{ join_algorithm }} --';
 SET join_algorithm = '{{ join_algorithm }}';
 
-{% for block_size in range(1, table_size + 1) -%}
+{% for block_size in range(1, table_size + 1, 4) -%}
 {% for kind in ['ALL', 'ANY'] -%}
 
 SET max_block_size = {{ block_size }};
diff --git a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2 b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2
index 2b8ca580f50..df968e86e8d 100644
--- a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2
+++ b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2
@@ -1,6 +1,6 @@
 {% for join_algorithm in ['full_sorting_merge', 'grace_hash'] -%}
 --- {{ join_algorithm }} ---
-{% for block_size in range(1, 11) -%}
+{% for block_size in range(1, 11, 4) -%}
 t1 ALL INNER JOIN t2 | bs = {{ block_size }}
 1	1	4	5
 1	1	4	5
diff --git a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2 b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2
index a1f7b133cf6..f8eb4b1a53e 100644
--- a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2
+++ b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2
@@ -26,7 +26,7 @@ SET join_algorithm = '{{ join_algorithm }}';
 
 SELECT '--- {{ join_algorithm }} ---';
 
-{% for block_size in range(1, 11) -%}
+{% for block_size in range(1, 11, 4) -%}
 SET max_block_size = {{ block_size }};
 
 {% for t1, t2 in [('t1', 't2'), ('t1', 'tn2'), ('tn1', 't2'), ('tn1', 'tn2')]  -%}

From a257ff6cf3ba1124e861452e4b3b52ada3ea2d5c Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Mon, 22 May 2023 10:41:22 +0200
Subject: [PATCH 689/689] address comment

---
 src/Common/OptimizedRegularExpression.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp
index 92b3ad32ecd..f2fe922ef19 100644
--- a/src/Common/OptimizedRegularExpression.cpp
+++ b/src/Common/OptimizedRegularExpression.cpp
@@ -68,7 +68,7 @@ const char * analyzeImpl(
     bool has_alternative_on_depth_0 = false;
     bool has_case_insensitive_flag = false;
 
-    /// Substring with a position.
+    /// Substring with is_prefix.
     using Substring = std::pair<std::string, bool>;
     using Substrings = std::vector<Substring>;