From a11e67d4aae4433dd0f3d8ee46ba40e1cd73fdd5 Mon Sep 17 00:00:00 2001
From: Alexander Sapin <alesapin@gmail.com>
Date: Fri, 16 Feb 2024 16:41:58 +0100
Subject: [PATCH 01/92] Make max_insert_delayed_streams_for_parallel_write
 actually work

---
 src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index 1fb2393948a..f5494e56049 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -323,6 +323,9 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
         if (!temp_part.part)
             continue;
 
+        if (!support_parallel_write && temp_part.part->getDataPartStorage().supportParallelWrite())
+            support_parallel_write = true;
+
         BlockIDsType block_id;
 
         if constexpr (async_insert)

From 458793cc50b92361848c91803d07105a91acea85 Mon Sep 17 00:00:00 2001
From: Alexander Sapin <alesapin@gmail.com>
Date: Fri, 16 Feb 2024 17:13:37 +0100
Subject: [PATCH 02/92] Review fix

---
 src/Storages/MergeTree/MergeTreeSink.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp
index 36816904a81..ebc49e22d03 100644
--- a/src/Storages/MergeTree/MergeTreeSink.cpp
+++ b/src/Storages/MergeTree/MergeTreeSink.cpp
@@ -109,9 +109,14 @@ void MergeTreeSink::consume(Chunk chunk)
             }
         }
 
-        size_t max_insert_delayed_streams_for_parallel_write = DEFAULT_DELAYED_STREAMS_FOR_PARALLEL_WRITE;
-        if (!support_parallel_write || settings.max_insert_delayed_streams_for_parallel_write.changed)
+        size_t max_insert_delayed_streams_for_parallel_write;
+
+        if (settings.max_insert_delayed_streams_for_parallel_write.changed)
             max_insert_delayed_streams_for_parallel_write = settings.max_insert_delayed_streams_for_parallel_write;
+        else if (support_parallel_write)
+            max_insert_delayed_streams_for_parallel_write = DEFAULT_DELAYED_STREAMS_FOR_PARALLEL_WRITE;
+        else
+            max_insert_delayed_streams_for_parallel_write = 0;
 
         /// In case of too much columns/parts in block, flush explicitly.
         streams += temp_part.streams.size();

From f7b524465c60b15c85f579ca22c48d4c165bf6f2 Mon Sep 17 00:00:00 2001
From: Alexander Sapin <alesapin@gmail.com>
Date: Fri, 16 Feb 2024 17:14:36 +0100
Subject: [PATCH 03/92] Followup

---
 src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index f5494e56049..3cbdcf5106e 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -368,9 +368,13 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
         profile_events_scope.reset();
         UInt64 elapsed_ns = watch.elapsed();
 
-        size_t max_insert_delayed_streams_for_parallel_write = DEFAULT_DELAYED_STREAMS_FOR_PARALLEL_WRITE;
-        if (!support_parallel_write || settings.max_insert_delayed_streams_for_parallel_write.changed)
+        size_t max_insert_delayed_streams_for_parallel_write;
+        if (settings.max_insert_delayed_streams_for_parallel_write.changed)
             max_insert_delayed_streams_for_parallel_write = settings.max_insert_delayed_streams_for_parallel_write;
+        else if (support_parallel_write)
+            max_insert_delayed_streams_for_parallel_write = DEFAULT_DELAYED_STREAMS_FOR_PARALLEL_WRITE;
+        else
+            max_insert_delayed_streams_for_parallel_write = 0;
 
         /// In case of too much columns/parts in block, flush explicitly.
         streams += temp_part.streams.size();

From 1768b4477f4ff5db238cd4cc553587b136ed015d Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 3 Mar 2024 11:50:22 +0100
Subject: [PATCH 04/92] Revert "Merge pull request #60690 from
 ClickHouse/remove-bad-test-8"

This reverts commit c77eb8b1427f98daf63f7087bbdc0530b07db825, reversing
changes made to bae4783fe9bd25decc41383a1234b0e936284c21.
---
 ..._external_tables_memory_tracking.reference | 16 ++++++
 ...52_http_external_tables_memory_tracking.sh | 51 +++++++++++++++++++
 2 files changed, 67 insertions(+)
 create mode 100644 tests/queries/0_stateless/02152_http_external_tables_memory_tracking.reference
 create mode 100755 tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh

diff --git a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.reference b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.reference
new file mode 100644
index 00000000000..1fc09c8d154
--- /dev/null
+++ b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.reference
@@ -0,0 +1,16 @@
+Checking input_format_parallel_parsing=false&
+1
+Checking input_format_parallel_parsing=false&cancel_http_readonly_queries_on_client_close=1&readonly=1
+1
+Checking input_format_parallel_parsing=false&send_progress_in_http_headers=true
+1
+Checking input_format_parallel_parsing=false&cancel_http_readonly_queries_on_client_close=1&readonly=1&send_progress_in_http_headers=true
+1
+Checking input_format_parallel_parsing=true&
+1
+Checking input_format_parallel_parsing=true&cancel_http_readonly_queries_on_client_close=1&readonly=1
+1
+Checking input_format_parallel_parsing=true&send_progress_in_http_headers=true
+1
+Checking input_format_parallel_parsing=true&cancel_http_readonly_queries_on_client_close=1&readonly=1&send_progress_in_http_headers=true
+1
diff --git a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh
new file mode 100755
index 00000000000..5f9eb460e44
--- /dev/null
+++ b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Tags: no-tsan, no-cpu-aarch64, no-parallel
+# TSan does not supports tracing.
+# trace_log doesn't work on aarch64
+
+# Regression for proper release of Context,
+# via tracking memory of external tables.
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+tmp_file=$(mktemp "$CURDIR/clickhouse.XXXXXX.csv")
+trap 'rm $tmp_file' EXIT
+
+$CLICKHOUSE_CLIENT -q "SELECT toString(number) FROM numbers(1e6) FORMAT TSV" > "$tmp_file"
+
+function run_and_check()
+{
+    local query_id
+    query_id="$(${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data-binary @- <<<'SELECT generateUUIDv4()')"
+
+    echo "Checking $*"
+
+    # Run query with external table (implicit StorageMemory user)
+    $CLICKHOUSE_CURL -sS -F "s=@$tmp_file;" "$CLICKHOUSE_URL&s_structure=key+Int&query=SELECT+count()+FROM+s&memory_profiler_sample_probability=1&max_untracked_memory=0&query_id=$query_id&$*" -o /dev/null
+
+    ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data-binary @- <<<'SYSTEM FLUSH LOGS'
+
+    # Check that temporary table had been destroyed.
+    ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&allow_introspection_functions=1" --data-binary @- <<<"
+    WITH arrayStringConcat(arrayMap(x -> demangle(addressToSymbol(x)), trace), '\n') AS sym
+    SELECT count()>0 FROM system.trace_log
+    WHERE
+        sym LIKE '%DB::StorageMemory::drop%\n%TemporaryTableHolder::~TemporaryTableHolder%' AND
+        query_id = '$query_id'
+    "
+}
+
+for input_format_parallel_parsing in false true; do
+    query_args_variants=(
+        ""
+        "cancel_http_readonly_queries_on_client_close=1&readonly=1"
+        "send_progress_in_http_headers=true"
+        # nested progress callback
+        "cancel_http_readonly_queries_on_client_close=1&readonly=1&send_progress_in_http_headers=true"
+    )
+    for query_args in "${query_args_variants[@]}"; do
+        run_and_check "input_format_parallel_parsing=$input_format_parallel_parsing&$query_args"
+    done
+done

From 048a042dc4963631a23358d3e454dcd8a9eaafa2 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 3 Mar 2024 11:50:46 +0100
Subject: [PATCH 05/92] Make 02152_http_external_tables_memory_tracking less
 flaky

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../02152_http_external_tables_memory_tracking.sh  | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh
index 5f9eb460e44..5494f7d59cb 100755
--- a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh
+++ b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: no-tsan, no-cpu-aarch64, no-parallel
+# Tags: no-tsan, no-cpu-aarch64, no-parallel, no-debug
 # TSan does not supports tracing.
 # trace_log doesn't work on aarch64
 
@@ -30,10 +30,16 @@ function run_and_check()
     # Check that temporary table had been destroyed.
     ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&allow_introspection_functions=1" --data-binary @- <<<"
     WITH arrayStringConcat(arrayMap(x -> demangle(addressToSymbol(x)), trace), '\n') AS sym
-    SELECT count()>0 FROM system.trace_log
+    SELECT 1 FROM system.trace_log
+    PREWHERE
+        query_id = '$query_id' AND
+        trace_type = 'MemorySample' AND
+        /* only deallocations */
+        size < 0 AND
+        event_date >= yesterday()
     WHERE
-        sym LIKE '%DB::StorageMemory::drop%\n%TemporaryTableHolder::~TemporaryTableHolder%' AND
-        query_id = '$query_id'
+        sym LIKE '%DB::StorageMemory::drop%\n%TemporaryTableHolder::~TemporaryTableHolder%'
+    LIMIT 1
     "
 }
 

From f2a3ffe9eb79046093e77ed39f2366754e7a8ba2 Mon Sep 17 00:00:00 2001
From: Amos Bird <amosbird@gmail.com>
Date: Thu, 7 Mar 2024 17:14:12 +0800
Subject: [PATCH 06/92] Replace some headers with forward decl

---
 src/Backups/BackupCoordinationRemote.cpp      |  1 +
 .../NamedCollections/NamedCollectionUtils.cpp |  1 +
 .../ObjectStorages/ObjectStorageFactory.cpp   |  3 +++
 src/Formats/ReadSchemaUtils.cpp               |  1 +
 src/Interpreters/DatabaseCatalog.cpp          |  1 -
 src/Interpreters/DatabaseCatalog.h            | 10 +++----
 src/Processors/QueryPlan/AggregatingStep.cpp  |  1 +
 src/Processors/QueryPlan/CubeStep.cpp         |  1 +
 src/Storages/StorageAzureBlob.cpp             |  4 +++
 src/Storages/StorageS3.h                      | 27 ++++++++++---------
 10 files changed, 29 insertions(+), 21 deletions(-)

diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp
index 9c509858b2a..b869f890f56 100644
--- a/src/Backups/BackupCoordinationRemote.cpp
+++ b/src/Backups/BackupCoordinationRemote.cpp
@@ -1,6 +1,7 @@
 #include <Backups/BackupCoordinationRemote.h>
 
 #include <base/hex.h>
+#include <boost/algorithm/string/split.hpp>
 
 #include <Access/Common/AccessEntityType.h>
 #include <Backups/BackupCoordinationReplicatedAccess.h>
diff --git a/src/Common/NamedCollections/NamedCollectionUtils.cpp b/src/Common/NamedCollections/NamedCollectionUtils.cpp
index fe0f42467c7..e3ff50f5e3f 100644
--- a/src/Common/NamedCollections/NamedCollectionUtils.cpp
+++ b/src/Common/NamedCollections/NamedCollectionUtils.cpp
@@ -17,6 +17,7 @@
 #include <Common/NamedCollections/NamedCollections.h>
 #include <Common/NamedCollections/NamedCollectionConfiguration.h>
 
+#include <filesystem>
 
 namespace fs = std::filesystem;
 
diff --git a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp
index 4f198be64fe..5fae257e8d4 100644
--- a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp
+++ b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp
@@ -21,6 +21,9 @@
 #include <Interpreters/Context.h>
 #include <Common/Macros.h>
 
+#include <filesystem>
+
+namespace fs = std::filesystem;
 
 namespace DB
 {
diff --git a/src/Formats/ReadSchemaUtils.cpp b/src/Formats/ReadSchemaUtils.cpp
index 5badf4301bf..736a35927c3 100644
--- a/src/Formats/ReadSchemaUtils.cpp
+++ b/src/Formats/ReadSchemaUtils.cpp
@@ -5,6 +5,7 @@
 #include <IO/WithFileSize.h>
 #include <IO/EmptyReadBuffer.h>
 #include <IO/PeekableReadBuffer.h>
+#include <Storages/IStorage.h>
 
 namespace DB
 {
diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index a9fd5c852ba..a5a523b658b 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -26,7 +26,6 @@
 #include <Common/noexcept_scope.h>
 #include <Common/checkStackSize.h>
 
-#include "Interpreters/Context_fwd.h"
 #include "config.h"
 
 #if USE_MYSQL
diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h
index 4fe114cc493..6995fc51941 100644
--- a/src/Interpreters/DatabaseCatalog.h
+++ b/src/Interpreters/DatabaseCatalog.h
@@ -1,15 +1,14 @@
 #pragma once
 
 #include <Core/UUID.h>
+#include <Databases/IDatabase.h>
+#include <Databases/TablesDependencyGraph.h>
 #include <Interpreters/Context_fwd.h>
 #include <Interpreters/StorageID.h>
-#include <Databases/TablesDependencyGraph.h>
 #include <Parsers/IAST_fwd.h>
 #include <Storages/IStorage_fwd.h>
-#include "Common/NamePrompter.h"
+#include <Common/NamePrompter.h>
 #include <Common/SharedMutex.h>
-#include "Storages/IStorage.h"
-#include "Databases/IDatabase.h"
 
 #include <boost/noncopyable.hpp>
 #include <Poco/Logger.h>
@@ -23,9 +22,6 @@
 #include <set>
 #include <unordered_map>
 #include <unordered_set>
-#include <filesystem>
-
-namespace fs = std::filesystem;
 
 namespace DB
 {
diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp
index f374a7b7b10..a76bacdd97b 100644
--- a/src/Processors/QueryPlan/AggregatingStep.cpp
+++ b/src/Processors/QueryPlan/AggregatingStep.cpp
@@ -8,6 +8,7 @@
 #include <IO/Operators.h>
 #include <Interpreters/Aggregator.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/ExpressionActions.h>
 #include <Processors/Merges/AggregatingSortedTransform.h>
 #include <Processors/Merges/FinishAggregatingInOrderTransform.h>
 #include <Processors/QueryPlan/AggregatingStep.h>
diff --git a/src/Processors/QueryPlan/CubeStep.cpp b/src/Processors/QueryPlan/CubeStep.cpp
index 0c632c346c7..bf2ce148529 100644
--- a/src/Processors/QueryPlan/CubeStep.cpp
+++ b/src/Processors/QueryPlan/CubeStep.cpp
@@ -5,6 +5,7 @@
 #include <QueryPipeline/QueryPipelineBuilder.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <Functions/FunctionFactory.h>
+#include <Interpreters/ExpressionActions.h>
 
 namespace DB
 {
diff --git a/src/Storages/StorageAzureBlob.cpp b/src/Storages/StorageAzureBlob.cpp
index 1f0fba99f84..2d4f1db04a1 100644
--- a/src/Storages/StorageAzureBlob.cpp
+++ b/src/Storages/StorageAzureBlob.cpp
@@ -41,6 +41,10 @@
 #include <Disks/IO/ReadBufferFromAzureBlobStorage.h>
 #include <Disks/IO/WriteBufferFromAzureBlobStorage.h>
 
+#include <filesystem>
+
+namespace fs = std::filesystem;
+
 using namespace Azure::Storage::Blobs;
 
 namespace CurrentMetrics
diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h
index 65fb3b51be2..bf81ead0599 100644
--- a/src/Storages/StorageS3.h
+++ b/src/Storages/StorageS3.h
@@ -4,27 +4,28 @@
 
 #if USE_AWS_S3
 
-#include <Core/Types.h>
-
 #include <Compression/CompressionInfo.h>
-
-#include <Storages/IStorage.h>
-#include <Storages/StorageS3Settings.h>
-
-#include <Processors/SourceWithKeyCondition.h>
-#include <Processors/Executors/PullingPipelineExecutor.h>
-#include <Processors/Formats/IInputFormat.h>
-#include <Poco/URI.h>
-#include <IO/S3/getObjectInfo.h>
+#include <Core/Types.h>
 #include <IO/CompressionMethod.h>
+#include <IO/S3/BlobStorageLogWriter.h>
+#include <IO/S3/getObjectInfo.h>
 #include <IO/SeekableReadBuffer.h>
 #include <Interpreters/Context.h>
-#include <Common/threadPoolCallbackRunner.h>
+#include <Processors/Executors/PullingPipelineExecutor.h>
+#include <Processors/Formats/IInputFormat.h>
+#include <Processors/SourceWithKeyCondition.h>
 #include <Storages/Cache/SchemaCache.h>
+#include <Storages/IStorage.h>
 #include <Storages/SelectQueryInfo.h>
 #include <Storages/StorageConfiguration.h>
+#include <Storages/StorageS3Settings.h>
 #include <Storages/prepareReadingFromFormat.h>
-#include <IO/S3/BlobStorageLogWriter.h>
+#include <Poco/URI.h>
+#include <Common/threadPoolCallbackRunner.h>
+
+#include <filesystem>
+
+namespace fs = std::filesystem;
 
 namespace Aws::S3
 {

From f4fc65449cc3ace36f33323600fd1a47fbfb9736 Mon Sep 17 00:00:00 2001
From: Peter <peter279k@gmail.com>
Date: Fri, 8 Mar 2024 01:20:50 +0800
Subject: [PATCH 07/92] Add another example dataset for presenting usage

---
 .../example-datasets/tw-weather.md            | 293 ++++++++++++++++++
 1 file changed, 293 insertions(+)
 create mode 100644 docs/en/getting-started/example-datasets/tw-weather.md

diff --git a/docs/en/getting-started/example-datasets/tw-weather.md b/docs/en/getting-started/example-datasets/tw-weather.md
new file mode 100644
index 00000000000..e5f16c403d5
--- /dev/null
+++ b/docs/en/getting-started/example-datasets/tw-weather.md
@@ -0,0 +1,293 @@
+---
+slug: /en/getting-started/example-datasets/tw-weather
+sidebar_label: Taiwan Historical Weather Datasets
+sidebar_position: 1
+description: 131 million rows of weather observation data for the last 128 yrs
+---
+
+# Taiwan Historical Weather Datasets
+
+This dataset contains historical meteorological observations measurements for the last 128 years. Each row is a measurement for a point in date time and weather station.
+
+The origin of this dataset is available [here](https://github.com/Raingel/historical_weather) and the list of weather station numbers can be found [here](https://github.com/Raingel/weather_station_list).
+
+> The sources of meteorological datasets include the meteorological stations that are established by the Central Weather Administration (station code is beginning with C0, C1, and 4) and the agricultural meteorological stations belonging to the Council of Agriculture (station code other than those mentioned above):
+
+    - StationId
+    - MeasuredDate, the observation time
+    - StnPres, the station air pressure
+    - SeaPres, the sea level pressure
+    - Td, the dew point temperature
+    - RH, the relative humidity
+    - Other elements where available
+
+## Downloading the data
+
+- A [pre-processed version](#pre-processed-data) of the data for the ClickHouse, which has been cleaned, re-structured, and enriched. This dataset covers the years from 1896 to 2023.
+- [Download the original raw data](#original-raw-data) and convert to the format required by ClickHouse. Users wanting to add their own columns may wish to explore or complete their approaches.
+
+### Pre-processed data
+
+The dataset has also been re-structured from a measurement per line to a row per weather station id and measured date, i.e.
+
+```csv
+StationId,MeasuredDate,StnPres,Tx,RH,WS,WD,WSGust,WDGust,Precp,GloblRad,TxSoil0cm,TxSoil5cm,TxSoil20cm,TxSoil50cm,TxSoil100cm,SeaPres,Td,PrecpHour,SunShine,TxSoil10cm,EvapA,Visb,UVI,Cloud Amount,TxSoil30cm,TxSoil200cm,TxSoil300cm,TxSoil500cm,VaporPressure
+C0X100,2016-01-01 01:00:00,1022.1,16.1,72,1.1,8.0,,,,,,,,,,,,,,,,,,,,,,,
+C0X100,2016-01-01 02:00:00,1021.6,16.0,73,1.2,358.0,,,,,,,,,,,,,,,,,,,,,,,
+C0X100,2016-01-01 03:00:00,1021.3,15.8,74,1.5,353.0,,,,,,,,,,,,,,,,,,,,,,,
+C0X100,2016-01-01 04:00:00,1021.2,15.8,74,1.7,8.0,,,,,,,,,,,,,,,,,,,,,,,
+```
+
+It is easy to query and ensure that the resulting table has less sparse and some elements are null because they're not available to be measured in this weather station.
+
+This dataset is available in the following Google CloudStorage location. Either download the dataset to your local filesystem (and insert them with the ClickHouse client) or insert them directly into the ClickHouse (see [Inserting from URL](#inserting-from-url)).
+
+To download:
+
+```bash
+wget https://storage.googleapis.com/taiwan-weather-observaiton-datasets/preprocessed_weather_daily_1896_2023.tar.gz
+
+# Option: Validate the checksum
+md5sum preprocessed_weather_daily_1896_2023.tar.gz
+# Checksum should be equal to: 11b484f5bd9ddafec5cfb131eb2dd008
+
+tar -xzvf preprocessed_weather_daily_1896_2023.tar.gz
+daily_weather_preprocessed_1896_2023.csv
+
+# Option: Validate the checksum
+md5sum daily_weather_preprocessed_1896_2023.csv
+# Checksum should be equal to: 1132248c78195c43d93f843753881754
+```
+
+### Original raw data
+
+The following details are about the steps to download the original raw data to transform and convert as you want.
+
+#### Download
+
+To download the original raw data:
+
+```bash
+mkdir tw_raw_weather_data && cd tw_raw_weather_data
+
+wget https://storage.googleapis.com/taiwan-weather-observaiton-datasets/raw_data_weather_daily_1896_2023.tar.gz
+
+# Option: Validate the checksum
+md5sum raw_data_weather_daily_1896_2023.tar.gz
+# Checksum should be equal to: b66b9f137217454d655e3004d7d1b51a
+
+tar -xzvf raw_data_weather_daily_1896_2023.tar.gz
+466920_1928.csv
+466920_1929.csv
+466920_1930.csv
+466920_1931.csv
+...
+
+# Option: Validate the checksum
+cat *.csv | md5sum
+# Checksum should be equal to: b26db404bf84d4063fac42e576464ce1
+```
+
+#### Retrieve the Taiwan weather stations
+
+```bash
+wget -O weather_sta_list.csv https://github.com/Raingel/weather_station_list/raw/main/data/weather_sta_list.csv
+
+# Option: Convert the UTF-8-BOM to UTF-8 encoding
+sed -i '1s/^\xEF\xBB\xBF//' weather_sta_list.csv
+```
+
+## Create table schema
+
+Create the MergeTree table in ClickHouse (from the ClickHouse client).
+
+```bash
+CREATE TABLE tw_weather_data (
+    StationId String null,
+    MeasuredDate DateTime64,
+    StnPres Float64 null,
+    SeaPres Float64 null,
+    Tx Float64 null,
+    Td Float64 null,
+    RH Float64 null,
+    WS Float64 null,
+    WD Float64 null,
+    WSGust Float64 null,
+    WDGust Float64 null,
+    Precp Float64 null,
+    PrecpHour Float64 null,
+    SunShine Float64 null,
+    GloblRad Float64 null,
+    TxSoil0cm Float64 null,
+    TxSoil5cm Float64 null,
+    TxSoil10cm Float64 null,
+    TxSoil20cm Float64 null,
+    TxSoil50cm Float64 null,
+    TxSoil100cm Float64 null,
+    TxSoil30cm Float64 null,
+    TxSoil200cm Float64 null,
+    TxSoil300cm Float64 null,
+    TxSoil500cm Float64 null,
+    VaporPressure Float64 null,
+    UVI Float64 null,
+    "Cloud Amount" Float64 null,
+    EvapA Float64 null,
+    Visb Float64 null
+)
+ENGINE = MergeTree
+ORDER BY (MeasuredDate);
+```
+
+## Inserting into ClickHouse
+
+### Inserting from local file
+
+Data can be inserted from a local file as follows (from the ClickHouse client):
+
+```sql
+INSERT INTO tw_weather_data FROM INFILE '/path/to/daily_weather_preprocessed_1896_2023.csv'
+```
+
+where `/path/to` represents the specific user path to the local file on the disk.
+
+And the sample response output is as follows after inserting data into the ClickHouse:
+
+```response
+Query id: 90e4b524-6e14-4855-817c-7e6f98fbeabb
+
+Ok.
+131985329 rows in set. Elapsed: 71.770 sec. Processed 131.99 million rows, 10.06 GB (1.84 million rows/s., 140.14 MB/s.)
+Peak memory usage: 583.23 MiB.
+```
+
+### Inserting from URL
+
+```sql
+INSERT INTO tw_weather_data SELECT *
+FROM url('https://storage.googleapis.com/taiwan-weather-observaiton-datasets/daily_weather_preprocessed_1896_2023.csv', 'CSVWithNames')
+
+```
+To know how to speed this up, please see our blog post on [tuning large data loads](https://clickhouse.com/blog/supercharge-your-clickhouse-data-loads-part2).
+
+## Check data rows and sizes
+
+1. Let's see how many rows are inserted:
+
+```sql
+SELECT formatReadableQuantity(count())
+FROM tw_weather_data;
+```
+
+```response
+┌─formatReadableQuantity(count())─┐
+│ 131.99 million                  │
+└─────────────────────────────────┘
+```
+
+2. Let's see how much disk space are used for this table:
+
+```sql
+SELECT
+    formatReadableSize(sum(bytes)) AS disk_size,
+    formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed_size
+FROM system.parts
+WHERE (`table` = 'tw_weather_data') AND active
+```
+
+```response
+┌─disk_size─┬─uncompressed_size─┐
+│ 2.13 GiB  │ 32.94 GiB         │
+└───────────┴───────────────────┘
+```
+
+## Sample queries
+
+### Q1: Retrieve the highest dew point temperature for each weather station in the specific year
+
+```sql
+SELECT
+    StationId,
+    max(Td) AS max_td
+FROM tw_weather_data
+WHERE (year(MeasuredDate) = 2023) AND (Td IS NOT NULL)
+GROUP BY StationId
+
+┌─StationId─┬─max_td─┐
+│ 466940    │      1 │
+│ 467300    │      1 │
+│ 467540    │      1 │
+│ 467490    │      1 │
+│ 467080    │      1 │
+│ 466910    │      1 │
+│ 467660    │      1 │
+│ 467270    │      1 │
+│ 467350    │      1 │
+│ 467571    │      1 │
+│ 466920    │      1 │
+│ 467650    │      1 │
+│ 467550    │      1 │
+│ 467480    │      1 │
+│ 467610    │      1 │
+│ 467050    │      1 │
+│ 467590    │      1 │
+│ 466990    │      1 │
+│ 467060    │      1 │
+│ 466950    │      1 │
+│ 467620    │      1 │
+│ 467990    │      1 │
+│ 466930    │      1 │
+│ 467110    │      1 │
+│ 466881    │      1 │
+│ 467410    │      1 │
+│ 467441    │      1 │
+│ 467420    │      1 │
+│ 467530    │      1 │
+│ 466900    │      1 │
+└───────────┴────────┘
+
+30 rows in set. Elapsed: 0.045 sec. Processed 6.41 million rows, 187.33 MB (143.92 million rows/s., 4.21 GB/s.)
+```
+
+### Q2: Raw data fetching with the specific duration time range, fields and weather station
+
+```sql
+SELECT
+    StnPres,
+    SeaPres,
+    Tx,
+    Td,
+    RH,
+    WS,
+    WD,
+    WSGust,
+    WDGust,
+    Precp,
+    PrecpHour
+FROM tw_weather_data
+WHERE (StationId = 'C0UB10') AND (MeasuredDate >= '2023-12-23') AND (MeasuredDate < '2023-12-24')
+ORDER BY MeasuredDate ASC
+LIMIT 10
+```
+
+```response
+┌─StnPres─┬─SeaPres─┬───Tx─┬───Td─┬─RH─┬──WS─┬──WD─┬─WSGust─┬─WDGust─┬─Precp─┬─PrecpHour─┐
+│  1029.5 │    ᴺᵁᴸᴸ │ 11.8 │ ᴺᵁᴸᴸ │ 78 │ 2.7 │ 271 │    5.5 │    275 │ -99.8 │     -99.8 │
+│  1029.8 │    ᴺᵁᴸᴸ │ 12.3 │ ᴺᵁᴸᴸ │ 78 │ 2.7 │ 289 │    5.5 │    308 │ -99.8 │     -99.8 │
+│  1028.6 │    ᴺᵁᴸᴸ │ 12.3 │ ᴺᵁᴸᴸ │ 79 │ 2.3 │ 251 │    6.1 │    289 │ -99.8 │     -99.8 │
+│  1028.2 │    ᴺᵁᴸᴸ │   13 │ ᴺᵁᴸᴸ │ 75 │ 4.3 │ 312 │    7.5 │    316 │ -99.8 │     -99.8 │
+│  1027.8 │    ᴺᵁᴸᴸ │ 11.1 │ ᴺᵁᴸᴸ │ 89 │ 7.1 │ 310 │   11.6 │    322 │ -99.8 │     -99.8 │
+│  1027.8 │    ᴺᵁᴸᴸ │ 11.6 │ ᴺᵁᴸᴸ │ 90 │ 3.1 │ 269 │   10.7 │    295 │ -99.8 │     -99.8 │
+│  1027.9 │    ᴺᵁᴸᴸ │ 12.3 │ ᴺᵁᴸᴸ │ 89 │ 4.7 │ 296 │    8.1 │    310 │ -99.8 │     -99.8 │
+│  1028.2 │    ᴺᵁᴸᴸ │ 12.2 │ ᴺᵁᴸᴸ │ 94 │ 2.5 │ 246 │    7.1 │    283 │ -99.8 │     -99.8 │
+│  1028.4 │    ᴺᵁᴸᴸ │ 12.5 │ ᴺᵁᴸᴸ │ 94 │ 3.1 │ 265 │    4.8 │    297 │ -99.8 │     -99.8 │
+│  1028.3 │    ᴺᵁᴸᴸ │ 13.6 │ ᴺᵁᴸᴸ │ 91 │ 1.2 │ 273 │    4.4 │    256 │ -99.8 │     -99.8 │
+└─────────┴─────────┴──────┴──────┴────┴─────┴─────┴────────┴────────┴───────┴───────────┘
+
+10 rows in set. Elapsed: 0.009 sec. Processed 91.70 thousand rows, 2.33 MB (9.67 million rows/s., 245.31 MB/s.)
+```
+
+## Credits
+
+We would like to acknowledge the efforts of the Central Weather Administration and Agricultural Meteorological Observation Network (Station) of the Council of Agriculture for preparing, cleaning, and distributing this dataset. We appreciate your efforts.
+
+Ou, J.-H., Kuo, C.-H., Wu, Y.-F., Lin, G.-C., Lee, M.-H., Chen, R.-K., Chou, H.-P., Wu, H.-Y., Chu, S.-C., Lai, Q.-J., Tsai, Y.-C., Lin, C.-C., Kuo, C.-C., Liao, C.-T., Chen, Y.-N., Chu, Y.-W., Chen, C.-Y., 2023. Application-oriented deep learning model for early warning of rice blast in Taiwan. Ecological Informatics 73, 101950. https://doi.org/10.1016/j.ecoinf.2022.101950 [13/12/2022]

From d6e0dd45b9cc88b9002de68138440cd24452fb17 Mon Sep 17 00:00:00 2001
From: Amos Bird <amosbird@gmail.com>
Date: Fri, 8 Mar 2024 22:57:49 +0800
Subject: [PATCH 08/92] Fix build

---
 src/Functions/coverage.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Functions/coverage.cpp b/src/Functions/coverage.cpp
index f4cac26df78..0f4cd1940b7 100644
--- a/src/Functions/coverage.cpp
+++ b/src/Functions/coverage.cpp
@@ -5,6 +5,7 @@
 #include <Columns/ColumnArray.h>
 #include <Columns/ColumnVector.h>
 #include <Columns/ColumnConst.h>
+#include <Columns/ColumnsNumber.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/IFunction.h>
 #include <Interpreters/Context.h>

From 5f1991fbef2f959f1d55c62194d948814d199fa9 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Tue, 12 Mar 2024 15:53:28 +0800
Subject: [PATCH 09/92] too big translation unit in Aggregator

---
 src/Common/HashTable/FixedHashMap.h           |    3 +
 .../HashTable/TwoLevelStringHashTable.h       |    1 +
 src/Interpreters/AggregatedData.h             |  142 +++
 src/Interpreters/AggregatedDataVariants.cpp   |  255 ++++
 src/Interpreters/AggregatedDataVariants.h     |  320 +++++
 src/Interpreters/AggregationMethod.cpp        |  215 ++++
 src/Interpreters/AggregationMethod.h          |  320 +++++
 src/Interpreters/Aggregator.cpp               |  512 ++++----
 src/Interpreters/Aggregator.h                 | 1035 +----------------
 9 files changed, 1541 insertions(+), 1262 deletions(-)
 create mode 100644 src/Interpreters/AggregatedData.h
 create mode 100644 src/Interpreters/AggregatedDataVariants.cpp
 create mode 100644 src/Interpreters/AggregatedDataVariants.h
 create mode 100644 src/Interpreters/AggregationMethod.cpp
 create mode 100644 src/Interpreters/AggregationMethod.h

diff --git a/src/Common/HashTable/FixedHashMap.h b/src/Common/HashTable/FixedHashMap.h
index e835a6fba94..537f37a9e6c 100644
--- a/src/Common/HashTable/FixedHashMap.h
+++ b/src/Common/HashTable/FixedHashMap.h
@@ -109,6 +109,9 @@ public:
 
     using Base::Base;
 
+    FixedHashMap() = default;
+    FixedHashMap(size_t ) {} /// NOLINT
+
     template <typename Func, bool>
     void ALWAYS_INLINE mergeToViaEmplace(Self & that, Func && func)
     {
diff --git a/src/Common/HashTable/TwoLevelStringHashTable.h b/src/Common/HashTable/TwoLevelStringHashTable.h
index 54c208c5b60..1ce6b3d02e3 100644
--- a/src/Common/HashTable/TwoLevelStringHashTable.h
+++ b/src/Common/HashTable/TwoLevelStringHashTable.h
@@ -38,6 +38,7 @@ public:
     Impl impls[NUM_BUCKETS];
 
     TwoLevelStringHashTable() = default;
+    TwoLevelStringHashTable(size_t ) {} /// NOLINT
 
     template <typename Source>
     explicit TwoLevelStringHashTable(const Source & src)
diff --git a/src/Interpreters/AggregatedData.h b/src/Interpreters/AggregatedData.h
new file mode 100644
index 00000000000..6cd6b190801
--- /dev/null
+++ b/src/Interpreters/AggregatedData.h
@@ -0,0 +1,142 @@
+#pragma once
+#include <AggregateFunctions/IAggregateFunction.h>
+
+#include <Common/HashTable/FixedHashMap.h>
+#include <Common/HashTable/StringHashMap.h>
+#include <Common/HashTable/TwoLevelHashMap.h>
+#include <Common/HashTable/TwoLevelStringHashMap.h>
+namespace DB
+{
+/** Different data structures that can be used for aggregation
+  * For efficiency, the aggregation data itself is put into the pool.
+  * Data and pool ownership (states of aggregate functions)
+  *  is acquired later - in `convertToBlocks` function, by the ColumnAggregateFunction object.
+  *
+  * Most data structures exist in two versions: normal and two-level (TwoLevel).
+  * A two-level hash table works a little slower with a small number of different keys,
+  *  but with a large number of different keys scales better, because it allows
+  *  parallelize some operations (merging, post-processing) in a natural way.
+  *
+  * To ensure efficient work over a wide range of conditions,
+  *  first single-level hash tables are used,
+  *  and when the number of different keys is large enough,
+  *  they are converted to two-level ones.
+  *
+  * PS. There are many different approaches to the effective implementation of parallel and distributed aggregation,
+  *  best suited for different cases, and this approach is just one of them, chosen for a combination of reasons.
+  */
+
+using AggregatedDataWithoutKey = AggregateDataPtr;
+
+using AggregatedDataWithUInt8Key = FixedImplicitZeroHashMapWithCalculatedSize<UInt8, AggregateDataPtr>;
+using AggregatedDataWithUInt16Key = FixedImplicitZeroHashMap<UInt16, AggregateDataPtr>;
+
+using AggregatedDataWithUInt32Key = HashMap<UInt32, AggregateDataPtr, HashCRC32<UInt32>>;
+using AggregatedDataWithUInt64Key = HashMap<UInt64, AggregateDataPtr, HashCRC32<UInt64>>;
+
+using AggregatedDataWithShortStringKey = StringHashMap<AggregateDataPtr>;
+
+using AggregatedDataWithStringKey = HashMapWithSavedHash<StringRef, AggregateDataPtr>;
+
+using AggregatedDataWithKeys128 = HashMap<UInt128, AggregateDataPtr, UInt128HashCRC32>;
+using AggregatedDataWithKeys256 = HashMap<UInt256, AggregateDataPtr, UInt256HashCRC32>;
+
+using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap<UInt32, AggregateDataPtr, HashCRC32<UInt32>>;
+using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap<UInt64, AggregateDataPtr, HashCRC32<UInt64>>;
+
+using AggregatedDataWithShortStringKeyTwoLevel = TwoLevelStringHashMap<AggregateDataPtr>;
+
+using AggregatedDataWithStringKeyTwoLevel = TwoLevelHashMapWithSavedHash<StringRef, AggregateDataPtr>;
+
+using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap<UInt128, AggregateDataPtr, UInt128HashCRC32>;
+using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap<UInt256, AggregateDataPtr, UInt256HashCRC32>;
+
+/** Variants with better hash function, using more than 32 bits for hash.
+  * Using for merging phase of external aggregation, where number of keys may be far greater than 4 billion,
+  *  but we keep in memory and merge only sub-partition of them simultaneously.
+  * TODO We need to switch for better hash function not only for external aggregation,
+  *  but also for huge aggregation results on machines with terabytes of RAM.
+  */
+
+using AggregatedDataWithUInt64KeyHash64 = HashMap<UInt64, AggregateDataPtr, DefaultHash<UInt64>>;
+using AggregatedDataWithStringKeyHash64 = HashMapWithSavedHash<StringRef, AggregateDataPtr, StringRefHash64>;
+using AggregatedDataWithKeys128Hash64 = HashMap<UInt128, AggregateDataPtr, UInt128Hash>;
+using AggregatedDataWithKeys256Hash64 = HashMap<UInt256, AggregateDataPtr, UInt256Hash>;
+
+template <typename Base>
+struct AggregationDataWithNullKey : public Base
+{
+    using Base::Base;
+
+    bool & hasNullKeyData() { return has_null_key_data; }
+    AggregateDataPtr & getNullKeyData() { return null_key_data; }
+    bool hasNullKeyData() const { return has_null_key_data; }
+    const AggregateDataPtr & getNullKeyData() const { return null_key_data; }
+    size_t size() const { return Base::size() + (has_null_key_data ? 1 : 0); }
+    bool empty() const { return Base::empty() && !has_null_key_data; }
+    void clear()
+    {
+        Base::clear();
+        has_null_key_data = false;
+    }
+    void clearAndShrink()
+    {
+        Base::clearAndShrink();
+        has_null_key_data = false;
+    }
+
+private:
+    bool has_null_key_data = false;
+    AggregateDataPtr null_key_data = nullptr;
+};
+
+template <typename Base>
+struct AggregationDataWithNullKeyTwoLevel : public Base
+{
+    using Base::Base;
+    using Base::impls;
+
+    AggregationDataWithNullKeyTwoLevel() = default;
+
+    template <typename Other>
+    explicit AggregationDataWithNullKeyTwoLevel(const Other & other) : Base(other)
+    {
+        impls[0].hasNullKeyData() = other.hasNullKeyData();
+        impls[0].getNullKeyData() = other.getNullKeyData();
+    }
+
+    bool & hasNullKeyData() { return impls[0].hasNullKeyData(); }
+    AggregateDataPtr & getNullKeyData() { return impls[0].getNullKeyData(); }
+    bool hasNullKeyData() const { return impls[0].hasNullKeyData(); }
+    const AggregateDataPtr & getNullKeyData() const { return impls[0].getNullKeyData(); }
+};
+
+template <typename ... Types>
+using HashTableWithNullKey = AggregationDataWithNullKey<HashMapTable<Types ...>>;
+template <typename ... Types>
+using StringHashTableWithNullKey = AggregationDataWithNullKey<StringHashMap<Types ...>>;
+
+using AggregatedDataWithNullableUInt8Key = AggregationDataWithNullKey<AggregatedDataWithUInt8Key>;
+using AggregatedDataWithNullableUInt16Key = AggregationDataWithNullKey<AggregatedDataWithUInt16Key>;
+using AggregatedDataWithNullableUInt32Key = AggregationDataWithNullKey<AggregatedDataWithUInt32Key>;
+
+
+using AggregatedDataWithNullableUInt64Key = AggregationDataWithNullKey<AggregatedDataWithUInt64Key>;
+using AggregatedDataWithNullableStringKey = AggregationDataWithNullKey<AggregatedDataWithStringKey>;
+using AggregatedDataWithNullableShortStringKey = AggregationDataWithNullKey<AggregatedDataWithShortStringKey>;
+
+
+using AggregatedDataWithNullableUInt32KeyTwoLevel = AggregationDataWithNullKeyTwoLevel<
+    TwoLevelHashMap<UInt32, AggregateDataPtr, HashCRC32<UInt32>,
+                    TwoLevelHashTableGrower<>, HashTableAllocator, HashTableWithNullKey>>;
+using AggregatedDataWithNullableUInt64KeyTwoLevel = AggregationDataWithNullKeyTwoLevel<
+        TwoLevelHashMap<UInt64, AggregateDataPtr, HashCRC32<UInt64>,
+        TwoLevelHashTableGrower<>, HashTableAllocator, HashTableWithNullKey>>;
+
+using AggregatedDataWithNullableShortStringKeyTwoLevel = AggregationDataWithNullKeyTwoLevel<
+        TwoLevelStringHashMap<AggregateDataPtr, HashTableAllocator, StringHashTableWithNullKey>>;
+
+using AggregatedDataWithNullableStringKeyTwoLevel = AggregationDataWithNullKeyTwoLevel<
+        TwoLevelHashMapWithSavedHash<StringRef, AggregateDataPtr, DefaultHash<StringRef>,
+        TwoLevelHashTableGrower<>, HashTableAllocator, HashTableWithNullKey>>;
+}
diff --git a/src/Interpreters/AggregatedDataVariants.cpp b/src/Interpreters/AggregatedDataVariants.cpp
new file mode 100644
index 00000000000..0c86c58bd3e
--- /dev/null
+++ b/src/Interpreters/AggregatedDataVariants.cpp
@@ -0,0 +1,255 @@
+#include <Interpreters/AggregatedDataVariants.h>
+#include <Interpreters/Aggregator.h>
+
+namespace ProfileEvents
+{
+    extern const Event AggregationPreallocatedElementsInHashTables;
+}
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int UNKNOWN_AGGREGATED_DATA_VARIANT;
+    extern const int LOGICAL_ERROR;
+
+}
+using ColumnsHashing::HashMethodContext;
+using ColumnsHashing::HashMethodContextPtr;
+using ColumnsHashing::LastElementCacheStats;
+
+AggregatedDataVariants::AggregatedDataVariants() : aggregates_pools(1, std::make_shared<Arena>()), aggregates_pool(aggregates_pools.back().get()) {}
+
+AggregatedDataVariants::~AggregatedDataVariants()
+{
+    if (aggregator && !aggregator->all_aggregates_has_trivial_destructor)
+    {
+        try
+        {
+            aggregator->destroyAllAggregateStates(*this);
+        }
+        catch (...)
+        {
+            tryLogCurrentException(__PRETTY_FUNCTION__);
+        }
+    }
+}
+
+// The std::is_constructible trait isn't suitable here because some classes have template constructors with semantics different from providing size hints.
+// Also string hash table variants are not supported due to the fact that both local perf tests and tests in CI showed slowdowns for them.
+template <typename...>
+struct HasConstructorOfNumberOfElements : std::false_type
+{
+};
+
+template <typename... Ts>
+struct HasConstructorOfNumberOfElements<HashMapTable<Ts...>> : std::true_type
+{
+};
+
+template <typename Key, typename Cell, typename Hash, typename Grower, typename Allocator, template <typename...> typename ImplTable>
+struct HasConstructorOfNumberOfElements<TwoLevelHashMapTable<Key, Cell, Hash, Grower, Allocator, ImplTable>> : std::true_type
+{
+};
+
+template <typename... Ts>
+struct HasConstructorOfNumberOfElements<HashTable<Ts...>> : std::true_type
+{
+};
+
+template <typename... Ts>
+struct HasConstructorOfNumberOfElements<TwoLevelHashTable<Ts...>> : std::true_type
+{
+};
+
+template <template <typename> typename Method, typename Base>
+struct HasConstructorOfNumberOfElements<Method<Base>> : HasConstructorOfNumberOfElements<Base>
+{
+};
+
+template <typename Method>
+auto constructWithReserveIfPossible(size_t size_hint)
+{
+    if constexpr (HasConstructorOfNumberOfElements<typename Method::Data>::value)
+    {
+        ProfileEvents::increment(ProfileEvents::AggregationPreallocatedElementsInHashTables, size_hint);
+        return std::make_unique<Method>(size_hint);
+    }
+    else
+        return std::make_unique<Method>();
+}
+
+void AggregatedDataVariants::init(Type type_, std::optional<size_t> size_hint)
+{
+    switch (type_)
+    {
+        case Type::EMPTY:
+        case Type::without_key:
+            break;
+
+    #define M(NAME, IS_TWO_LEVEL) \
+        case Type::NAME: \
+            if (size_hint) \
+                (NAME) = constructWithReserveIfPossible<decltype(NAME)::element_type>(*size_hint); \
+            else \
+                (NAME) = std::make_unique<decltype(NAME)::element_type>(); \
+            break;
+        APPLY_FOR_AGGREGATED_VARIANTS(M)
+#undef M
+    }
+
+    type = type_;
+}
+
+size_t AggregatedDataVariants::size() const
+{
+    switch (type)
+    {
+        case Type::EMPTY:
+            return 0;
+        case Type::without_key:
+            return 1;
+
+    #define M(NAME, IS_TWO_LEVEL) \
+        case Type::NAME: \
+            return (NAME)->data.size() + (without_key != nullptr);
+        APPLY_FOR_AGGREGATED_VARIANTS(M)
+    #undef M
+    }
+
+    UNREACHABLE();
+}
+
+size_t AggregatedDataVariants::sizeWithoutOverflowRow() const
+{
+    switch (type)
+    {
+        case Type::EMPTY:
+        return 0;
+        case Type::without_key:
+        return 1;
+
+    #define M(NAME, IS_TWO_LEVEL) \
+        case Type::NAME: \
+            return (NAME)->data.size();
+        APPLY_FOR_AGGREGATED_VARIANTS(M)
+    #undef M
+    }
+
+    UNREACHABLE();
+}
+
+const char * AggregatedDataVariants::getMethodName() const
+{
+    switch (type)
+    {
+        case Type::EMPTY:
+        return "EMPTY";
+        case Type::without_key:
+        return "without_key";
+
+    #define M(NAME, IS_TWO_LEVEL) \
+        case Type::NAME: \
+            return #NAME;
+        APPLY_FOR_AGGREGATED_VARIANTS(M)
+    #undef M
+    }
+
+    UNREACHABLE();
+}
+
+bool AggregatedDataVariants::isTwoLevel() const
+{
+    switch (type)
+    {
+        case Type::EMPTY:
+        return false;
+        case Type::without_key:
+        return false;
+
+    #define M(NAME, IS_TWO_LEVEL) \
+        case Type::NAME: \
+            return IS_TWO_LEVEL;
+        APPLY_FOR_AGGREGATED_VARIANTS(M)
+    #undef M
+    }
+
+    UNREACHABLE();
+}
+
+bool AggregatedDataVariants::isConvertibleToTwoLevel() const
+{
+    switch (type)
+    {
+    #define M(NAME) \
+        case Type::NAME: \
+            return true;
+
+        APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M)
+
+    #undef M
+        default:
+        return false;
+    }
+}
+
+void AggregatedDataVariants::convertToTwoLevel()
+{
+    if (aggregator)
+        LOG_TRACE(aggregator->log, "Converting aggregation data to two-level.");
+
+    switch (type)
+    {
+#define M(NAME) \
+        case Type::NAME: \
+            NAME ## _two_level = std::make_unique<decltype(NAME ## _two_level)::element_type>(*(NAME)); \
+            (NAME).reset(); \
+            type = Type::NAME ## _two_level; \
+            break;
+
+        APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M)
+
+    #undef M
+
+        default:
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong data variant passed.");
+    }
+}
+
+bool AggregatedDataVariants::isLowCardinality() const
+{
+    switch (type)
+    {
+    #define M(NAME) \
+        case Type::NAME: \
+            return true;
+
+        APPLY_FOR_LOW_CARDINALITY_VARIANTS(M)
+    #undef M
+        default:
+            return false;
+    }
+}
+
+HashMethodContextPtr AggregatedDataVariants::createCache(Type type, const HashMethodContext::Settings & settings)
+{
+    switch (type)
+    {
+        case Type::without_key:
+            return nullptr;
+
+    #define M(NAME, IS_TWO_LEVEL) \
+        case Type::NAME: { \
+            using TPtr##NAME = decltype(AggregatedDataVariants::NAME); \
+            using T##NAME = typename TPtr##NAME ::element_type; \
+            return T##NAME ::State::createContext(settings); \
+        }
+
+            APPLY_FOR_AGGREGATED_VARIANTS(M)
+    #undef M
+
+        default:
+            throw Exception(ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT, "Unknown aggregated data variant.");
+    }
+}
+}
diff --git a/src/Interpreters/AggregatedDataVariants.h b/src/Interpreters/AggregatedDataVariants.h
new file mode 100644
index 00000000000..8b82c5d9842
--- /dev/null
+++ b/src/Interpreters/AggregatedDataVariants.h
@@ -0,0 +1,320 @@
+#pragma once
+#include <boost/noncopyable.hpp>
+#include <memory.h>
+#include <AggregateFunctions/IAggregateFunction.h>
+#include <Common/ColumnsHashing.h>
+#include <Interpreters/AggregatedData.h>
+#include <Interpreters/AggregationMethod.h>
+
+namespace DB
+{
+class Arena;
+class Aggregator;
+
+struct AggregatedDataVariants : private boost::noncopyable
+{
+    /** Working with states of aggregate functions in the pool is arranged in the following (inconvenient) way:
+      * - when aggregating, states are created in the pool using IAggregateFunction::create (inside - `placement new` of arbitrary structure);
+      * - they must then be destroyed using IAggregateFunction::destroy (inside - calling the destructor of arbitrary structure);
+      * - if aggregation is complete, then, in the Aggregator::convertToBlocks function, pointers to the states of aggregate functions
+      *   are written to ColumnAggregateFunction; ColumnAggregateFunction "acquires ownership" of them, that is - calls `destroy` in its destructor.
+      * - if during the aggregation, before call to Aggregator::convertToBlocks, an exception was thrown,
+      *   then the states of aggregate functions must still be destroyed,
+      *   otherwise, for complex states (eg, AggregateFunctionUniq), there will be memory leaks;
+      * - in this case, to destroy states, the destructor calls Aggregator::destroyAggregateStates method,
+      *   but only if the variable aggregator (see below) is not nullptr;
+      * - that is, until you transfer ownership of the aggregate function states in the ColumnAggregateFunction, set the variable `aggregator`,
+      *   so that when an exception occurs, the states are correctly destroyed.
+      *
+      * PS. This can be corrected by making a pool that knows about which states of aggregate functions and in which order are put in it, and knows how to destroy them.
+      * But this can hardly be done simply because it is planned to put variable-length strings into the same pool.
+      * In this case, the pool will not be able to know with what offsets objects are stored.
+      */
+    const Aggregator * aggregator = nullptr;
+
+    size_t keys_size{};  /// Number of keys. NOTE do we need this field?
+    Sizes key_sizes;     /// Dimensions of keys, if keys of fixed length
+
+    /// Pools for states of aggregate functions. Ownership will be later transferred to ColumnAggregateFunction.
+    using ArenaPtr = std::shared_ptr<Arena>;
+    using Arenas = std::vector<ArenaPtr>;
+    Arenas aggregates_pools;
+    Arena * aggregates_pool{};    /// The pool that is currently used for allocation.
+
+    /** Specialization for the case when there are no keys, and for keys not fitted into max_rows_to_group_by.
+      */
+    AggregatedDataWithoutKey without_key = nullptr;
+
+    /// Stats of a cache for consecutive keys optimization.
+    /// Stats can be used to disable the cache in case of a lot of misses.
+    ColumnsHashing::LastElementCacheStats consecutive_keys_cache_stats;
+
+    // Disable consecutive key optimization for Uint8/16, because they use a FixedHashMap
+    // and the lookup there is almost free, so we don't need to cache the last lookup result
+    std::unique_ptr<AggregationMethodOneNumber<UInt8, AggregatedDataWithUInt8Key, false>>           key8;
+    std::unique_ptr<AggregationMethodOneNumber<UInt16, AggregatedDataWithUInt16Key, false>>         key16;
+
+    std::unique_ptr<AggregationMethodOneNumber<UInt32, AggregatedDataWithUInt64Key>>         key32;
+    std::unique_ptr<AggregationMethodOneNumber<UInt64, AggregatedDataWithUInt64Key>>         key64;
+    std::unique_ptr<AggregationMethodStringNoCache<AggregatedDataWithShortStringKey>>               key_string;
+    std::unique_ptr<AggregationMethodFixedStringNoCache<AggregatedDataWithShortStringKey>>          key_fixed_string;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithUInt16Key, false, false, false>>  keys16;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithUInt32Key>>                   keys32;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithUInt64Key>>                   keys64;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128>>                   keys128;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256>>                   keys256;
+    std::unique_ptr<AggregationMethodSerialized<AggregatedDataWithStringKey>>                          serialized;
+    std::unique_ptr<AggregationMethodNullableSerialized<AggregatedDataWithStringKey>>                  nullable_serialized;
+    std::unique_ptr<AggregationMethodPreallocSerialized<AggregatedDataWithStringKey>>                  prealloc_serialized;
+    std::unique_ptr<AggregationMethodNullablePreallocSerialized<AggregatedDataWithStringKey>>          nullable_prealloc_serialized;
+
+    std::unique_ptr<AggregationMethodOneNumber<UInt32, AggregatedDataWithUInt64KeyTwoLevel>> key32_two_level;
+    std::unique_ptr<AggregationMethodOneNumber<UInt64, AggregatedDataWithUInt64KeyTwoLevel>> key64_two_level;
+    std::unique_ptr<AggregationMethodStringNoCache<AggregatedDataWithShortStringKeyTwoLevel>>       key_string_two_level;
+    std::unique_ptr<AggregationMethodFixedStringNoCache<AggregatedDataWithShortStringKeyTwoLevel>>  key_fixed_string_two_level;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithUInt32KeyTwoLevel>>           keys32_two_level;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithUInt64KeyTwoLevel>>           keys64_two_level;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128TwoLevel>>           keys128_two_level;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256TwoLevel>>           keys256_two_level;
+    std::unique_ptr<AggregationMethodSerialized<AggregatedDataWithStringKeyTwoLevel>>                  serialized_two_level;
+    std::unique_ptr<AggregationMethodNullableSerialized<AggregatedDataWithStringKeyTwoLevel>>          nullable_serialized_two_level;
+    std::unique_ptr<AggregationMethodPreallocSerialized<AggregatedDataWithStringKeyTwoLevel>>          prealloc_serialized_two_level;
+    std::unique_ptr<AggregationMethodNullablePreallocSerialized<AggregatedDataWithStringKeyTwoLevel>>  nullable_prealloc_serialized_two_level;
+
+    std::unique_ptr<AggregationMethodOneNumber<UInt64, AggregatedDataWithUInt64KeyHash64>>   key64_hash64;
+    std::unique_ptr<AggregationMethodString<AggregatedDataWithStringKeyHash64>>              key_string_hash64;
+    std::unique_ptr<AggregationMethodFixedString<AggregatedDataWithStringKeyHash64>>         key_fixed_string_hash64;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128Hash64>>             keys128_hash64;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256Hash64>>             keys256_hash64;
+    std::unique_ptr<AggregationMethodSerialized<AggregatedDataWithStringKeyHash64>>                  serialized_hash64;
+    std::unique_ptr<AggregationMethodNullableSerialized<AggregatedDataWithStringKeyHash64>>          nullable_serialized_hash64;
+    std::unique_ptr<AggregationMethodPreallocSerialized<AggregatedDataWithStringKeyHash64>>          prealloc_serialized_hash64;
+    std::unique_ptr<AggregationMethodNullablePreallocSerialized<AggregatedDataWithStringKeyHash64>>  nullable_prealloc_serialized_hash64;
+
+    /// Support for nullable keys.
+    std::unique_ptr<AggregationMethodOneNumber<UInt8, AggregatedDataWithNullableUInt8Key, false, true>>         nullable_key8;
+    std::unique_ptr<AggregationMethodOneNumber<UInt16, AggregatedDataWithNullableUInt16Key, false, true>>         nullable_key16;
+    std::unique_ptr<AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt32Key, true, true>>         nullable_key32;
+    std::unique_ptr<AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64Key, true, true>>         nullable_key64;
+    std::unique_ptr<AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt32KeyTwoLevel, true, true>>         nullable_key32_two_level;
+    std::unique_ptr<AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64KeyTwoLevel, true, true>>         nullable_key64_two_level;
+
+    std::unique_ptr<AggregationMethodStringNoCache<AggregatedDataWithNullableShortStringKey, true>> nullable_key_string;
+    std::unique_ptr<AggregationMethodFixedStringNoCache<AggregatedDataWithNullableShortStringKey, true>> nullable_key_fixed_string;
+    std::unique_ptr<AggregationMethodStringNoCache<AggregatedDataWithNullableShortStringKeyTwoLevel, true>> nullable_key_string_two_level;
+    std::unique_ptr<AggregationMethodFixedStringNoCache<AggregatedDataWithNullableShortStringKeyTwoLevel, true>> nullable_key_fixed_string_two_level;
+
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128, true>>             nullable_keys128;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256, true>>             nullable_keys256;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128TwoLevel, true>>     nullable_keys128_two_level;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256TwoLevel, true>>     nullable_keys256_two_level;
+
+    /// Support for low cardinality.
+    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt8, AggregatedDataWithNullableUInt8Key, false>>> low_cardinality_key8;
+    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt16, AggregatedDataWithNullableUInt16Key, false>>> low_cardinality_key16;
+    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt64Key>>> low_cardinality_key32;
+    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64Key>>> low_cardinality_key64;
+    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodString<AggregatedDataWithNullableStringKey>>> low_cardinality_key_string;
+    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodFixedString<AggregatedDataWithNullableStringKey>>> low_cardinality_key_fixed_string;
+
+    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt64KeyTwoLevel>>> low_cardinality_key32_two_level;
+    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64KeyTwoLevel>>> low_cardinality_key64_two_level;
+    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodString<AggregatedDataWithNullableStringKeyTwoLevel>>> low_cardinality_key_string_two_level;
+    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodFixedString<AggregatedDataWithNullableStringKeyTwoLevel>>> low_cardinality_key_fixed_string_two_level;
+
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128, false, true>>      low_cardinality_keys128;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256, false, true>>      low_cardinality_keys256;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128TwoLevel, false, true>> low_cardinality_keys128_two_level;
+    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256TwoLevel, false, true>> low_cardinality_keys256_two_level;
+
+    /// In this and similar macros, the option without_key is not considered.
+    #define APPLY_FOR_AGGREGATED_VARIANTS(M) \
+        M(key8,                       false) \
+        M(key16,                      false) \
+        M(key32,                      false) \
+        M(key64,                      false) \
+        M(key_string,                 false) \
+        M(key_fixed_string,           false) \
+        M(keys16,                    false) \
+        M(keys32,                    false) \
+        M(keys64,                    false) \
+        M(keys128,                    false) \
+        M(keys256,                    false) \
+        M(serialized,                   false) \
+        M(nullable_serialized,          false) \
+        M(prealloc_serialized,          false) \
+        M(nullable_prealloc_serialized, false) \
+        M(key32_two_level,            true) \
+        M(key64_two_level,            true) \
+        M(key_string_two_level,       true) \
+        M(key_fixed_string_two_level, true) \
+        M(keys32_two_level,          true) \
+        M(keys64_two_level,          true) \
+        M(keys128_two_level,          true) \
+        M(keys256_two_level,          true) \
+        M(serialized_two_level,                   true) \
+        M(nullable_serialized_two_level,          true) \
+        M(prealloc_serialized_two_level,          true) \
+        M(nullable_prealloc_serialized_two_level, true) \
+        M(key64_hash64,               false) \
+        M(key_string_hash64,          false) \
+        M(key_fixed_string_hash64,    false) \
+        M(keys128_hash64,             false) \
+        M(keys256_hash64,             false) \
+        M(serialized_hash64,                   false) \
+        M(nullable_serialized_hash64,          false) \
+        M(prealloc_serialized_hash64,          false) \
+        M(nullable_prealloc_serialized_hash64, false) \
+        M(nullable_key8,             false) \
+        M(nullable_key16,             false) \
+        M(nullable_key32,             false) \
+        M(nullable_key64,             false) \
+        M(nullable_key32_two_level,   true) \
+        M(nullable_key64_two_level,   true) \
+        M(nullable_key_string,        false) \
+        M(nullable_key_fixed_string,  false) \
+        M(nullable_key_string_two_level, true) \
+        M(nullable_key_fixed_string_two_level, true) \
+        M(nullable_keys128,           false) \
+        M(nullable_keys256,           false) \
+        M(nullable_keys128_two_level, true) \
+        M(nullable_keys256_two_level, true) \
+        M(low_cardinality_key8, false) \
+        M(low_cardinality_key16, false) \
+        M(low_cardinality_key32, false) \
+        M(low_cardinality_key64, false) \
+        M(low_cardinality_keys128, false) \
+        M(low_cardinality_keys256, false) \
+        M(low_cardinality_key_string, false) \
+        M(low_cardinality_key_fixed_string, false) \
+        M(low_cardinality_key32_two_level, true) \
+        M(low_cardinality_key64_two_level, true) \
+        M(low_cardinality_keys128_two_level, true) \
+        M(low_cardinality_keys256_two_level, true) \
+        M(low_cardinality_key_string_two_level, true) \
+        M(low_cardinality_key_fixed_string_two_level, true) \
+
+    #define APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) \
+        M(key32)            \
+        M(key64)            \
+        M(key_string)       \
+        M(key_fixed_string) \
+        M(keys32)           \
+        M(keys64)           \
+        M(keys128)          \
+        M(keys256)          \
+        M(serialized)       \
+        M(nullable_serialized) \
+        M(prealloc_serialized) \
+        M(nullable_prealloc_serialized) \
+        M(nullable_key32) \
+        M(nullable_key64) \
+        M(nullable_key_string) \
+        M(nullable_key_fixed_string) \
+        M(nullable_keys128) \
+        M(nullable_keys256) \
+        M(low_cardinality_key32) \
+        M(low_cardinality_key64) \
+        M(low_cardinality_keys128) \
+        M(low_cardinality_keys256) \
+        M(low_cardinality_key_string) \
+        M(low_cardinality_key_fixed_string) \
+
+    /// NOLINTNEXTLINE
+    #define APPLY_FOR_VARIANTS_NOT_CONVERTIBLE_TO_TWO_LEVEL(M) \
+        M(key8)             \
+        M(key16)            \
+        M(nullable_key8) \
+        M(nullable_key16) \
+        M(keys16)           \
+        M(key64_hash64)     \
+        M(key_string_hash64)\
+        M(key_fixed_string_hash64) \
+        M(keys128_hash64)   \
+        M(keys256_hash64)   \
+        M(serialized_hash64) \
+        M(nullable_serialized_hash64) \
+        M(prealloc_serialized_hash64) \
+        M(nullable_prealloc_serialized_hash64) \
+        M(low_cardinality_key8) \
+        M(low_cardinality_key16) \
+
+    /// NOLINTNEXTLINE
+    #define APPLY_FOR_VARIANTS_SINGLE_LEVEL(M) \
+        APPLY_FOR_VARIANTS_NOT_CONVERTIBLE_TO_TWO_LEVEL(M) \
+        APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) \
+
+    /// NOLINTNEXTLINE
+    #define APPLY_FOR_VARIANTS_TWO_LEVEL(M) \
+        M(key32_two_level)            \
+        M(key64_two_level)            \
+        M(key_string_two_level)       \
+        M(key_fixed_string_two_level) \
+        M(keys32_two_level)           \
+        M(keys64_two_level)           \
+        M(keys128_two_level)          \
+        M(keys256_two_level)          \
+        M(serialized_two_level)       \
+        M(nullable_serialized_two_level)       \
+        M(prealloc_serialized_two_level)       \
+        M(nullable_prealloc_serialized_two_level)       \
+        M(nullable_key32_two_level) \
+        M(nullable_key64_two_level) \
+        M(nullable_key_string_two_level) \
+        M(nullable_key_fixed_string_two_level) \
+        M(nullable_keys128_two_level) \
+        M(nullable_keys256_two_level) \
+        M(low_cardinality_key32_two_level) \
+        M(low_cardinality_key64_two_level) \
+        M(low_cardinality_keys128_two_level) \
+        M(low_cardinality_keys256_two_level) \
+        M(low_cardinality_key_string_two_level) \
+        M(low_cardinality_key_fixed_string_two_level) \
+
+    #define APPLY_FOR_LOW_CARDINALITY_VARIANTS(M) \
+        M(low_cardinality_key8) \
+        M(low_cardinality_key16) \
+        M(low_cardinality_key32) \
+        M(low_cardinality_key64) \
+        M(low_cardinality_keys128) \
+        M(low_cardinality_keys256) \
+        M(low_cardinality_key_string) \
+        M(low_cardinality_key_fixed_string) \
+        M(low_cardinality_key32_two_level) \
+        M(low_cardinality_key64_two_level) \
+        M(low_cardinality_keys128_two_level) \
+        M(low_cardinality_keys256_two_level) \
+        M(low_cardinality_key_string_two_level) \
+        M(low_cardinality_key_fixed_string_two_level)
+
+    enum class Type
+    {
+        EMPTY = 0,
+        without_key,
+
+    #define M(NAME, IS_TWO_LEVEL) NAME,
+        APPLY_FOR_AGGREGATED_VARIANTS(M)
+    #undef M
+    };
+    Type type = Type::EMPTY;
+    AggregatedDataVariants();
+    ~AggregatedDataVariants();
+    bool empty() const { return type == Type::EMPTY; }
+    void invalidate() { type = Type::EMPTY; }
+    void init(Type type_, std::optional<size_t> size_hint = std::nullopt);
+    /// Number of rows (different keys).
+    size_t size() const;
+    size_t sizeWithoutOverflowRow() const;
+    const char * getMethodName() const;
+    bool isTwoLevel() const;
+    bool isConvertibleToTwoLevel() const;
+    void convertToTwoLevel();
+    bool isLowCardinality() const;
+    static ColumnsHashing::HashMethodContextPtr createCache(Type type, const ColumnsHashing::HashMethodContext::Settings & settings);
+
+};
+
+using AggregatedDataVariantsPtr = std::shared_ptr<AggregatedDataVariants>;
+using ManyAggregatedDataVariants = std::vector<AggregatedDataVariantsPtr>;
+using ManyAggregatedDataVariantsPtr = std::shared_ptr<ManyAggregatedDataVariants>;
+}
diff --git a/src/Interpreters/AggregationMethod.cpp b/src/Interpreters/AggregationMethod.cpp
new file mode 100644
index 00000000000..3ff4f0cae43
--- /dev/null
+++ b/src/Interpreters/AggregationMethod.cpp
@@ -0,0 +1,215 @@
+#include <Interpreters/AggregationMethod.h>
+
+namespace DB
+{
+template <typename FieldType, typename TData, bool consecutive_keys_optimization, bool nullable>
+void AggregationMethodOneNumber<FieldType, TData, consecutive_keys_optimization, nullable>::insertKeyIntoColumns(
+    const AggregationMethodOneNumber::Key & key, std::vector<IColumn *> & key_columns, const Sizes & /*key_sizes*/)
+{
+    ColumnFixedSizeHelper * column;
+    if constexpr (nullable)
+    {
+        ColumnNullable & nullable_col = assert_cast<ColumnNullable &>(*key_columns[0]);
+        ColumnUInt8 * null_map = assert_cast<ColumnUInt8 *>(&nullable_col.getNullMapColumn());
+        null_map->insertDefault();
+        column = static_cast<ColumnFixedSizeHelper *>(&nullable_col.getNestedColumn());
+    }
+    else
+    {
+        column = static_cast<ColumnFixedSizeHelper *>(key_columns[0]);
+    }
+    static_assert(sizeof(FieldType) <= sizeof(Key));
+    const auto * key_holder = reinterpret_cast<const char *>(&key);
+    if constexpr (sizeof(FieldType) < sizeof(Key) && std::endian::native == std::endian::big)
+        column->insertRawData<sizeof(FieldType)>(key_holder + (sizeof(Key) - sizeof(FieldType)));
+    else
+        column->insertRawData<sizeof(FieldType)>(key_holder);
+}
+
+template struct AggregationMethodOneNumber<UInt8, AggregatedDataWithUInt8Key, false>;
+template struct AggregationMethodOneNumber<UInt16, AggregatedDataWithUInt16Key, false>;
+template struct AggregationMethodOneNumber<UInt32, AggregatedDataWithUInt64Key>;
+template struct AggregationMethodOneNumber<UInt64, AggregatedDataWithUInt64Key>;
+template struct AggregationMethodOneNumber<UInt32, AggregatedDataWithUInt64KeyTwoLevel>;
+template struct AggregationMethodOneNumber<UInt64, AggregatedDataWithUInt64KeyTwoLevel>;
+template struct AggregationMethodOneNumber<UInt64, AggregatedDataWithUInt64KeyHash64>;
+template struct AggregationMethodOneNumber<UInt8, AggregatedDataWithNullableUInt8Key, false, true>;
+template struct AggregationMethodOneNumber<UInt16, AggregatedDataWithNullableUInt16Key, false, true>;
+template struct AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt32Key, true, true>;
+template struct AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64Key, true, true>;
+template struct AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt32KeyTwoLevel, true, true>;
+template struct AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64KeyTwoLevel, true, true>;
+template struct AggregationMethodOneNumber<UInt8, AggregatedDataWithNullableUInt8Key, false>;
+template struct AggregationMethodOneNumber<UInt16, AggregatedDataWithNullableUInt16Key, false>;
+template struct AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt64Key>;
+template struct AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64Key>;
+template struct AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt64KeyTwoLevel>;
+template struct AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64KeyTwoLevel>;
+
+template <typename TData, bool nullable>
+void AggregationMethodStringNoCache<TData, nullable>::insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &)
+{
+    if constexpr (nullable)
+    {
+        ColumnNullable & column_nullable = assert_cast<ColumnNullable &>(*key_columns[0]);
+        assert_cast<ColumnString &>(column_nullable.getNestedColumn()).insertData(key.data, key.size);
+        column_nullable.getNullMapData().push_back(0);
+    }
+    else
+    {
+        assert_cast<ColumnString &>(*key_columns[0]).insertData(key.data, key.size);
+    }
+}
+template struct AggregationMethodStringNoCache<AggregatedDataWithShortStringKey>;
+template struct AggregationMethodStringNoCache<AggregatedDataWithShortStringKeyTwoLevel>;
+template struct AggregationMethodStringNoCache<AggregatedDataWithNullableShortStringKey, true>;
+template struct AggregationMethodStringNoCache<AggregatedDataWithNullableShortStringKeyTwoLevel, true>;
+
+template <typename TData>
+void AggregationMethodFixedString<TData>::insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &)
+{
+    assert_cast<ColumnFixedString &>(*key_columns[0]).insertData(key.data, key.size);
+}
+template struct AggregationMethodFixedString<AggregatedDataWithStringKeyHash64>;
+template struct AggregationMethodFixedString<AggregatedDataWithNullableStringKey>;
+template struct AggregationMethodFixedString<AggregatedDataWithNullableStringKeyTwoLevel>;
+
+
+template <typename TData, bool nullable>
+void AggregationMethodFixedStringNoCache<TData, nullable>::insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &)
+{
+    if constexpr (nullable)
+        assert_cast<ColumnNullable &>(*key_columns[0]).insertData(key.data, key.size);
+    else
+        assert_cast<ColumnFixedString &>(*key_columns[0]).insertData(key.data, key.size);
+}
+template struct AggregationMethodFixedStringNoCache<AggregatedDataWithShortStringKey>;
+template struct AggregationMethodFixedStringNoCache<AggregatedDataWithShortStringKeyTwoLevel>;
+template struct AggregationMethodFixedStringNoCache<AggregatedDataWithNullableShortStringKey, true>;
+template struct AggregationMethodFixedStringNoCache<AggregatedDataWithNullableShortStringKeyTwoLevel, true>;
+
+
+template <typename SingleColumnMethod>
+void AggregationMethodSingleLowCardinalityColumn<SingleColumnMethod>::insertKeyIntoColumns(
+    const Key & key, std::vector<IColumn *> & key_columns_low_cardinality, const Sizes & /*key_sizes*/)
+{
+    auto * col = assert_cast<ColumnLowCardinality *>(key_columns_low_cardinality[0]);
+
+    if constexpr (std::is_same_v<Key, StringRef>)
+        col->insertData(key.data, key.size);
+    else
+        col->insertData(reinterpret_cast<const char *>(&key), sizeof(key));
+}
+template struct AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt8, AggregatedDataWithNullableUInt8Key, false>>;
+template struct AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt16, AggregatedDataWithNullableUInt16Key, false>>;
+template struct AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt64Key>>;
+template struct AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64Key>>;
+template struct AggregationMethodSingleLowCardinalityColumn<AggregationMethodString<AggregatedDataWithNullableStringKey>>;
+template struct AggregationMethodSingleLowCardinalityColumn<AggregationMethodFixedString<AggregatedDataWithNullableStringKey>>;
+template struct AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt64KeyTwoLevel>>;
+template struct AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64KeyTwoLevel>>;
+template struct AggregationMethodSingleLowCardinalityColumn<AggregationMethodString<AggregatedDataWithNullableStringKeyTwoLevel>>;
+template struct AggregationMethodSingleLowCardinalityColumn<AggregationMethodFixedString<AggregatedDataWithNullableStringKeyTwoLevel>>;
+
+
+template <typename TData, bool has_nullable_keys, bool has_low_cardinality, bool consecutive_keys_optimization>
+void AggregationMethodKeysFixed<TData, has_nullable_keys, has_low_cardinality,consecutive_keys_optimization>::insertKeyIntoColumns(const Key & key, std::vector<IColumn *> & key_columns, const Sizes & key_sizes)
+{
+    size_t keys_size = key_columns.size();
+
+    static constexpr auto bitmap_size = has_nullable_keys ? std::tuple_size<KeysNullMap<Key>>::value : 0;
+    /// In any hash key value, column values to be read start just after the bitmap, if it exists.
+    size_t pos = bitmap_size;
+
+    for (size_t i = 0; i < keys_size; ++i)
+    {
+        IColumn * observed_column;
+        ColumnUInt8 * null_map;
+
+        bool column_nullable = false;
+        if constexpr (has_nullable_keys)
+            column_nullable = isColumnNullable(*key_columns[i]);
+
+        /// If we have a nullable column, get its nested column and its null map.
+        if (column_nullable)
+        {
+            ColumnNullable & nullable_col = assert_cast<ColumnNullable &>(*key_columns[i]);
+            observed_column = &nullable_col.getNestedColumn();
+            null_map = assert_cast<ColumnUInt8 *>(&nullable_col.getNullMapColumn());
+        }
+        else
+        {
+            observed_column = key_columns[i];
+            null_map = nullptr;
+        }
+
+        bool is_null = false;
+        if (column_nullable)
+        {
+            /// The current column is nullable. Check if the value of the
+            /// corresponding key is nullable. Update the null map accordingly.
+            size_t bucket = i / 8;
+            size_t offset = i % 8;
+            UInt8 val = (reinterpret_cast<const UInt8 *>(&key)[bucket] >> offset) & 1;
+            null_map->insertValue(val);
+            is_null = val == 1;
+        }
+
+        if (has_nullable_keys && is_null)
+            observed_column->insertDefault();
+        else
+        {
+            size_t size = key_sizes[i];
+            size_t offset_to = pos;
+            if constexpr (std::endian::native == std::endian::big)
+                offset_to = sizeof(Key) - size - pos;
+            observed_column->insertData(reinterpret_cast<const char *>(&key) + offset_to, size);
+            pos += size;
+        }
+    }
+}
+template struct AggregationMethodKeysFixed<AggregatedDataWithUInt16Key, false, false, false>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithUInt32Key>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithUInt64Key>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys128>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys256>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithUInt32KeyTwoLevel>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithUInt64KeyTwoLevel>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys128TwoLevel>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys256TwoLevel>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys128Hash64>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys256Hash64>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys128, true>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys256, true>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys128TwoLevel, true>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys256TwoLevel, true>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys128, false, true>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys256, false, true>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys128TwoLevel, false, true>;
+template struct AggregationMethodKeysFixed<AggregatedDataWithKeys256TwoLevel, false, true>;
+
+
+template <typename TData, bool nullable, bool prealloc>
+void AggregationMethodSerialized<TData, nullable, prealloc>::insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &)
+{
+    const auto * pos = key.data;
+    for (auto & column : key_columns)
+        pos = column->deserializeAndInsertFromArena(pos);
+}
+template struct AggregationMethodSerialized<AggregatedDataWithStringKey>;
+template struct AggregationMethodSerialized<AggregatedDataWithStringKeyTwoLevel>;
+template struct AggregationMethodSerialized<AggregatedDataWithStringKeyHash64>;
+// AggregationMethodNullableSerialized
+template struct AggregationMethodSerialized<AggregatedDataWithStringKey, true, false>;
+template struct AggregationMethodSerialized<AggregatedDataWithStringKeyTwoLevel, true, false>;
+template struct AggregationMethodSerialized<AggregatedDataWithStringKeyHash64, true, false>;
+// AggregationMethodPreallocSerialized
+template struct AggregationMethodSerialized<AggregatedDataWithStringKey, false, true>;
+template struct AggregationMethodSerialized<AggregatedDataWithStringKeyTwoLevel, false, true>;
+template struct AggregationMethodSerialized<AggregatedDataWithStringKeyHash64, false, true>;
+// AggregationMethodNullablePreallocSerialized
+template struct AggregationMethodSerialized<AggregatedDataWithStringKey, true, true>;
+template struct AggregationMethodSerialized<AggregatedDataWithStringKeyTwoLevel, true, true>;
+template struct AggregationMethodSerialized<AggregatedDataWithStringKeyHash64, true, true>;
+
+}
diff --git a/src/Interpreters/AggregationMethod.h b/src/Interpreters/AggregationMethod.h
new file mode 100644
index 00000000000..41345d91990
--- /dev/null
+++ b/src/Interpreters/AggregationMethod.h
@@ -0,0 +1,320 @@
+#pragma once
+#include <vector>
+#include <Common/ColumnsHashing.h>
+#include <Interpreters/AggregationCommon.h>
+#include <Interpreters/AggregatedData.h>
+
+#include <Columns/ColumnString.h>
+#include <Columns/ColumnFixedString.h>
+#include <Columns/ColumnAggregateFunction.h>
+#include <Columns/ColumnVector.h>
+#include <Columns/ColumnNullable.h>
+#include <Columns/ColumnLowCardinality.h>
+
+namespace DB
+{
+class IColumn;
+/// For the case where there is one numeric key.
+/// FieldType is UInt8/16/32/64 for any type with corresponding bit width.
+template <typename FieldType, typename TData,
+        bool consecutive_keys_optimization = true, bool nullable = false>
+struct AggregationMethodOneNumber
+{
+    using Data = TData;
+    using Key = typename Data::key_type;
+    using Mapped = typename Data::mapped_type;
+
+    Data data;
+
+    AggregationMethodOneNumber() = default;
+
+    explicit AggregationMethodOneNumber(size_t size_hint) : data(size_hint) { }
+
+    template <typename Other>
+    explicit AggregationMethodOneNumber(const Other & other) : data(other.data)
+    {
+    }
+
+    /// To use one `Method` in different threads, use different `State`.
+    template <bool use_cache>
+    using StateImpl = ColumnsHashing::HashMethodOneNumber<
+        typename Data::value_type,
+        Mapped,
+        FieldType,
+        use_cache && consecutive_keys_optimization,
+        /*need_offset=*/ false,
+        nullable>;
+
+    using State = StateImpl<true>;
+    using StateNoCache = StateImpl<false>;
+
+    /// Use optimization for low cardinality.
+    static const bool low_cardinality_optimization = false;
+    static const bool one_key_nullable_optimization = nullable;
+
+    /// Shuffle key columns before `insertKeyIntoColumns` call if needed.
+    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
+
+    // Insert the key from the hash table into columns.
+    static void insertKeyIntoColumns(const Key & key, std::vector<IColumn *> & key_columns, const Sizes & /*key_sizes*/);
+};
+
+/// For the case where there is one string key.
+template <typename TData>
+struct AggregationMethodString
+{
+    using Data = TData;
+    using Key = typename Data::key_type;
+    using Mapped = typename Data::mapped_type;
+
+    Data data;
+
+    AggregationMethodString() = default;
+
+    template <typename Other>
+    explicit AggregationMethodString(const Other & other) : data(other.data)
+    {
+    }
+
+    explicit AggregationMethodString(size_t size_hint) : data(size_hint) { }
+
+    template <bool use_cache>
+    using StateImpl = ColumnsHashing::HashMethodString<typename Data::value_type, Mapped, /*place_string_to_arena=*/ true, use_cache>;
+
+    using State = StateImpl<true>;
+    using StateNoCache = StateImpl<false>;
+
+    static const bool low_cardinality_optimization = false;
+    static const bool one_key_nullable_optimization = false;
+
+    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
+
+    static void insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &)
+    {
+        static_cast<ColumnString *>(key_columns[0])->insertData(key.data, key.size);
+    }
+};
+
+/// Same as above but without cache
+template <typename TData, bool nullable = false>
+struct AggregationMethodStringNoCache
+{
+    using Data = TData;
+    using Key = typename Data::key_type;
+    using Mapped = typename Data::mapped_type;
+
+    Data data;
+
+    AggregationMethodStringNoCache() = default;
+
+    explicit AggregationMethodStringNoCache(size_t size_hint) : data(size_hint) { }
+
+    template <typename Other>
+    explicit AggregationMethodStringNoCache(const Other & other) : data(other.data)
+    {
+    }
+
+    template <bool use_cache>
+    using StateImpl = ColumnsHashing::HashMethodString<typename Data::value_type, Mapped, true, false, false, nullable>;
+
+    using State = StateImpl<true>;
+    using StateNoCache = StateImpl<false>;
+
+    static const bool low_cardinality_optimization = false;
+    static const bool one_key_nullable_optimization = nullable;
+
+    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
+
+    static void insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &);
+};
+
+/// For the case where there is one fixed-length string key.
+template <typename TData>
+struct AggregationMethodFixedString
+{
+    using Data = TData;
+    using Key = typename Data::key_type;
+    using Mapped = typename Data::mapped_type;
+
+    Data data;
+
+    AggregationMethodFixedString() = default;
+
+    explicit AggregationMethodFixedString(size_t size_hint) : data(size_hint) { }
+
+    template <typename Other>
+    explicit AggregationMethodFixedString(const Other & other) : data(other.data)
+    {
+    }
+
+    template <bool use_cache>
+    using StateImpl = ColumnsHashing::HashMethodFixedString<typename Data::value_type, Mapped, /*place_string_to_arena=*/ true, use_cache>;
+
+    using State = StateImpl<true>;
+    using StateNoCache = StateImpl<false>;
+
+    static const bool low_cardinality_optimization = false;
+    static const bool one_key_nullable_optimization = false;
+
+    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
+
+    static void insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &);
+};
+
+/// Same as above but without cache
+template <typename TData, bool nullable = false>
+struct AggregationMethodFixedStringNoCache
+{
+    using Data = TData;
+    using Key = typename Data::key_type;
+    using Mapped = typename Data::mapped_type;
+
+    Data data;
+
+    AggregationMethodFixedStringNoCache() = default;
+
+    explicit AggregationMethodFixedStringNoCache(size_t size_hint) : data(size_hint) { }
+
+    template <typename Other>
+    explicit AggregationMethodFixedStringNoCache(const Other & other) : data(other.data)
+    {
+    }
+
+    template <bool use_cache>
+    using StateImpl = ColumnsHashing::HashMethodFixedString<typename Data::value_type, Mapped, true, false, false, nullable>;
+
+    using State = StateImpl<true>;
+    using StateNoCache = StateImpl<false>;
+
+    static const bool low_cardinality_optimization = false;
+    static const bool one_key_nullable_optimization = nullable;
+
+    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
+
+    static void insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &);
+};
+
+/// Single low cardinality column.
+template <typename SingleColumnMethod>
+struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod
+{
+    using Base = SingleColumnMethod;
+    using Data = typename Base::Data;
+    using Key = typename Base::Key;
+    using Mapped = typename Base::Mapped;
+    using Base::data;
+
+    template <bool use_cache>
+    using BaseStateImpl = typename Base::template StateImpl<use_cache>;
+
+    AggregationMethodSingleLowCardinalityColumn() = default;
+
+    template <typename Other>
+    explicit AggregationMethodSingleLowCardinalityColumn(const Other & other) : Base(other) {}
+
+    template <bool use_cache>
+    using StateImpl = ColumnsHashing::HashMethodSingleLowCardinalityColumn<BaseStateImpl<use_cache>, Mapped, use_cache>;
+
+    using State = StateImpl<true>;
+    using StateNoCache = StateImpl<false>;
+
+    static const bool low_cardinality_optimization = true;
+
+    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
+
+    static void insertKeyIntoColumns(const Key & key,
+         std::vector<IColumn *> & key_columns_low_cardinality, const Sizes & /*key_sizes*/);
+};
+
+/// For the case where all keys are of fixed length, and they fit in N (for example, 128) bits.
+template <typename TData, bool has_nullable_keys_ = false, bool has_low_cardinality_ = false, bool consecutive_keys_optimization = false>
+struct AggregationMethodKeysFixed
+{
+    using Data = TData;
+    using Key = typename Data::key_type;
+    using Mapped = typename Data::mapped_type;
+    static constexpr bool has_nullable_keys = has_nullable_keys_;
+    static constexpr bool has_low_cardinality = has_low_cardinality_;
+
+    Data data;
+
+    AggregationMethodKeysFixed() = default;
+
+    explicit AggregationMethodKeysFixed(size_t size_hint) : data(size_hint) { }
+
+    template <typename Other>
+    explicit AggregationMethodKeysFixed(const Other & other) : data(other.data)
+    {
+    }
+
+    template <bool use_cache>
+    using StateImpl = ColumnsHashing::HashMethodKeysFixed<
+        typename Data::value_type,
+        Key,
+        Mapped,
+        has_nullable_keys,
+        has_low_cardinality,
+        use_cache && consecutive_keys_optimization>;
+
+    using State = StateImpl<true>;
+    using StateNoCache = StateImpl<false>;
+
+    static const bool low_cardinality_optimization = false;
+    static const bool one_key_nullable_optimization = false;
+
+    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> & key_columns, const Sizes & key_sizes)
+    {
+        return State::shuffleKeyColumns(key_columns, key_sizes);
+    }
+
+    static void insertKeyIntoColumns(const Key & key, std::vector<IColumn *> & key_columns, const Sizes & key_sizes);
+};
+
+/** Aggregates by concatenating serialized key values.
+  * The serialized value differs in that it uniquely allows to deserialize it, having only the position with which it starts.
+  * That is, for example, for strings, it contains first the serialized length of the string, and then the bytes.
+  * Therefore, when aggregating by several strings, there is no ambiguity.
+  */
+template <typename TData, bool nullable = false, bool prealloc = false>
+struct AggregationMethodSerialized
+{
+    using Data = TData;
+    using Key = typename Data::key_type;
+    using Mapped = typename Data::mapped_type;
+
+    Data data;
+
+    AggregationMethodSerialized() = default;
+
+    explicit AggregationMethodSerialized(size_t size_hint) : data(size_hint) { }
+
+    template <typename Other>
+    explicit AggregationMethodSerialized(const Other & other) : data(other.data)
+    {
+    }
+
+    template <bool use_cache>
+    using StateImpl = ColumnsHashing::HashMethodSerialized<typename Data::value_type, Mapped, nullable, prealloc>;
+
+    using State = StateImpl<true>;
+    using StateNoCache = StateImpl<false>;
+
+    static const bool low_cardinality_optimization = false;
+    static const bool one_key_nullable_optimization = false;
+
+    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
+
+    static void insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &);
+};
+
+template <typename TData>
+using AggregationMethodNullableSerialized = AggregationMethodSerialized<TData, true>;
+
+template <typename TData>
+using AggregationMethodPreallocSerialized = AggregationMethodSerialized<TData, false, true>;
+
+template <typename TData>
+using AggregationMethodNullablePreallocSerialized = AggregationMethodSerialized<TData, true, true>;
+
+
+}
diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp
index 80a98683867..3596e2fcc4a 100644
--- a/src/Interpreters/Aggregator.cpp
+++ b/src/Interpreters/Aggregator.cpp
@@ -48,7 +48,6 @@ namespace ProfileEvents
     extern const Event ExternalAggregationUncompressedBytes;
     extern const Event ExternalProcessingCompressedBytesTotal;
     extern const Event ExternalProcessingUncompressedBytesTotal;
-    extern const Event AggregationPreallocatedElementsInHashTables;
     extern const Event AggregationHashTablesInitializedAsTwoLevel;
     extern const Event OverflowThrow;
     extern const Event OverflowBreak;
@@ -269,50 +268,6 @@ void updateStatistics(const DB::ManyAggregatedDataVariants & data_variants, cons
     getHashTablesStatistics().update(sum_of_sizes, *median_size, params);
 }
 
-// The std::is_constructible trait isn't suitable here because some classes have template constructors with semantics different from providing size hints.
-// Also string hash table variants are not supported due to the fact that both local perf tests and tests in CI showed slowdowns for them.
-template <typename...>
-struct HasConstructorOfNumberOfElements : std::false_type
-{
-};
-
-template <typename... Ts>
-struct HasConstructorOfNumberOfElements<HashMapTable<Ts...>> : std::true_type
-{
-};
-
-template <typename Key, typename Cell, typename Hash, typename Grower, typename Allocator, template <typename...> typename ImplTable>
-struct HasConstructorOfNumberOfElements<TwoLevelHashMapTable<Key, Cell, Hash, Grower, Allocator, ImplTable>> : std::true_type
-{
-};
-
-template <typename... Ts>
-struct HasConstructorOfNumberOfElements<HashTable<Ts...>> : std::true_type
-{
-};
-
-template <typename... Ts>
-struct HasConstructorOfNumberOfElements<TwoLevelHashTable<Ts...>> : std::true_type
-{
-};
-
-template <template <typename> typename Method, typename Base>
-struct HasConstructorOfNumberOfElements<Method<Base>> : HasConstructorOfNumberOfElements<Base>
-{
-};
-
-template <typename Method>
-auto constructWithReserveIfPossible(size_t size_hint)
-{
-    if constexpr (HasConstructorOfNumberOfElements<typename Method::Data>::value)
-    {
-        ProfileEvents::increment(ProfileEvents::AggregationPreallocatedElementsInHashTables, size_hint);
-        return std::make_unique<Method>(size_hint);
-    }
-    else
-        return std::make_unique<Method>();
-}
-
 DB::ColumnNumbers calculateKeysPositions(const DB::Block & header, const DB::Aggregator::Params & params)
 {
     DB::ColumnNumbers keys_positions(params.keys_size);
@@ -345,71 +300,11 @@ size_t getMinBytesForPrefetch()
 namespace DB
 {
 
-AggregatedDataVariants::~AggregatedDataVariants()
-{
-    if (aggregator && !aggregator->all_aggregates_has_trivial_destructor)
-    {
-        try
-        {
-            aggregator->destroyAllAggregateStates(*this);
-        }
-        catch (...)
-        {
-            tryLogCurrentException(__PRETTY_FUNCTION__);
-        }
-    }
-}
-
 std::optional<HashTablesCacheStatistics> getHashTablesCacheStatistics()
 {
     return getHashTablesStatistics().getCacheStats();
 }
 
-void AggregatedDataVariants::convertToTwoLevel()
-{
-    if (aggregator)
-        LOG_TRACE(aggregator->log, "Converting aggregation data to two-level.");
-
-    switch (type)
-    {
-#define M(NAME) \
-        case Type::NAME: \
-            NAME ## _two_level = std::make_unique<decltype(NAME ## _two_level)::element_type>(*(NAME)); \
-            (NAME).reset(); \
-            type = Type::NAME ## _two_level; \
-            break;
-
-        APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M)
-
-    #undef M
-
-        default:
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong data variant passed.");
-    }
-}
-
-void AggregatedDataVariants::init(Type type_, std::optional<size_t> size_hint)
-{
-    switch (type_)
-    {
-        case Type::EMPTY:
-        case Type::without_key:
-            break;
-
-#define M(NAME, IS_TWO_LEVEL) \
-    case Type::NAME: \
-        if (size_hint) \
-            (NAME) = constructWithReserveIfPossible<decltype(NAME)::element_type>(*size_hint); \
-        else \
-            (NAME) = std::make_unique<decltype(NAME)::element_type>(); \
-        break;
-            APPLY_FOR_AGGREGATED_VARIANTS(M)
-#undef M
-    }
-
-    type = type_;
-}
-
 Aggregator::Params::StatsCollectingParams::StatsCollectingParams() = default;
 
 Aggregator::Params::StatsCollectingParams::StatsCollectingParams(
@@ -1121,30 +1016,30 @@ void NO_INLINE Aggregator::executeImpl(
         if (compiled_aggregate_functions_holder && !hasSparseArguments(aggregate_instructions))
         {
             if (prefetch)
-                executeImplBatch<false, true, true>(
-                    method, state, aggregates_pool, row_begin, row_end, aggregate_instructions, all_keys_are_const, overflow_row);
+                executeImplBatch<true>(
+                    method, state, aggregates_pool, row_begin, row_end, aggregate_instructions, false, all_keys_are_const, true, overflow_row);
             else
-                executeImplBatch<false, true, false>(
-                    method, state, aggregates_pool, row_begin, row_end, aggregate_instructions, all_keys_are_const, overflow_row);
+                executeImplBatch<false>(
+                    method, state, aggregates_pool, row_begin, row_end, aggregate_instructions, false, all_keys_are_const, true, overflow_row);
         }
         else
 #endif
         {
             if (prefetch)
-                executeImplBatch<false, false, true>(
-                    method, state, aggregates_pool, row_begin, row_end, aggregate_instructions, all_keys_are_const, overflow_row);
+                executeImplBatch<true>(
+                    method, state, aggregates_pool, row_begin, row_end, aggregate_instructions, false, all_keys_are_const, false, overflow_row);
             else
-                executeImplBatch<false, false, false>(
-                    method, state, aggregates_pool, row_begin, row_end, aggregate_instructions, all_keys_are_const, overflow_row);
+                executeImplBatch<false>(
+                    method, state, aggregates_pool, row_begin, row_end, aggregate_instructions, false, all_keys_are_const, false, overflow_row);
         }
     }
     else
     {
-        executeImplBatch<true, false, false>(method, state, aggregates_pool, row_begin, row_end, aggregate_instructions, all_keys_are_const, overflow_row);
+        executeImplBatch<false>(method, state, aggregates_pool, row_begin, row_end, aggregate_instructions, true, all_keys_are_const, false, overflow_row);
     }
 }
 
-template <bool no_more_keys, bool use_compiled_functions, bool prefetch, typename Method, typename State>
+template <bool prefetch, typename Method, typename State>
 void NO_INLINE Aggregator::executeImplBatch(
     Method & method,
     State & state,
@@ -1152,7 +1047,9 @@ void NO_INLINE Aggregator::executeImplBatch(
     size_t row_begin,
     size_t row_end,
     AggregateFunctionInstruction * aggregate_instructions,
+    bool no_more_keys,
     bool all_keys_are_const,
+    bool use_compiled_functions,
     AggregateDataPtr overflow_row) const
 {
     using KeyHolder = decltype(state.getKeyHolder(0, std::declval<Arena &>()));
@@ -1164,7 +1061,7 @@ void NO_INLINE Aggregator::executeImplBatch(
     /// Optimization for special case when there are no aggregate functions.
     if (params.aggregates_size == 0)
     {
-        if constexpr (no_more_keys)
+        if (no_more_keys)
             return;
 
         /// This pointer is unused, but the logic will compare it for nullptr to check if the cell is set.
@@ -1197,39 +1094,42 @@ void NO_INLINE Aggregator::executeImplBatch(
     }
 
     /// Optimization for special case when aggregating by 8bit key.
-    if constexpr (!no_more_keys && std::is_same_v<Method, typename decltype(AggregatedDataVariants::key8)::element_type>)
+    if (!no_more_keys)
     {
-        /// We use another method if there are aggregate functions with -Array combinator.
-        bool has_arrays = false;
-        for (AggregateFunctionInstruction * inst = aggregate_instructions; inst->that; ++inst)
-        {
-            if (inst->offsets)
-            {
-                has_arrays = true;
-                break;
-            }
-        }
-
-        if (!has_arrays && !hasSparseArguments(aggregate_instructions) && !all_keys_are_const)
+        if constexpr (std::is_same_v<Method, typename decltype(AggregatedDataVariants::key8)::element_type>)
         {
+            /// We use another method if there are aggregate functions with -Array combinator.
+            bool has_arrays = false;
             for (AggregateFunctionInstruction * inst = aggregate_instructions; inst->that; ++inst)
             {
-                inst->batch_that->addBatchLookupTable8(
-                    row_begin,
-                    row_end,
-                    reinterpret_cast<AggregateDataPtr *>(method.data.data()),
-                    inst->state_offset,
-                    [&](AggregateDataPtr & aggregate_data)
-                    {
-                        AggregateDataPtr place = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states);
-                        createAggregateStates(place);
-                        aggregate_data = place;
-                    },
-                    state.getKeyData(),
-                    inst->batch_arguments,
-                    aggregates_pool);
+                if (inst->offsets)
+                {
+                    has_arrays = true;
+                    break;
+                }
+            }
+
+            if (!has_arrays && !hasSparseArguments(aggregate_instructions) && !all_keys_are_const)
+            {
+                for (AggregateFunctionInstruction * inst = aggregate_instructions; inst->that; ++inst)
+                {
+                    inst->batch_that->addBatchLookupTable8(
+                        row_begin,
+                        row_end,
+                        reinterpret_cast<AggregateDataPtr *>(method.data.data()),
+                        inst->state_offset,
+                        [&](AggregateDataPtr & aggregate_data)
+                        {
+                            AggregateDataPtr place = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states);
+                            createAggregateStates(place);
+                            aggregate_data = place;
+                        },
+                        state.getKeyData(),
+                        inst->batch_arguments,
+                        aggregates_pool);
+                }
+                return;
             }
-            return;
         }
     }
 
@@ -1255,12 +1155,12 @@ void NO_INLINE Aggregator::executeImplBatch(
     state.resetCache();
 
     /// For all rows.
-    for (size_t i = key_start; i < key_end; ++i)
+    if (!no_more_keys)
     {
-        AggregateDataPtr aggregate_data = nullptr;
-
-        if constexpr (!no_more_keys)
+        for (size_t i = key_start; i < key_end; ++i)
         {
+            AggregateDataPtr aggregate_data = nullptr;
+
             if constexpr (prefetch && HasPrefetchMemberFunc<decltype(method.data), KeyHolder>)
             {
                 if (i == key_start + prefetching.iterationsToMeasure())
@@ -1284,7 +1184,7 @@ void NO_INLINE Aggregator::executeImplBatch(
                 aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states);
 
 #if USE_EMBEDDED_COMPILER
-                if constexpr (use_compiled_functions)
+                if (use_compiled_functions)
                 {
                     const auto & compiled_aggregate_functions = compiled_aggregate_functions_holder->compiled_aggregate_functions;
                     compiled_aggregate_functions.create_aggregate_states_function(aggregate_data);
@@ -1297,7 +1197,8 @@ void NO_INLINE Aggregator::executeImplBatch(
 #if defined(MEMORY_SANITIZER)
 
                     /// We compile only functions that do not allocate some data in Arena. Only store necessary state in AggregateData place.
-                    for (size_t aggregate_function_index = 0; aggregate_function_index < aggregate_functions.size(); ++aggregate_function_index)
+                    for (size_t aggregate_function_index = 0; aggregate_function_index < aggregate_functions.size();
+                         ++aggregate_function_index)
                     {
                         if (!is_aggregate_function_compiled[aggregate_function_index])
                             continue;
@@ -1320,26 +1221,51 @@ void NO_INLINE Aggregator::executeImplBatch(
                 aggregate_data = emplace_result.getMapped();
 
             assert(aggregate_data != nullptr);
+            places[i] = aggregate_data;
         }
-        else
+    }
+    else
+    {
+        for (size_t i = key_start; i < key_end; ++i)
         {
+            AggregateDataPtr aggregate_data = nullptr;
             /// Add only if the key already exists.
             auto find_result = state.findKey(method.data, i, *aggregates_pool);
             if (find_result.isFound())
-            {
                 aggregate_data = find_result.getMapped();
-            }
             else
-            {
                 aggregate_data = overflow_row;
-            }
+            places[i] = aggregate_data;
         }
-
-        places[i] = aggregate_data;
     }
 
+    executeAggregateInstructions(
+        aggregates_pool,
+        row_begin,
+        row_end,
+        aggregate_instructions,
+        places,
+        key_start,
+        state.hasOnlyOneValueSinceLastReset(),
+        no_more_keys,
+        all_keys_are_const,
+        use_compiled_functions);
+}
+
+void Aggregator::executeAggregateInstructions(
+    Arena * aggregates_pool,
+    size_t row_begin,
+    size_t row_end,
+    AggregateFunctionInstruction * aggregate_instructions,
+    const std::unique_ptr<AggregateDataPtr[]> &places,
+    size_t key_start,
+    bool has_only_one_value_since_last_reset,
+    bool no_more_keys,
+    bool all_keys_are_const,
+    bool use_compiled_functions) const
+{
 #if USE_EMBEDDED_COMPILER
-    if constexpr (use_compiled_functions)
+    if (use_compiled_functions)
     {
         std::vector<ColumnData> columns_data;
 
@@ -1355,7 +1281,7 @@ void NO_INLINE Aggregator::executeImplBatch(
                 columns_data.emplace_back(getColumnData(inst->batch_arguments[argument_index]));
         }
 
-        if (all_keys_are_const || (!no_more_keys && state.hasOnlyOneValueSinceLastReset()))
+        if (all_keys_are_const || (!no_more_keys && has_only_one_value_since_last_reset))
         {
             auto add_into_aggregate_states_function_single_place = compiled_aggregate_functions_holder->compiled_aggregate_functions.add_into_aggregate_states_function_single_place;
             add_into_aggregate_states_function_single_place(row_begin, row_end, columns_data.data(), places[key_start]);
@@ -1372,33 +1298,34 @@ void NO_INLINE Aggregator::executeImplBatch(
     for (size_t i = 0; i < aggregate_functions.size(); ++i)
     {
 #if USE_EMBEDDED_COMPILER
-        if constexpr (use_compiled_functions)
+        if (use_compiled_functions)
             if (is_aggregate_function_compiled[i])
                 continue;
 #endif
 
         AggregateFunctionInstruction * inst = aggregate_instructions + i;
 
-        if (all_keys_are_const || (!no_more_keys && state.hasOnlyOneValueSinceLastReset()))
+        if (all_keys_are_const || (!no_more_keys && has_only_one_value_since_last_reset))
             addBatchSinglePlace(row_begin, row_end, inst, places[key_start] + inst->state_offset, aggregates_pool);
         else
             addBatch(row_begin, row_end, inst, places.get(), aggregates_pool);
     }
+
 }
 
 
-template <bool use_compiled_functions>
 void NO_INLINE Aggregator::executeWithoutKeyImpl(
     AggregatedDataWithoutKey & res,
     size_t row_begin, size_t row_end,
     AggregateFunctionInstruction * aggregate_instructions,
-    Arena * arena) const
+    Arena * arena,
+    bool use_compiled_functions) const
 {
     if (row_begin == row_end)
         return;
 
 #if USE_EMBEDDED_COMPILER
-    if constexpr (use_compiled_functions)
+    if (use_compiled_functions)
     {
         std::vector<ColumnData> columns_data;
 
@@ -1441,7 +1368,7 @@ void NO_INLINE Aggregator::executeWithoutKeyImpl(
         AggregateFunctionInstruction * inst = aggregate_instructions + i;
 
 #if USE_EMBEDDED_COMPILER
-        if constexpr (use_compiled_functions)
+        if (use_compiled_functions)
             if (is_aggregate_function_compiled[i])
                 continue;
 #endif
@@ -1707,12 +1634,12 @@ bool Aggregator::executeOnBlock(Columns columns,
 // #if USE_EMBEDDED_COMPILER
 //         if (compiled_aggregate_functions_holder)
 //         {
-//             executeWithoutKeyImpl<true>(result.without_key, row_begin, row_end, aggregate_functions_instructions.data(), result.aggregates_pool);
+//             executeWithoutKeyImpl(result.without_key, row_begin, row_end, aggregate_functions_instructions.data(), result.aggregates_pool,true);
 //         }
 //         else
 // #endif
         {
-            executeWithoutKeyImpl<false>(result.without_key, row_begin, row_end, aggregate_functions_instructions.data(), result.aggregates_pool);
+            executeWithoutKeyImpl(result.without_key, row_begin, row_end, aggregate_functions_instructions.data(), result.aggregates_pool, false);
         }
     }
     else
@@ -1832,8 +1759,14 @@ Block Aggregator::convertOneBucketToBlock(
 {
     // Used in ConvertingAggregatedToChunksSource -> ConvertingAggregatedToChunksTransform (expects single chunk for each bucket_id).
     constexpr bool return_single_block = true;
-    Block block = convertToBlockImpl<return_single_block>(
-        method, method.data.impls[bucket], arena, data_variants.aggregates_pools, final, method.data.impls[bucket].size());
+    Block block = std::get<Block>(convertToBlockImpl(
+        method,
+        method.data.impls[bucket],
+        arena,
+        data_variants.aggregates_pools,
+        final,
+        method.data.impls[bucket].size(),
+        return_single_block));
 
     block.info.bucket_num = static_cast<int>(bucket);
     return block;
@@ -1953,35 +1886,33 @@ bool Aggregator::checkLimits(size_t result_size, bool & no_more_keys) const
 }
 
 
-template <bool return_single_block, typename Method, typename Table>
-Aggregator::ConvertToBlockRes<return_single_block>
-Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final, size_t rows) const
+template <typename Method, typename Table>
+Aggregator::ConvertToBlockResVariant
+Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final,size_t rows, bool return_single_block) const
 {
     if (data.empty())
     {
         auto && out_cols = prepareOutputBlockColumns(params, aggregate_functions, getHeader(final), aggregates_pools, final, rows);
         return {finalizeBlock(params, getHeader(final), std::move(out_cols), final, rows)};
     }
-
-    ConvertToBlockRes<return_single_block> res;
-
+    ConvertToBlockResVariant res;
     if (final)
     {
 #if USE_EMBEDDED_COMPILER
         if (compiled_aggregate_functions_holder)
         {
             static constexpr bool use_compiled_functions = !Method::low_cardinality_optimization;
-            res = convertToBlockImplFinal<Method, use_compiled_functions, return_single_block>(method, data, arena, aggregates_pools, rows);
+            res = convertToBlockImplFinal<Method>(method, data, arena, aggregates_pools, use_compiled_functions, true);
         }
         else
 #endif
         {
-            res = convertToBlockImplFinal<Method, false, return_single_block>(method, data, arena, aggregates_pools, rows);
+            res = convertToBlockImplFinal<Method>(method, data, arena, aggregates_pools, false, return_single_block);
         }
     }
     else
     {
-        res = convertToBlockImplNotFinal<return_single_block>(method, data, aggregates_pools, rows);
+        res = convertToBlockImplNotFinal(method, data, aggregates_pools, rows, return_single_block);
     }
 
     /// In order to release memory early.
@@ -2146,19 +2077,27 @@ Block Aggregator::insertResultsIntoColumns(PaddedPODArray<AggregateDataPtr> & pl
     return finalizeBlock(params, getHeader(/* final */ true), std::move(out_cols), /* final */ true, places.size());
 }
 
-template <typename Method, bool use_compiled_functions, bool return_single_block, typename Table>
-Aggregator::ConvertToBlockRes<return_single_block> NO_INLINE
-Aggregator::convertToBlockImplFinal(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, size_t) const
+// template <typename Method, bool use_compiled_functions, bool return_single_block, typename Table>
+// Aggregator::ConvertToBlockRes<return_single_block> NO_INLINE
+// Aggregator::convertToBlockImplFinal(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, size_t) const
+template <typename Method, typename Table>
+Aggregator::ConvertToBlockResVariant Aggregator::convertToBlockImplFinal(
+    Method & method,
+    Table & data,
+    Arena * arena,
+    Arenas & aggregates_pools,
+    bool use_compiled_functions,
+    bool return_single_block) const
 {
     /// +1 for nullKeyData, if `data` doesn't have it - not a problem, just some memory for one excessive row will be preallocated
     const size_t max_block_size = (return_single_block ? data.size() : std::min(params.max_block_size, data.size())) + 1;
     const bool final = true;
-    ConvertToBlockRes<return_single_block> res;
 
     std::optional<OutputBlockColumns> out_cols;
     std::optional<Sizes> shuffled_key_sizes;
     PaddedPODArray<AggregateDataPtr> places;
     bool has_null_key_data = false;
+    BlocksList blocks;
 
     auto init_out_cols = [&]()
     {
@@ -2187,51 +2126,78 @@ Aggregator::convertToBlockImplFinal(Method & method, Table & data, Arena * arena
     // should be invoked at least once, because null data might be the only content of the `data`
     init_out_cols();
 
-    data.forEachValue(
-        [&](const auto & key, auto & mapped)
-        {
-            if (!out_cols.has_value())
-                init_out_cols();
-
-            const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes;
-            method.insertKeyIntoColumns(key, out_cols->raw_key_columns, key_sizes_ref);
-            places.emplace_back(mapped);
-
-            /// Mark the cell as destroyed so it will not be destroyed in destructor.
-            mapped = nullptr;
-
-            if constexpr (!return_single_block)
+    if (return_single_block)
+    {
+        data.forEachValue(
+            [&](const auto & key, auto & mapped)
             {
+                if (!out_cols.has_value())
+                    init_out_cols();
+
+                const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes;
+                method.insertKeyIntoColumns(key, out_cols->raw_key_columns, key_sizes_ref);
+                places.emplace_back(mapped);
+
+                /// Mark the cell as destroyed so it will not be destroyed in destructor.
+                mapped = nullptr;
+            });
+    }
+    else
+    {
+        data.forEachValue(
+            [&](const auto & key, auto & mapped)
+            {
+                if (!out_cols.has_value())
+                    init_out_cols();
+
+                const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes;
+                method.insertKeyIntoColumns(key, out_cols->raw_key_columns, key_sizes_ref);
+                places.emplace_back(mapped);
+
+                /// Mark the cell as destroyed so it will not be destroyed in destructor.
+                mapped = nullptr;
+
                 if (places.size() >= max_block_size)
                 {
-                    res.emplace_back(insertResultsIntoColumns<use_compiled_functions>(places, std::move(out_cols.value()), arena, has_null_key_data));
+                    if (use_compiled_functions)
+                        blocks.emplace_back(insertResultsIntoColumns<true>(places, std::move(out_cols.value()), arena, has_null_key_data));
+                    else
+                        blocks.emplace_back(insertResultsIntoColumns<false>(places, std::move(out_cols.value()), arena, has_null_key_data));
                     places.clear();
                     out_cols.reset();
                     has_null_key_data = false;
                 }
-            }
-        });
+            });
+    }
 
-    if constexpr (return_single_block)
+    if (return_single_block)
     {
-        return insertResultsIntoColumns<use_compiled_functions>(places, std::move(out_cols.value()), arena, has_null_key_data);
+        if (use_compiled_functions)
+            return insertResultsIntoColumns<true>(places, std::move(out_cols.value()), arena, has_null_key_data);
+        else
+            return insertResultsIntoColumns<false>(places, std::move(out_cols.value()), arena, has_null_key_data);
     }
     else
     {
         if (out_cols.has_value())
-            res.emplace_back(insertResultsIntoColumns<use_compiled_functions>(places, std::move(out_cols.value()), arena, has_null_key_data));
-        return res;
+        {
+            if (use_compiled_functions)
+                blocks.emplace_back(insertResultsIntoColumns<true>(places, std::move(out_cols.value()), arena, has_null_key_data));
+            else
+                blocks.emplace_back(insertResultsIntoColumns<false>(places, std::move(out_cols.value()), arena, has_null_key_data));
+        }
+        return blocks;
     }
 }
 
-template <bool return_single_block, typename Method, typename Table>
-Aggregator::ConvertToBlockRes<return_single_block> NO_INLINE
-Aggregator::convertToBlockImplNotFinal(Method & method, Table & data, Arenas & aggregates_pools, size_t) const
+template <typename Method, typename Table>
+Aggregator::ConvertToBlockResVariant NO_INLINE
+Aggregator::convertToBlockImplNotFinal(Method & method, Table & data, Arenas & aggregates_pools, size_t, bool return_single_block) const
 {
     /// +1 for nullKeyData, if `data` doesn't have it - not a problem, just some memory for one excessive row will be preallocated
     const size_t max_block_size = (return_single_block ? data.size() : std::min(params.max_block_size, data.size())) + 1;
     const bool final = false;
-    ConvertToBlockRes<return_single_block> res;
+    BlocksList res_blocks;
 
     std::optional<OutputBlockColumns> out_cols;
     std::optional<Sizes> shuffled_key_sizes;
@@ -2261,46 +2227,65 @@ Aggregator::convertToBlockImplNotFinal(Method & method, Table & data, Arenas & a
 
     // should be invoked at least once, because null data might be the only content of the `data`
     init_out_cols();
-
-    data.forEachValue(
-        [&](const auto & key, auto & mapped)
-        {
-            if (!out_cols.has_value())
-                init_out_cols();
-
-            const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes;
-            method.insertKeyIntoColumns(key, out_cols->raw_key_columns, key_sizes_ref);
-
-            /// reserved, so push_back does not throw exceptions
-            for (size_t i = 0; i < params.aggregates_size; ++i)
-                out_cols->aggregate_columns_data[i]->push_back(mapped + offsets_of_aggregate_states[i]);
-
-            mapped = nullptr;
-
-            ++rows_in_current_block;
-
-            if constexpr (!return_single_block)
+    if (return_single_block)
+    {
+        data.forEachValue(
+            [&](const auto & key, auto & mapped)
             {
+                if (!out_cols.has_value())
+                    init_out_cols();
+
+                const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes;
+                method.insertKeyIntoColumns(key, out_cols->raw_key_columns, key_sizes_ref);
+
+                /// reserved, so push_back does not throw exceptions
+                for (size_t i = 0; i < params.aggregates_size; ++i)
+                    out_cols->aggregate_columns_data[i]->push_back(mapped + offsets_of_aggregate_states[i]);
+
+                mapped = nullptr;
+
+                ++rows_in_current_block;
+            });
+    }
+    else
+    {
+        data.forEachValue(
+            [&](const auto & key, auto & mapped)
+            {
+                if (!out_cols.has_value())
+                    init_out_cols();
+
+                const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes;
+                method.insertKeyIntoColumns(key, out_cols->raw_key_columns, key_sizes_ref);
+
+                /// reserved, so push_back does not throw exceptions
+                for (size_t i = 0; i < params.aggregates_size; ++i)
+                    out_cols->aggregate_columns_data[i]->push_back(mapped + offsets_of_aggregate_states[i]);
+
+                mapped = nullptr;
+
+                ++rows_in_current_block;
+
                 if (rows_in_current_block >= max_block_size)
                 {
-                    res.emplace_back(finalizeBlock(params, getHeader(final), std::move(out_cols.value()), final, rows_in_current_block));
+                    res_blocks.emplace_back(finalizeBlock(params, getHeader(final), std::move(out_cols.value()), final, rows_in_current_block));
                     out_cols.reset();
                     rows_in_current_block = 0;
                 }
-            }
-        });
+            });
+    }
 
-    if constexpr (return_single_block)
+    if (return_single_block)
     {
         return finalizeBlock(params, getHeader(final), std::move(out_cols).value(), final, rows_in_current_block);
     }
     else
     {
         if (rows_in_current_block)
-            res.emplace_back(finalizeBlock(params, getHeader(final), std::move(out_cols).value(), final, rows_in_current_block));
-        return res;
+            res_blocks.emplace_back(finalizeBlock(params, getHeader(final), std::move(out_cols).value(), final, rows_in_current_block));
+        return res_blocks;
     }
-    return res;
+    return res_blocks;
 }
 
 void Aggregator::addSingleKeyToAggregateColumns(
@@ -2406,18 +2391,23 @@ template <bool return_single_block>
 Aggregator::ConvertToBlockRes<return_single_block>
 Aggregator::prepareBlockAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final) const
 {
+    ConvertToBlockResVariant res_variant;
     const size_t rows = data_variants.sizeWithoutOverflowRow();
 #define M(NAME) \
     else if (data_variants.type == AggregatedDataVariants::Type::NAME) \
     { \
-        return convertToBlockImpl<return_single_block>( \
-            *data_variants.NAME, data_variants.NAME->data, data_variants.aggregates_pool, data_variants.aggregates_pools, final, rows); \
+        res_variant = convertToBlockImpl( \
+            *data_variants.NAME, data_variants.NAME->data, data_variants.aggregates_pool, data_variants.aggregates_pools, final, rows, return_single_block); \
     }
 
     if (false) {} // NOLINT
     APPLY_FOR_VARIANTS_SINGLE_LEVEL(M)
 #undef M
     else throw Exception(ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT, "Unknown aggregated data variant.");
+    if constexpr (return_single_block)
+        return std::get<Block>(res_variant);
+    else
+        return std::get<BlocksList>(res_variant);
 }
 
 
@@ -2543,7 +2533,7 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b
     if (data_variants.type != AggregatedDataVariants::Type::without_key)
     {
         if (!data_variants.isTwoLevel())
-            blocks.splice(blocks.end(), prepareBlockAndFillSingleLevel</* return_single_block */ false>(data_variants, final));
+            blocks.splice(blocks.end(), prepareBlockAndFillSingleLevel<false>(data_variants, final));
         else
             blocks.splice(blocks.end(), prepareBlocksAndFillTwoLevel(data_variants, final, thread_pool.get()));
     }
@@ -2609,8 +2599,8 @@ void NO_INLINE Aggregator::mergeDataNullKey(
     }
 }
 
-template <typename Method, bool use_compiled_functions, bool prefetch, typename Table>
-void NO_INLINE Aggregator::mergeDataImpl(Table & table_dst, Table & table_src, Arena * arena) const
+template <typename Method, typename Table>
+void NO_INLINE Aggregator::mergeDataImpl(Table & table_dst, Table & table_src, Arena * arena, bool use_compiled_functions, bool prefetch) const
 {
     if constexpr (Method::low_cardinality_optimization || Method::one_key_nullable_optimization)
         mergeDataNullKey<Method, Table>(table_dst, table_src, arena);
@@ -2633,11 +2623,14 @@ void NO_INLINE Aggregator::mergeDataImpl(Table & table_dst, Table & table_src, A
         src = nullptr;
     };
 
-    table_src.template mergeToViaEmplace<decltype(merge), prefetch>(table_dst, std::move(merge));
+    if (prefetch)
+        table_src.template mergeToViaEmplace<decltype(merge), true>(table_dst, std::move(merge));
+    else
+        table_src.template mergeToViaEmplace<decltype(merge), false>(table_dst, std::move(merge));
     table_src.clearAndShrink();
 
 #if USE_EMBEDDED_COMPILER
-    if constexpr (use_compiled_functions)
+    if (use_compiled_functions)
     {
         const auto & compiled_functions = compiled_aggregate_functions_holder->compiled_aggregate_functions;
         compiled_functions.merge_aggregate_states_function(dst_places.data(), src_places.data(), dst_places.size());
@@ -2790,22 +2783,14 @@ void NO_INLINE Aggregator::mergeSingleLevelDataImpl(
 #if USE_EMBEDDED_COMPILER
             if (compiled_aggregate_functions_holder)
             {
-                if (prefetch)
-                    mergeDataImpl<Method, true, true>(
-                        getDataVariant<Method>(*res).data, getDataVariant<Method>(current).data, res->aggregates_pool);
-                else
-                    mergeDataImpl<Method, true, false>(
-                        getDataVariant<Method>(*res).data, getDataVariant<Method>(current).data, res->aggregates_pool);
+                mergeDataImpl<Method>(
+                    getDataVariant<Method>(*res).data, getDataVariant<Method>(current).data, res->aggregates_pool, true, prefetch);
             }
             else
 #endif
             {
-                if (prefetch)
-                    mergeDataImpl<Method, false, true>(
-                        getDataVariant<Method>(*res).data, getDataVariant<Method>(current).data, res->aggregates_pool);
-                else
-                    mergeDataImpl<Method, false, false>(
-                        getDataVariant<Method>(*res).data, getDataVariant<Method>(current).data, res->aggregates_pool);
+                mergeDataImpl<Method>(
+                    getDataVariant<Method>(*res).data, getDataVariant<Method>(current).data, res->aggregates_pool, false, prefetch);
             }
         }
         else if (res->without_key)
@@ -2854,22 +2839,18 @@ void NO_INLINE Aggregator::mergeBucketImpl(
 #if USE_EMBEDDED_COMPILER
         if (compiled_aggregate_functions_holder)
         {
-            if (prefetch)
-                mergeDataImpl<Method, true, true>(
-                    getDataVariant<Method>(*res).data.impls[bucket], getDataVariant<Method>(current).data.impls[bucket], arena);
-            else
-                mergeDataImpl<Method, true, false>(
-                    getDataVariant<Method>(*res).data.impls[bucket], getDataVariant<Method>(current).data.impls[bucket], arena);
+            mergeDataImpl<Method>(
+                getDataVariant<Method>(*res).data.impls[bucket], getDataVariant<Method>(current).data.impls[bucket], arena, true, prefetch);
         }
         else
 #endif
         {
-            if (prefetch)
-                mergeDataImpl<Method, false, true>(
-                    getDataVariant<Method>(*res).data.impls[bucket], getDataVariant<Method>(current).data.impls[bucket], arena);
-            else
-                mergeDataImpl<Method, false, false>(
-                    getDataVariant<Method>(*res).data.impls[bucket], getDataVariant<Method>(current).data.impls[bucket], arena);
+            mergeDataImpl<Method>(
+                getDataVariant<Method>(*res).data.impls[bucket],
+                getDataVariant<Method>(current).data.impls[bucket],
+                arena,
+                false,
+                prefetch);
         }
     }
 }
@@ -3589,7 +3570,4 @@ void Aggregator::destroyAllAggregateStates(AggregatedDataVariants & result) cons
         throw Exception(ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT, "Unknown aggregated data variant.");
 }
 
-
-template Aggregator::ConvertToBlockRes<false>
-Aggregator::prepareBlockAndFillSingleLevel<false>(AggregatedDataVariants & data_variants, bool final) const;
 }
diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h
index 375b8986101..1d887d2a2c3 100644
--- a/src/Interpreters/Aggregator.h
+++ b/src/Interpreters/Aggregator.h
@@ -4,6 +4,7 @@
 #include <memory>
 #include <mutex>
 #include <type_traits>
+#include <variant>
 
 
 #include <base/StringRef.h>
@@ -38,998 +39,21 @@
 
 #include <Parsers/IAST_fwd.h>
 
+#include <Interpreters/AggregationMethod.h>
+#include <Interpreters/AggregatedData.h>
+#include <Interpreters/AggregatedDataVariants.h>
+
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int UNKNOWN_AGGREGATED_DATA_VARIANT;
-}
-
 class Arena;
 using ArenaPtr = std::shared_ptr<Arena>;
 using Arenas = std::vector<ArenaPtr>;
 
-/** Different data structures that can be used for aggregation
-  * For efficiency, the aggregation data itself is put into the pool.
-  * Data and pool ownership (states of aggregate functions)
-  *  is acquired later - in `convertToBlocks` function, by the ColumnAggregateFunction object.
-  *
-  * Most data structures exist in two versions: normal and two-level (TwoLevel).
-  * A two-level hash table works a little slower with a small number of different keys,
-  *  but with a large number of different keys scales better, because it allows
-  *  parallelize some operations (merging, post-processing) in a natural way.
-  *
-  * To ensure efficient work over a wide range of conditions,
-  *  first single-level hash tables are used,
-  *  and when the number of different keys is large enough,
-  *  they are converted to two-level ones.
-  *
-  * PS. There are many different approaches to the effective implementation of parallel and distributed aggregation,
-  *  best suited for different cases, and this approach is just one of them, chosen for a combination of reasons.
-  */
-
-using AggregatedDataWithoutKey = AggregateDataPtr;
-
-using AggregatedDataWithUInt8Key = FixedImplicitZeroHashMapWithCalculatedSize<UInt8, AggregateDataPtr>;
-using AggregatedDataWithUInt16Key = FixedImplicitZeroHashMap<UInt16, AggregateDataPtr>;
-
-using AggregatedDataWithUInt32Key = HashMap<UInt32, AggregateDataPtr, HashCRC32<UInt32>>;
-using AggregatedDataWithUInt64Key = HashMap<UInt64, AggregateDataPtr, HashCRC32<UInt64>>;
-
-using AggregatedDataWithShortStringKey = StringHashMap<AggregateDataPtr>;
-
-using AggregatedDataWithStringKey = HashMapWithSavedHash<StringRef, AggregateDataPtr>;
-
-using AggregatedDataWithKeys128 = HashMap<UInt128, AggregateDataPtr, UInt128HashCRC32>;
-using AggregatedDataWithKeys256 = HashMap<UInt256, AggregateDataPtr, UInt256HashCRC32>;
-
-using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap<UInt32, AggregateDataPtr, HashCRC32<UInt32>>;
-using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap<UInt64, AggregateDataPtr, HashCRC32<UInt64>>;
-
-using AggregatedDataWithShortStringKeyTwoLevel = TwoLevelStringHashMap<AggregateDataPtr>;
-
-using AggregatedDataWithStringKeyTwoLevel = TwoLevelHashMapWithSavedHash<StringRef, AggregateDataPtr>;
-
-using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap<UInt128, AggregateDataPtr, UInt128HashCRC32>;
-using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap<UInt256, AggregateDataPtr, UInt256HashCRC32>;
-
-/** Variants with better hash function, using more than 32 bits for hash.
-  * Using for merging phase of external aggregation, where number of keys may be far greater than 4 billion,
-  *  but we keep in memory and merge only sub-partition of them simultaneously.
-  * TODO We need to switch for better hash function not only for external aggregation,
-  *  but also for huge aggregation results on machines with terabytes of RAM.
-  */
-
-using AggregatedDataWithUInt64KeyHash64 = HashMap<UInt64, AggregateDataPtr, DefaultHash<UInt64>>;
-using AggregatedDataWithStringKeyHash64 = HashMapWithSavedHash<StringRef, AggregateDataPtr, StringRefHash64>;
-using AggregatedDataWithKeys128Hash64 = HashMap<UInt128, AggregateDataPtr, UInt128Hash>;
-using AggregatedDataWithKeys256Hash64 = HashMap<UInt256, AggregateDataPtr, UInt256Hash>;
-
-template <typename Base>
-struct AggregationDataWithNullKey : public Base
-{
-    using Base::Base;
-
-    bool & hasNullKeyData() { return has_null_key_data; }
-    AggregateDataPtr & getNullKeyData() { return null_key_data; }
-    bool hasNullKeyData() const { return has_null_key_data; }
-    const AggregateDataPtr & getNullKeyData() const { return null_key_data; }
-    size_t size() const { return Base::size() + (has_null_key_data ? 1 : 0); }
-    bool empty() const { return Base::empty() && !has_null_key_data; }
-    void clear()
-    {
-        Base::clear();
-        has_null_key_data = false;
-    }
-    void clearAndShrink()
-    {
-        Base::clearAndShrink();
-        has_null_key_data = false;
-    }
-
-private:
-    bool has_null_key_data = false;
-    AggregateDataPtr null_key_data = nullptr;
-};
-
-template <typename Base>
-struct AggregationDataWithNullKeyTwoLevel : public Base
-{
-    using Base::Base;
-    using Base::impls;
-
-    AggregationDataWithNullKeyTwoLevel() = default;
-
-    template <typename Other>
-    explicit AggregationDataWithNullKeyTwoLevel(const Other & other) : Base(other)
-    {
-        impls[0].hasNullKeyData() = other.hasNullKeyData();
-        impls[0].getNullKeyData() = other.getNullKeyData();
-    }
-
-    bool & hasNullKeyData() { return impls[0].hasNullKeyData(); }
-    AggregateDataPtr & getNullKeyData() { return impls[0].getNullKeyData(); }
-    bool hasNullKeyData() const { return impls[0].hasNullKeyData(); }
-    const AggregateDataPtr & getNullKeyData() const { return impls[0].getNullKeyData(); }
-};
-
-template <typename ... Types>
-using HashTableWithNullKey = AggregationDataWithNullKey<HashMapTable<Types ...>>;
-template <typename ... Types>
-using StringHashTableWithNullKey = AggregationDataWithNullKey<StringHashMap<Types ...>>;
-
-using AggregatedDataWithNullableUInt8Key = AggregationDataWithNullKey<AggregatedDataWithUInt8Key>;
-using AggregatedDataWithNullableUInt16Key = AggregationDataWithNullKey<AggregatedDataWithUInt16Key>;
-using AggregatedDataWithNullableUInt32Key = AggregationDataWithNullKey<AggregatedDataWithUInt32Key>;
-
-
-using AggregatedDataWithNullableUInt64Key = AggregationDataWithNullKey<AggregatedDataWithUInt64Key>;
-using AggregatedDataWithNullableStringKey = AggregationDataWithNullKey<AggregatedDataWithStringKey>;
-using AggregatedDataWithNullableShortStringKey = AggregationDataWithNullKey<AggregatedDataWithShortStringKey>;
-
-
-using AggregatedDataWithNullableUInt32KeyTwoLevel = AggregationDataWithNullKeyTwoLevel<
-    TwoLevelHashMap<UInt32, AggregateDataPtr, HashCRC32<UInt32>,
-                    TwoLevelHashTableGrower<>, HashTableAllocator, HashTableWithNullKey>>;
-using AggregatedDataWithNullableUInt64KeyTwoLevel = AggregationDataWithNullKeyTwoLevel<
-        TwoLevelHashMap<UInt64, AggregateDataPtr, HashCRC32<UInt64>,
-        TwoLevelHashTableGrower<>, HashTableAllocator, HashTableWithNullKey>>;
-
-using AggregatedDataWithNullableShortStringKeyTwoLevel = AggregationDataWithNullKeyTwoLevel<
-        TwoLevelStringHashMap<AggregateDataPtr, HashTableAllocator, StringHashTableWithNullKey>>;
-
-using AggregatedDataWithNullableStringKeyTwoLevel = AggregationDataWithNullKeyTwoLevel<
-        TwoLevelHashMapWithSavedHash<StringRef, AggregateDataPtr, DefaultHash<StringRef>,
-        TwoLevelHashTableGrower<>, HashTableAllocator, HashTableWithNullKey>>;
-
-/// For the case where there is one numeric key.
-/// FieldType is UInt8/16/32/64 for any type with corresponding bit width.
-template <typename FieldType, typename TData,
-        bool consecutive_keys_optimization = true, bool nullable = false>
-struct AggregationMethodOneNumber
-{
-    using Data = TData;
-    using Key = typename Data::key_type;
-    using Mapped = typename Data::mapped_type;
-
-    Data data;
-
-    AggregationMethodOneNumber() = default;
-
-    explicit AggregationMethodOneNumber(size_t size_hint) : data(size_hint) { }
-
-    template <typename Other>
-    explicit AggregationMethodOneNumber(const Other & other) : data(other.data)
-    {
-    }
-
-    /// To use one `Method` in different threads, use different `State`.
-    template <bool use_cache>
-    using StateImpl = ColumnsHashing::HashMethodOneNumber<
-        typename Data::value_type,
-        Mapped,
-        FieldType,
-        use_cache && consecutive_keys_optimization,
-        /*need_offset=*/ false,
-        nullable>;
-
-    using State = StateImpl<true>;
-    using StateNoCache = StateImpl<false>;
-
-    /// Use optimization for low cardinality.
-    static const bool low_cardinality_optimization = false;
-    static const bool one_key_nullable_optimization = nullable;
-
-    /// Shuffle key columns before `insertKeyIntoColumns` call if needed.
-    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
-
-    // Insert the key from the hash table into columns.
-    static void insertKeyIntoColumns(const Key & key, std::vector<IColumn *> & key_columns, const Sizes & /*key_sizes*/)
-    {
-        ColumnFixedSizeHelper * column;
-        if constexpr (nullable)
-        {
-            ColumnNullable & nullable_col = assert_cast<ColumnNullable &>(*key_columns[0]);
-            ColumnUInt8 * null_map = assert_cast<ColumnUInt8 *>(&nullable_col.getNullMapColumn());
-            null_map->insertDefault();
-            column = static_cast<ColumnFixedSizeHelper *>(&nullable_col.getNestedColumn());
-        }
-        else
-        {
-            column = static_cast<ColumnFixedSizeHelper *>(key_columns[0]);
-        }
-        static_assert(sizeof(FieldType) <= sizeof(Key));
-        const auto * key_holder = reinterpret_cast<const char *>(&key);
-        if constexpr (sizeof(FieldType) < sizeof(Key) && std::endian::native == std::endian::big)
-            column->insertRawData<sizeof(FieldType)>(key_holder + (sizeof(Key) - sizeof(FieldType)));
-        else
-            column->insertRawData<sizeof(FieldType)>(key_holder);
-    }
-};
-
-
-/// For the case where there is one string key.
-template <typename TData>
-struct AggregationMethodString
-{
-    using Data = TData;
-    using Key = typename Data::key_type;
-    using Mapped = typename Data::mapped_type;
-
-    Data data;
-
-    AggregationMethodString() = default;
-
-    template <typename Other>
-    explicit AggregationMethodString(const Other & other) : data(other.data)
-    {
-    }
-
-    explicit AggregationMethodString(size_t size_hint) : data(size_hint) { }
-
-    template <bool use_cache>
-    using StateImpl = ColumnsHashing::HashMethodString<typename Data::value_type, Mapped, /*place_string_to_arena=*/ true, use_cache>;
-
-    using State = StateImpl<true>;
-    using StateNoCache = StateImpl<false>;
-
-    static const bool low_cardinality_optimization = false;
-    static const bool one_key_nullable_optimization = false;
-
-    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
-
-    static void insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &)
-    {
-        static_cast<ColumnString *>(key_columns[0])->insertData(key.data, key.size);
-    }
-};
-
-
-/// Same as above but without cache
-template <typename TData, bool nullable = false>
-struct AggregationMethodStringNoCache
-{
-    using Data = TData;
-    using Key = typename Data::key_type;
-    using Mapped = typename Data::mapped_type;
-
-    Data data;
-
-    AggregationMethodStringNoCache() = default;
-
-    explicit AggregationMethodStringNoCache(size_t size_hint) : data(size_hint) { }
-
-    template <typename Other>
-    explicit AggregationMethodStringNoCache(const Other & other) : data(other.data)
-    {
-    }
-
-    template <bool use_cache>
-    using StateImpl = ColumnsHashing::HashMethodString<typename Data::value_type, Mapped, true, false, false, nullable>;
-
-    using State = StateImpl<true>;
-    using StateNoCache = StateImpl<false>;
-
-    static const bool low_cardinality_optimization = false;
-    static const bool one_key_nullable_optimization = nullable;
-
-    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
-
-    static void insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &)
-    {
-        if constexpr (nullable)
-        {
-            ColumnNullable & column_nullable = assert_cast<ColumnNullable &>(*key_columns[0]);
-            assert_cast<ColumnString &>(column_nullable.getNestedColumn()).insertData(key.data, key.size);
-            column_nullable.getNullMapData().push_back(0);
-        }
-        else
-        {
-            assert_cast<ColumnString &>(*key_columns[0]).insertData(key.data, key.size);
-        }
-    }
-};
-
-
-/// For the case where there is one fixed-length string key.
-template <typename TData>
-struct AggregationMethodFixedString
-{
-    using Data = TData;
-    using Key = typename Data::key_type;
-    using Mapped = typename Data::mapped_type;
-
-    Data data;
-
-    AggregationMethodFixedString() = default;
-
-    explicit AggregationMethodFixedString(size_t size_hint) : data(size_hint) { }
-
-    template <typename Other>
-    explicit AggregationMethodFixedString(const Other & other) : data(other.data)
-    {
-    }
-
-    template <bool use_cache>
-    using StateImpl = ColumnsHashing::HashMethodFixedString<typename Data::value_type, Mapped, /*place_string_to_arena=*/ true, use_cache>;
-
-    using State = StateImpl<true>;
-    using StateNoCache = StateImpl<false>;
-
-    static const bool low_cardinality_optimization = false;
-    static const bool one_key_nullable_optimization = false;
-
-    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
-
-    static void insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &)
-    {
-        assert_cast<ColumnFixedString &>(*key_columns[0]).insertData(key.data, key.size);
-    }
-};
-
-/// Same as above but without cache
-template <typename TData, bool nullable = false>
-struct AggregationMethodFixedStringNoCache
-{
-    using Data = TData;
-    using Key = typename Data::key_type;
-    using Mapped = typename Data::mapped_type;
-
-    Data data;
-
-    AggregationMethodFixedStringNoCache() = default;
-
-    explicit AggregationMethodFixedStringNoCache(size_t size_hint) : data(size_hint) { }
-
-    template <typename Other>
-    explicit AggregationMethodFixedStringNoCache(const Other & other) : data(other.data)
-    {
-    }
-
-    template <bool use_cache>
-    using StateImpl = ColumnsHashing::HashMethodFixedString<typename Data::value_type, Mapped, true, false, false, nullable>;
-
-    using State = StateImpl<true>;
-    using StateNoCache = StateImpl<false>;
-
-    static const bool low_cardinality_optimization = false;
-    static const bool one_key_nullable_optimization = nullable;
-
-    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
-
-    static void insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &)
-    {
-        if constexpr (nullable)
-        {
-            assert_cast<ColumnNullable &>(*key_columns[0]).insertData(key.data, key.size);
-        }
-        else
-        {
-            assert_cast<ColumnFixedString &>(*key_columns[0]).insertData(key.data, key.size);
-        }
-    }
-};
-
-
-/// Single low cardinality column.
-template <typename SingleColumnMethod>
-struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod
-{
-    using Base = SingleColumnMethod;
-    using Data = typename Base::Data;
-    using Key = typename Base::Key;
-    using Mapped = typename Base::Mapped;
-    using Base::data;
-
-    template <bool use_cache>
-    using BaseStateImpl = typename Base::template StateImpl<use_cache>;
-
-    AggregationMethodSingleLowCardinalityColumn() = default;
-
-    template <typename Other>
-    explicit AggregationMethodSingleLowCardinalityColumn(const Other & other) : Base(other) {}
-
-    template <bool use_cache>
-    using StateImpl = ColumnsHashing::HashMethodSingleLowCardinalityColumn<BaseStateImpl<use_cache>, Mapped, use_cache>;
-
-    using State = StateImpl<true>;
-    using StateNoCache = StateImpl<false>;
-
-    static const bool low_cardinality_optimization = true;
-
-    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
-
-    static void insertKeyIntoColumns(const Key & key,
-         std::vector<IColumn *> & key_columns_low_cardinality, const Sizes & /*key_sizes*/)
-    {
-        auto * col = assert_cast<ColumnLowCardinality *>(key_columns_low_cardinality[0]);
-
-        if constexpr (std::is_same_v<Key, StringRef>)
-        {
-            col->insertData(key.data, key.size);
-        }
-        else
-        {
-            col->insertData(reinterpret_cast<const char *>(&key), sizeof(key));
-        }
-    }
-};
-
-
-/// For the case where all keys are of fixed length, and they fit in N (for example, 128) bits.
-template <typename TData, bool has_nullable_keys_ = false, bool has_low_cardinality_ = false, bool consecutive_keys_optimization = false>
-struct AggregationMethodKeysFixed
-{
-    using Data = TData;
-    using Key = typename Data::key_type;
-    using Mapped = typename Data::mapped_type;
-    static constexpr bool has_nullable_keys = has_nullable_keys_;
-    static constexpr bool has_low_cardinality = has_low_cardinality_;
-
-    Data data;
-
-    AggregationMethodKeysFixed() = default;
-
-    explicit AggregationMethodKeysFixed(size_t size_hint) : data(size_hint) { }
-
-    template <typename Other>
-    explicit AggregationMethodKeysFixed(const Other & other) : data(other.data)
-    {
-    }
-
-    template <bool use_cache>
-    using StateImpl = ColumnsHashing::HashMethodKeysFixed<
-        typename Data::value_type,
-        Key,
-        Mapped,
-        has_nullable_keys,
-        has_low_cardinality,
-        use_cache && consecutive_keys_optimization>;
-
-    using State = StateImpl<true>;
-    using StateNoCache = StateImpl<false>;
-
-    static const bool low_cardinality_optimization = false;
-    static const bool one_key_nullable_optimization = false;
-
-    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> & key_columns, const Sizes & key_sizes)
-    {
-        return State::shuffleKeyColumns(key_columns, key_sizes);
-    }
-
-    static void insertKeyIntoColumns(const Key & key, std::vector<IColumn *> & key_columns, const Sizes & key_sizes)
-    {
-        size_t keys_size = key_columns.size();
-
-        static constexpr auto bitmap_size = has_nullable_keys ? std::tuple_size<KeysNullMap<Key>>::value : 0;
-        /// In any hash key value, column values to be read start just after the bitmap, if it exists.
-        size_t pos = bitmap_size;
-
-        for (size_t i = 0; i < keys_size; ++i)
-        {
-            IColumn * observed_column;
-            ColumnUInt8 * null_map;
-
-            bool column_nullable = false;
-            if constexpr (has_nullable_keys)
-                column_nullable = isColumnNullable(*key_columns[i]);
-
-            /// If we have a nullable column, get its nested column and its null map.
-            if (column_nullable)
-            {
-                ColumnNullable & nullable_col = assert_cast<ColumnNullable &>(*key_columns[i]);
-                observed_column = &nullable_col.getNestedColumn();
-                null_map = assert_cast<ColumnUInt8 *>(&nullable_col.getNullMapColumn());
-            }
-            else
-            {
-                observed_column = key_columns[i];
-                null_map = nullptr;
-            }
-
-            bool is_null = false;
-            if (column_nullable)
-            {
-                /// The current column is nullable. Check if the value of the
-                /// corresponding key is nullable. Update the null map accordingly.
-                size_t bucket = i / 8;
-                size_t offset = i % 8;
-                UInt8 val = (reinterpret_cast<const UInt8 *>(&key)[bucket] >> offset) & 1;
-                null_map->insertValue(val);
-                is_null = val == 1;
-            }
-
-            if (has_nullable_keys && is_null)
-                observed_column->insertDefault();
-            else
-            {
-                size_t size = key_sizes[i];
-                size_t offset_to = pos;
-                if constexpr (std::endian::native == std::endian::big)
-                   offset_to = sizeof(Key) - size - pos;
-                observed_column->insertData(reinterpret_cast<const char *>(&key) + offset_to, size);
-                pos += size;
-            }
-        }
-    }
-};
-
-
-/** Aggregates by concatenating serialized key values.
-  * The serialized value differs in that it uniquely allows to deserialize it, having only the position with which it starts.
-  * That is, for example, for strings, it contains first the serialized length of the string, and then the bytes.
-  * Therefore, when aggregating by several strings, there is no ambiguity.
-  */
-template <typename TData, bool nullable = false, bool prealloc = false>
-struct AggregationMethodSerialized
-{
-    using Data = TData;
-    using Key = typename Data::key_type;
-    using Mapped = typename Data::mapped_type;
-
-    Data data;
-
-    AggregationMethodSerialized() = default;
-
-    explicit AggregationMethodSerialized(size_t size_hint) : data(size_hint) { }
-
-    template <typename Other>
-    explicit AggregationMethodSerialized(const Other & other) : data(other.data)
-    {
-    }
-
-    template <bool use_cache>
-    using StateImpl = ColumnsHashing::HashMethodSerialized<typename Data::value_type, Mapped, nullable, prealloc>;
-
-    using State = StateImpl<true>;
-    using StateNoCache = StateImpl<false>;
-
-    static const bool low_cardinality_optimization = false;
-    static const bool one_key_nullable_optimization = false;
-
-    std::optional<Sizes> shuffleKeyColumns(std::vector<IColumn *> &, const Sizes &) { return {}; }
-
-    static void insertKeyIntoColumns(StringRef key, std::vector<IColumn *> & key_columns, const Sizes &)
-    {
-        const auto * pos = key.data;
-        for (auto & column : key_columns)
-            pos = column->deserializeAndInsertFromArena(pos);
-    }
-};
-
-template <typename TData>
-using AggregationMethodNullableSerialized = AggregationMethodSerialized<TData, true>;
-
-template <typename TData>
-using AggregationMethodPreallocSerialized = AggregationMethodSerialized<TData, false, true>;
-
-template <typename TData>
-using AggregationMethodNullablePreallocSerialized = AggregationMethodSerialized<TData, true, true>;
-
-class Aggregator;
-
 using ColumnsHashing::HashMethodContext;
 using ColumnsHashing::HashMethodContextPtr;
 using ColumnsHashing::LastElementCacheStats;
 
-struct AggregatedDataVariants : private boost::noncopyable
-{
-    /** Working with states of aggregate functions in the pool is arranged in the following (inconvenient) way:
-      * - when aggregating, states are created in the pool using IAggregateFunction::create (inside - `placement new` of arbitrary structure);
-      * - they must then be destroyed using IAggregateFunction::destroy (inside - calling the destructor of arbitrary structure);
-      * - if aggregation is complete, then, in the Aggregator::convertToBlocks function, pointers to the states of aggregate functions
-      *   are written to ColumnAggregateFunction; ColumnAggregateFunction "acquires ownership" of them, that is - calls `destroy` in its destructor.
-      * - if during the aggregation, before call to Aggregator::convertToBlocks, an exception was thrown,
-      *   then the states of aggregate functions must still be destroyed,
-      *   otherwise, for complex states (eg, AggregateFunctionUniq), there will be memory leaks;
-      * - in this case, to destroy states, the destructor calls Aggregator::destroyAggregateStates method,
-      *   but only if the variable aggregator (see below) is not nullptr;
-      * - that is, until you transfer ownership of the aggregate function states in the ColumnAggregateFunction, set the variable `aggregator`,
-      *   so that when an exception occurs, the states are correctly destroyed.
-      *
-      * PS. This can be corrected by making a pool that knows about which states of aggregate functions and in which order are put in it, and knows how to destroy them.
-      * But this can hardly be done simply because it is planned to put variable-length strings into the same pool.
-      * In this case, the pool will not be able to know with what offsets objects are stored.
-      */
-    const Aggregator * aggregator = nullptr;
-
-    size_t keys_size{};  /// Number of keys. NOTE do we need this field?
-    Sizes key_sizes;     /// Dimensions of keys, if keys of fixed length
-
-    /// Pools for states of aggregate functions. Ownership will be later transferred to ColumnAggregateFunction.
-    Arenas aggregates_pools;
-    Arena * aggregates_pool{};    /// The pool that is currently used for allocation.
-
-    /** Specialization for the case when there are no keys, and for keys not fitted into max_rows_to_group_by.
-      */
-    AggregatedDataWithoutKey without_key = nullptr;
-
-    /// Stats of a cache for consecutive keys optimization.
-    /// Stats can be used to disable the cache in case of a lot of misses.
-    LastElementCacheStats consecutive_keys_cache_stats;
-
-    // Disable consecutive key optimization for Uint8/16, because they use a FixedHashMap
-    // and the lookup there is almost free, so we don't need to cache the last lookup result
-    std::unique_ptr<AggregationMethodOneNumber<UInt8, AggregatedDataWithUInt8Key, false>>           key8;
-    std::unique_ptr<AggregationMethodOneNumber<UInt16, AggregatedDataWithUInt16Key, false>>         key16;
-
-    std::unique_ptr<AggregationMethodOneNumber<UInt32, AggregatedDataWithUInt64Key>>         key32;
-    std::unique_ptr<AggregationMethodOneNumber<UInt64, AggregatedDataWithUInt64Key>>         key64;
-    std::unique_ptr<AggregationMethodStringNoCache<AggregatedDataWithShortStringKey>>               key_string;
-    std::unique_ptr<AggregationMethodFixedStringNoCache<AggregatedDataWithShortStringKey>>          key_fixed_string;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithUInt16Key, false, false, false>>  keys16;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithUInt32Key>>                   keys32;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithUInt64Key>>                   keys64;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128>>                   keys128;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256>>                   keys256;
-    std::unique_ptr<AggregationMethodSerialized<AggregatedDataWithStringKey>>                          serialized;
-    std::unique_ptr<AggregationMethodNullableSerialized<AggregatedDataWithStringKey>>                  nullable_serialized;
-    std::unique_ptr<AggregationMethodPreallocSerialized<AggregatedDataWithStringKey>>                  prealloc_serialized;
-    std::unique_ptr<AggregationMethodNullablePreallocSerialized<AggregatedDataWithStringKey>>          nullable_prealloc_serialized;
-
-    std::unique_ptr<AggregationMethodOneNumber<UInt32, AggregatedDataWithUInt64KeyTwoLevel>> key32_two_level;
-    std::unique_ptr<AggregationMethodOneNumber<UInt64, AggregatedDataWithUInt64KeyTwoLevel>> key64_two_level;
-    std::unique_ptr<AggregationMethodStringNoCache<AggregatedDataWithShortStringKeyTwoLevel>>       key_string_two_level;
-    std::unique_ptr<AggregationMethodFixedStringNoCache<AggregatedDataWithShortStringKeyTwoLevel>>  key_fixed_string_two_level;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithUInt32KeyTwoLevel>>           keys32_two_level;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithUInt64KeyTwoLevel>>           keys64_two_level;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128TwoLevel>>           keys128_two_level;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256TwoLevel>>           keys256_two_level;
-    std::unique_ptr<AggregationMethodSerialized<AggregatedDataWithStringKeyTwoLevel>>                  serialized_two_level;
-    std::unique_ptr<AggregationMethodNullableSerialized<AggregatedDataWithStringKeyTwoLevel>>          nullable_serialized_two_level;
-    std::unique_ptr<AggregationMethodPreallocSerialized<AggregatedDataWithStringKeyTwoLevel>>          prealloc_serialized_two_level;
-    std::unique_ptr<AggregationMethodNullablePreallocSerialized<AggregatedDataWithStringKeyTwoLevel>>  nullable_prealloc_serialized_two_level;
-
-    std::unique_ptr<AggregationMethodOneNumber<UInt64, AggregatedDataWithUInt64KeyHash64>>   key64_hash64;
-    std::unique_ptr<AggregationMethodString<AggregatedDataWithStringKeyHash64>>              key_string_hash64;
-    std::unique_ptr<AggregationMethodFixedString<AggregatedDataWithStringKeyHash64>>         key_fixed_string_hash64;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128Hash64>>             keys128_hash64;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256Hash64>>             keys256_hash64;
-    std::unique_ptr<AggregationMethodSerialized<AggregatedDataWithStringKeyHash64>>                  serialized_hash64;
-    std::unique_ptr<AggregationMethodNullableSerialized<AggregatedDataWithStringKeyHash64>>          nullable_serialized_hash64;
-    std::unique_ptr<AggregationMethodPreallocSerialized<AggregatedDataWithStringKeyHash64>>          prealloc_serialized_hash64;
-    std::unique_ptr<AggregationMethodNullablePreallocSerialized<AggregatedDataWithStringKeyHash64>>  nullable_prealloc_serialized_hash64;
-
-    /// Support for nullable keys.
-    std::unique_ptr<AggregationMethodOneNumber<UInt8, AggregatedDataWithNullableUInt8Key, false, true>>         nullable_key8;
-    std::unique_ptr<AggregationMethodOneNumber<UInt16, AggregatedDataWithNullableUInt16Key, false, true>>         nullable_key16;
-    std::unique_ptr<AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt32Key, true, true>>         nullable_key32;
-    std::unique_ptr<AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64Key, true, true>>         nullable_key64;
-    std::unique_ptr<AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt32KeyTwoLevel, true, true>>         nullable_key32_two_level;
-    std::unique_ptr<AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64KeyTwoLevel, true, true>>         nullable_key64_two_level;
-
-    std::unique_ptr<AggregationMethodStringNoCache<AggregatedDataWithNullableShortStringKey, true>> nullable_key_string;
-    std::unique_ptr<AggregationMethodFixedStringNoCache<AggregatedDataWithNullableShortStringKey, true>> nullable_key_fixed_string;
-    std::unique_ptr<AggregationMethodStringNoCache<AggregatedDataWithNullableShortStringKeyTwoLevel, true>> nullable_key_string_two_level;
-    std::unique_ptr<AggregationMethodFixedStringNoCache<AggregatedDataWithNullableShortStringKeyTwoLevel, true>> nullable_key_fixed_string_two_level;
-
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128, true>>             nullable_keys128;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256, true>>             nullable_keys256;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128TwoLevel, true>>     nullable_keys128_two_level;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256TwoLevel, true>>     nullable_keys256_two_level;
-
-    /// Support for low cardinality.
-    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt8, AggregatedDataWithNullableUInt8Key, false>>> low_cardinality_key8;
-    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt16, AggregatedDataWithNullableUInt16Key, false>>> low_cardinality_key16;
-    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt64Key>>> low_cardinality_key32;
-    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64Key>>> low_cardinality_key64;
-    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodString<AggregatedDataWithNullableStringKey>>> low_cardinality_key_string;
-    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodFixedString<AggregatedDataWithNullableStringKey>>> low_cardinality_key_fixed_string;
-
-    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt64KeyTwoLevel>>> low_cardinality_key32_two_level;
-    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64KeyTwoLevel>>> low_cardinality_key64_two_level;
-    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodString<AggregatedDataWithNullableStringKeyTwoLevel>>> low_cardinality_key_string_two_level;
-    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodFixedString<AggregatedDataWithNullableStringKeyTwoLevel>>> low_cardinality_key_fixed_string_two_level;
-
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128, false, true>>      low_cardinality_keys128;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256, false, true>>      low_cardinality_keys256;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys128TwoLevel, false, true>> low_cardinality_keys128_two_level;
-    std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256TwoLevel, false, true>> low_cardinality_keys256_two_level;
-
-    /// In this and similar macros, the option without_key is not considered.
-    #define APPLY_FOR_AGGREGATED_VARIANTS(M) \
-        M(key8,                       false) \
-        M(key16,                      false) \
-        M(key32,                      false) \
-        M(key64,                      false) \
-        M(key_string,                 false) \
-        M(key_fixed_string,           false) \
-        M(keys16,                    false) \
-        M(keys32,                    false) \
-        M(keys64,                    false) \
-        M(keys128,                    false) \
-        M(keys256,                    false) \
-        M(serialized,                   false) \
-        M(nullable_serialized,          false) \
-        M(prealloc_serialized,          false) \
-        M(nullable_prealloc_serialized, false) \
-        M(key32_two_level,            true) \
-        M(key64_two_level,            true) \
-        M(key_string_two_level,       true) \
-        M(key_fixed_string_two_level, true) \
-        M(keys32_two_level,          true) \
-        M(keys64_two_level,          true) \
-        M(keys128_two_level,          true) \
-        M(keys256_two_level,          true) \
-        M(serialized_two_level,                   true) \
-        M(nullable_serialized_two_level,          true) \
-        M(prealloc_serialized_two_level,          true) \
-        M(nullable_prealloc_serialized_two_level, true) \
-        M(key64_hash64,               false) \
-        M(key_string_hash64,          false) \
-        M(key_fixed_string_hash64,    false) \
-        M(keys128_hash64,             false) \
-        M(keys256_hash64,             false) \
-        M(serialized_hash64,                   false) \
-        M(nullable_serialized_hash64,          false) \
-        M(prealloc_serialized_hash64,          false) \
-        M(nullable_prealloc_serialized_hash64, false) \
-        M(nullable_key8,             false) \
-        M(nullable_key16,             false) \
-        M(nullable_key32,             false) \
-        M(nullable_key64,             false) \
-        M(nullable_key32_two_level,   true) \
-        M(nullable_key64_two_level,   true) \
-        M(nullable_key_string,        false) \
-        M(nullable_key_fixed_string,  false) \
-        M(nullable_key_string_two_level, true) \
-        M(nullable_key_fixed_string_two_level, true) \
-        M(nullable_keys128,           false) \
-        M(nullable_keys256,           false) \
-        M(nullable_keys128_two_level, true) \
-        M(nullable_keys256_two_level, true) \
-        M(low_cardinality_key8, false) \
-        M(low_cardinality_key16, false) \
-        M(low_cardinality_key32, false) \
-        M(low_cardinality_key64, false) \
-        M(low_cardinality_keys128, false) \
-        M(low_cardinality_keys256, false) \
-        M(low_cardinality_key_string, false) \
-        M(low_cardinality_key_fixed_string, false) \
-        M(low_cardinality_key32_two_level, true) \
-        M(low_cardinality_key64_two_level, true) \
-        M(low_cardinality_keys128_two_level, true) \
-        M(low_cardinality_keys256_two_level, true) \
-        M(low_cardinality_key_string_two_level, true) \
-        M(low_cardinality_key_fixed_string_two_level, true) \
-
-    enum class Type
-    {
-        EMPTY = 0,
-        without_key,
-
-    #define M(NAME, IS_TWO_LEVEL) NAME,
-        APPLY_FOR_AGGREGATED_VARIANTS(M)
-    #undef M
-    };
-    Type type = Type::EMPTY;
-
-    AggregatedDataVariants() : aggregates_pools(1, std::make_shared<Arena>()), aggregates_pool(aggregates_pools.back().get()) {}
-    bool empty() const { return type == Type::EMPTY; }
-    void invalidate() { type = Type::EMPTY; }
-
-    ~AggregatedDataVariants();
-
-    void init(Type type_, std::optional<size_t> size_hint = std::nullopt);
-
-    /// Number of rows (different keys).
-    size_t size() const
-    {
-        switch (type)
-        {
-            case Type::EMPTY:       return 0;
-            case Type::without_key: return 1;
-
-        #define M(NAME, IS_TWO_LEVEL) \
-            case Type::NAME: return (NAME)->data.size() + (without_key != nullptr);
-            APPLY_FOR_AGGREGATED_VARIANTS(M)
-        #undef M
-        }
-
-        UNREACHABLE();
-    }
-
-    /// The size without taking into account the row in which data is written for the calculation of TOTALS.
-    size_t sizeWithoutOverflowRow() const
-    {
-        switch (type)
-        {
-            case Type::EMPTY:       return 0;
-            case Type::without_key: return 1;
-
-            #define M(NAME, IS_TWO_LEVEL) \
-            case Type::NAME: return (NAME)->data.size();
-            APPLY_FOR_AGGREGATED_VARIANTS(M)
-            #undef M
-        }
-
-        UNREACHABLE();
-    }
-
-    const char * getMethodName() const
-    {
-        switch (type)
-        {
-            case Type::EMPTY:       return "EMPTY";
-            case Type::without_key: return "without_key";
-
-        #define M(NAME, IS_TWO_LEVEL) \
-            case Type::NAME: return #NAME;
-            APPLY_FOR_AGGREGATED_VARIANTS(M)
-        #undef M
-        }
-
-        UNREACHABLE();
-    }
-
-    bool isTwoLevel() const
-    {
-        switch (type)
-        {
-            case Type::EMPTY:       return false;
-            case Type::without_key: return false;
-
-        #define M(NAME, IS_TWO_LEVEL) \
-            case Type::NAME: return IS_TWO_LEVEL;
-            APPLY_FOR_AGGREGATED_VARIANTS(M)
-        #undef M
-        }
-
-        UNREACHABLE();
-    }
-
-    #define APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) \
-        M(key32)            \
-        M(key64)            \
-        M(key_string)       \
-        M(key_fixed_string) \
-        M(keys32)           \
-        M(keys64)           \
-        M(keys128)          \
-        M(keys256)          \
-        M(serialized)       \
-        M(nullable_serialized) \
-        M(prealloc_serialized) \
-        M(nullable_prealloc_serialized) \
-        M(nullable_key32) \
-        M(nullable_key64) \
-        M(nullable_key_string) \
-        M(nullable_key_fixed_string) \
-        M(nullable_keys128) \
-        M(nullable_keys256) \
-        M(low_cardinality_key32) \
-        M(low_cardinality_key64) \
-        M(low_cardinality_keys128) \
-        M(low_cardinality_keys256) \
-        M(low_cardinality_key_string) \
-        M(low_cardinality_key_fixed_string) \
-
-    /// NOLINTNEXTLINE
-    #define APPLY_FOR_VARIANTS_NOT_CONVERTIBLE_TO_TWO_LEVEL(M) \
-        M(key8)             \
-        M(key16)            \
-        M(nullable_key8) \
-        M(nullable_key16) \
-        M(keys16)           \
-        M(key64_hash64)     \
-        M(key_string_hash64)\
-        M(key_fixed_string_hash64) \
-        M(keys128_hash64)   \
-        M(keys256_hash64)   \
-        M(serialized_hash64) \
-        M(nullable_serialized_hash64) \
-        M(prealloc_serialized_hash64) \
-        M(nullable_prealloc_serialized_hash64) \
-        M(low_cardinality_key8) \
-        M(low_cardinality_key16) \
-
-    /// NOLINTNEXTLINE
-    #define APPLY_FOR_VARIANTS_SINGLE_LEVEL(M) \
-        APPLY_FOR_VARIANTS_NOT_CONVERTIBLE_TO_TWO_LEVEL(M) \
-        APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) \
-
-    bool isConvertibleToTwoLevel() const
-    {
-        switch (type)
-        {
-        #define M(NAME) \
-            case Type::NAME: return true;
-
-            APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M)
-
-        #undef M
-            default:
-                return false;
-        }
-    }
-
-    void convertToTwoLevel();
-
-    /// NOLINTNEXTLINE
-    #define APPLY_FOR_VARIANTS_TWO_LEVEL(M) \
-        M(key32_two_level)            \
-        M(key64_two_level)            \
-        M(key_string_two_level)       \
-        M(key_fixed_string_two_level) \
-        M(keys32_two_level)           \
-        M(keys64_two_level)           \
-        M(keys128_two_level)          \
-        M(keys256_two_level)          \
-        M(serialized_two_level)       \
-        M(nullable_serialized_two_level)       \
-        M(prealloc_serialized_two_level)       \
-        M(nullable_prealloc_serialized_two_level)       \
-        M(nullable_key32_two_level) \
-        M(nullable_key64_two_level) \
-        M(nullable_key_string_two_level) \
-        M(nullable_key_fixed_string_two_level) \
-        M(nullable_keys128_two_level) \
-        M(nullable_keys256_two_level) \
-        M(low_cardinality_key32_two_level) \
-        M(low_cardinality_key64_two_level) \
-        M(low_cardinality_keys128_two_level) \
-        M(low_cardinality_keys256_two_level) \
-        M(low_cardinality_key_string_two_level) \
-        M(low_cardinality_key_fixed_string_two_level) \
-
-    #define APPLY_FOR_LOW_CARDINALITY_VARIANTS(M) \
-        M(low_cardinality_key8) \
-        M(low_cardinality_key16) \
-        M(low_cardinality_key32) \
-        M(low_cardinality_key64) \
-        M(low_cardinality_keys128) \
-        M(low_cardinality_keys256) \
-        M(low_cardinality_key_string) \
-        M(low_cardinality_key_fixed_string) \
-        M(low_cardinality_key32_two_level) \
-        M(low_cardinality_key64_two_level) \
-        M(low_cardinality_keys128_two_level) \
-        M(low_cardinality_keys256_two_level) \
-        M(low_cardinality_key_string_two_level) \
-        M(low_cardinality_key_fixed_string_two_level)
-
-    bool isLowCardinality() const
-    {
-        switch (type)
-        {
-        #define M(NAME) \
-            case Type::NAME: return true;
-
-            APPLY_FOR_LOW_CARDINALITY_VARIANTS(M)
-        #undef M
-            default:
-                return false;
-        }
-    }
-
-    static HashMethodContextPtr createCache(Type type, const HashMethodContext::Settings & settings)
-    {
-        switch (type)
-        {
-            case Type::without_key: return nullptr;
-
-            #define M(NAME, IS_TWO_LEVEL) \
-            case Type::NAME: \
-            { \
-                using TPtr ## NAME = decltype(AggregatedDataVariants::NAME); \
-                using T ## NAME = typename TPtr ## NAME ::element_type; \
-                return T ## NAME ::State::createContext(settings); \
-            }
-
-            APPLY_FOR_AGGREGATED_VARIANTS(M)
-            #undef M
-
-            default:
-                throw Exception(ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT, "Unknown aggregated data variant.");
-        }
-    }
-};
-
-using AggregatedDataVariantsPtr = std::shared_ptr<AggregatedDataVariants>;
-using ManyAggregatedDataVariants = std::vector<AggregatedDataVariantsPtr>;
-using ManyAggregatedDataVariantsPtr = std::shared_ptr<ManyAggregatedDataVariants>;
-
 class CompiledAggregateFunctionsHolder;
 class NativeWriter;
 struct OutputBlockColumns;
@@ -1395,7 +419,7 @@ private:
         AggregateDataPtr overflow_row) const;
 
     /// Specialization for a particular value no_more_keys.
-    template <bool no_more_keys, bool use_compiled_functions, bool prefetch, typename Method, typename State>
+    template <bool prefetch, typename Method, typename State>
     void executeImplBatch(
         Method & method,
         State & state,
@@ -1403,17 +427,31 @@ private:
         size_t row_begin,
         size_t row_end,
         AggregateFunctionInstruction * aggregate_instructions,
+        bool no_more_keys,
         bool all_keys_are_const,
+        bool use_compiled_functions,
         AggregateDataPtr overflow_row) const;
 
+    void executeAggregateInstructions(
+        Arena * aggregates_pool,
+        size_t row_begin,
+        size_t row_end,
+        AggregateFunctionInstruction * aggregate_instructions,
+        const std::unique_ptr<AggregateDataPtr[]> & places,
+        size_t key_start,
+        bool has_only_one_value_since_last_reset,
+        bool no_more_keys,
+        bool all_keys_are_const,
+        bool use_compiled_functions) const;
+
     /// For case when there are no keys (all aggregate into one row).
-    template <bool use_compiled_functions>
     void executeWithoutKeyImpl(
         AggregatedDataWithoutKey & res,
         size_t row_begin,
         size_t row_end,
         AggregateFunctionInstruction * aggregate_instructions,
-        Arena * arena) const;
+        Arena * arena,
+        bool use_compiled_functions) const;
 
     template <typename Method>
     void writeToTemporaryFileImpl(
@@ -1429,8 +467,9 @@ private:
             Arena * arena) const;
 
     /// Merge data from hash table `src` into `dst`.
-    template <typename Method, bool use_compiled_functions, bool prefetch, typename Table>
-    void mergeDataImpl(Table & table_dst, Table & table_src, Arena * arena) const;
+    // template <typename Method, bool use_compiled_functions, booal prefetch, typename Table>
+    template <typename Method, typename Table>
+    void mergeDataImpl(Table & table_dst, Table & table_src, Arena * arena, bool use_compiled_functions, bool prefetch) const;
 
     /// Merge data from hash table `src` into `dst`, but only for keys that already exist in dst. In other cases, merge the data into `overflows`.
     template <typename Method, typename Table>
@@ -1456,10 +495,11 @@ private:
 
     template <bool return_single_block>
     using ConvertToBlockRes = std::conditional_t<return_single_block, Block, BlocksList>;
+    using ConvertToBlockResVariant = std::variant<Block, BlocksList>;
 
-    template <bool return_single_block, typename Method, typename Table>
-    ConvertToBlockRes<return_single_block>
-    convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final, size_t rows) const;
+    template <typename Method, typename Table>
+    ConvertToBlockResVariant
+    convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block) const;
 
     template <typename Mapped>
     void insertAggregatesIntoColumns(
@@ -1470,13 +510,18 @@ private:
     template <bool use_compiled_functions>
     Block insertResultsIntoColumns(PaddedPODArray<AggregateDataPtr> & places, OutputBlockColumns && out_cols, Arena * arena, bool has_null_key_data) const;
 
-    template <typename Method, bool use_compiled_functions, bool return_single_block, typename Table>
-    ConvertToBlockRes<return_single_block>
-    convertToBlockImplFinal(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, size_t rows) const;
+    template <typename Method, typename Table>
+    ConvertToBlockResVariant convertToBlockImplFinal(
+        Method & method,
+        Table & data,
+        Arena * arena,
+        Arenas & aggregates_pools,
+        bool use_compiled_functions,
+        bool return_single_block) const;
 
-    template <bool return_single_block, typename Method, typename Table>
-    ConvertToBlockRes<return_single_block>
-    convertToBlockImplNotFinal(Method & method, Table & data, Arenas & aggregates_pools, size_t rows) const;
+    template <typename Method, typename Table>
+    ConvertToBlockResVariant
+    convertToBlockImplNotFinal(Method & method, Table & data, Arenas & aggregates_pools, size_t rows, bool return_single_block) const;
 
     template <typename Method>
     Block convertOneBucketToBlock(

From f559767d2201e6f3a88a97cb72a5be6646b38585 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 15 Feb 2024 16:56:04 +0100
Subject: [PATCH 10/92] Fix finished_mutations_to_keep=0 for MergeTree (as docs
 says 0 is to keep everything)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Storages/StorageMergeTree.cpp                  |  5 ++++-
 .../02994_merge_tree_mutations_cleanup.reference   |  4 ++++
 .../02994_merge_tree_mutations_cleanup.sql.j2      | 14 ++++++++++++++
 3 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/02994_merge_tree_mutations_cleanup.reference
 create mode 100644 tests/queries/0_stateless/02994_merge_tree_mutations_cleanup.sql.j2

diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 663e7f435b7..6bb8695d673 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -1479,8 +1479,11 @@ UInt64 StorageMergeTree::getCurrentMutationVersion(
 
 size_t StorageMergeTree::clearOldMutations(bool truncate)
 {
-    size_t finished_mutations_to_keep = truncate ? 0 : getSettings()->finished_mutations_to_keep;
+    size_t finished_mutations_to_keep = getSettings()->finished_mutations_to_keep;
+    if (!truncate && !finished_mutations_to_keep)
+        return 0;
 
+    finished_mutations_to_keep = truncate ? 0 : finished_mutations_to_keep;
     std::vector<MergeTreeMutationEntry> mutations_to_delete;
     {
         std::lock_guard lock(currently_processing_in_background_mutex);
diff --git a/tests/queries/0_stateless/02994_merge_tree_mutations_cleanup.reference b/tests/queries/0_stateless/02994_merge_tree_mutations_cleanup.reference
new file mode 100644
index 00000000000..8a3a458c753
--- /dev/null
+++ b/tests/queries/0_stateless/02994_merge_tree_mutations_cleanup.reference
@@ -0,0 +1,4 @@
+mutations after ALTER for data_rmt	1
+mutations after cleanup for data_rmt	1
+mutations after ALTER for data_mt	1
+mutations after cleanup for data_mt	1
diff --git a/tests/queries/0_stateless/02994_merge_tree_mutations_cleanup.sql.j2 b/tests/queries/0_stateless/02994_merge_tree_mutations_cleanup.sql.j2
new file mode 100644
index 00000000000..2740b6070c0
--- /dev/null
+++ b/tests/queries/0_stateless/02994_merge_tree_mutations_cleanup.sql.j2
@@ -0,0 +1,14 @@
+drop table if exists data_rmt;
+drop table if exists data_mt;
+
+create table data_rmt (key Int) engine=ReplicatedMergeTree('/tables/{database}/data', '{table}') order by tuple() settings finished_mutations_to_keep=0, merge_tree_clear_old_parts_interval_seconds=1;
+create table data_mt (key Int) engine=MergeTree() order by tuple() settings finished_mutations_to_keep=0, merge_tree_clear_old_parts_interval_seconds=1;
+
+{% for table in ['data_rmt', 'data_mt'] %}
+alter table {{table}} delete where 1;
+select 'mutations after ALTER for {{table}}', count() from system.mutations where database = currentDatabase() and table = '{{table}}';
+-- merge_tree_clear_old_parts_interval_seconds=1, but wait few seconds more
+select sleep(5) settings function_sleep_max_microseconds_per_block=10e6 format Null;
+select 'mutations after cleanup for {{table}}', count() from system.mutations where database = currentDatabase() and table = '{{table}}';
+drop table {{table}};
+{% endfor %}

From 6c74ab18c0d42082d49786fddb4612c813f86c0e Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 22 Feb 2024 15:10:52 +0100
Subject: [PATCH 11/92] Do not clean mutations for MergeTree on DROP PART (to
 match ReplicatedMergeTree)

Even if this should be done, only related mutations should be cleaned,
not all.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Storages/StorageMergeTree.cpp | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 6bb8695d673..2345cc8e79d 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -1902,8 +1902,6 @@ void StorageMergeTree::dropPart(const String & part_name, bool detach, ContextPt
         }
     }
 
-    /// Old part objects is needed to be destroyed before clearing them from filesystem.
-    clearOldMutations(true);
     clearOldPartsFromFilesystem();
     clearEmptyParts();
 }
@@ -1988,8 +1986,6 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, Cont
         }
     }
 
-    /// Old parts are needed to be destroyed before clearing them from filesystem.
-    clearOldMutations(true);
     clearOldPartsFromFilesystem();
     clearEmptyParts();
 }

From 4bd6d4456e2a6267677941437389558b1eed1402 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 24 Feb 2024 21:25:48 +0100
Subject: [PATCH 12/92] Fix expectations for
 test_all_projection_files_are_dropped_when_part_is_dropped
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Since after DROP PART mutations are not cleaned anymore.

Here an example:

                objects_at_the_end = list_objects(cluster)
    >           assert objects_at_the_end == objects_empty_table
    E           AssertionError: assert ['data/evt/iczupcswcatzvjikqwmovahturdht', 'data/oxq/dmkgqbdzeeiwmukogoxawfoxxnrqs'] == ['data/oxq/dmkgqbdzeeiwmukogoxawfoxxnrqs']
    E             At index 0 diff: 'data/evt/iczupcswcatzvjikqwmovahturdht' != 'data/oxq/dmkgqbdzeeiwmukogoxawfoxxnrqs'
    E             Left contains one more item: 'data/oxq/dmkgqbdzeeiwmukogoxawfoxxnrqs'
    E             Full diff:
    E               [
    E             +  'data/evt/iczupcswcatzvjikqwmovahturdht',
    E                'data/oxq/dmkgqbdzeeiwmukogoxawfoxxnrqs',
    E               ]

    test_replicated_zero_copy_projection_mutation/test.py:155: AssertionError

And decoded paths:

    node1 :) select local_path from system.blob_storage_log where remote_path = 'data/evt/iczupcswcatzvjikqwmovahturdht'

    SELECT local_path
    FROM system.blob_storage_log
    WHERE remote_path = 'data/evt/iczupcswcatzvjikqwmovahturdht'

    Query id: 9ee5a9c0-c3b7-46ad-82bd-64c8bcbda78d

    ┌─local_path────────────────────────────────────────────────────────┐
    │ store/bce/bcea71c9-35cd-4368-9504-c563253b1964/tmp_mutation_1.txt │
    │ store/bce/bcea71c9-35cd-4368-9504-c563253b1964/mutation_1.txt     │
    └───────────────────────────────────────────────────────────────────┘

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../test_replicated_zero_copy_projection_mutation/test.py      | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/integration/test_replicated_zero_copy_projection_mutation/test.py b/tests/integration/test_replicated_zero_copy_projection_mutation/test.py
index 4839919e23d..e0af85c3887 100644
--- a/tests/integration/test_replicated_zero_copy_projection_mutation/test.py
+++ b/tests/integration/test_replicated_zero_copy_projection_mutation/test.py
@@ -131,14 +131,13 @@ def test_all_projection_files_are_dropped_when_part_is_dropped(
             """
         )
 
-        objects_empty_table = list_objects(cluster)
-
         node.query(
             "ALTER TABLE test_all_projection_files_are_dropped ADD projection b_order (SELECT a, b ORDER BY b)"
         )
         node.query(
             "ALTER TABLE test_all_projection_files_are_dropped MATERIALIZE projection b_order"
         )
+        objects_empty_table = list_objects(cluster)
 
         node.query(
             """

From 3c08e198adb9bb62f1480f94470741ae988b249e Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 7 Mar 2024 09:59:19 +0100
Subject: [PATCH 13/92] tests: wait for mutations in
 02994_merge_tree_mutations_cleanup.sql.j2
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Otherwise it fails in private fork CI

Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
---
 .../0_stateless/02994_merge_tree_mutations_cleanup.sql.j2       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02994_merge_tree_mutations_cleanup.sql.j2 b/tests/queries/0_stateless/02994_merge_tree_mutations_cleanup.sql.j2
index 2740b6070c0..1b9be79dbe4 100644
--- a/tests/queries/0_stateless/02994_merge_tree_mutations_cleanup.sql.j2
+++ b/tests/queries/0_stateless/02994_merge_tree_mutations_cleanup.sql.j2
@@ -5,7 +5,7 @@ create table data_rmt (key Int) engine=ReplicatedMergeTree('/tables/{database}/d
 create table data_mt (key Int) engine=MergeTree() order by tuple() settings finished_mutations_to_keep=0, merge_tree_clear_old_parts_interval_seconds=1;
 
 {% for table in ['data_rmt', 'data_mt'] %}
-alter table {{table}} delete where 1;
+alter table {{table}} delete where 1 settings mutations_sync = 1;
 select 'mutations after ALTER for {{table}}', count() from system.mutations where database = currentDatabase() and table = '{{table}}';
 -- merge_tree_clear_old_parts_interval_seconds=1, but wait few seconds more
 select sleep(5) settings function_sleep_max_microseconds_per_block=10e6 format Null;

From f9dcef774a5a6adac29cebc17827eef0cefe9ba9 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 14 Feb 2024 14:58:49 +0100
Subject: [PATCH 14/92] Fix function execution over const and LowCardinality
 with GROUP BY const for analyzer

Consider the following example:

    SELECT concatWithSeparator('|', 'a', concatWithSeparator('|', CAST('x', 'LowCardinality(String)'))) GROUP BY 'a'

Under analyzer it fails, UBsan report:

    ==15121==WARNING: MemorySanitizer: use-of-uninitialized-value
      ...
      8 0x5555601880ed in void DB::FormatStringImpl::format<true, false>() /src/ch/clickhouse/src/Functions/formatString.h:125:21
      9 0x55556017aeb8 in void DB::FormatStringImpl::formatExecute<>() /src/ch/clickhouse/src/Functions/formatString.h:30:13
      10 0x555560196779 in DB::()::ConcatWithSeparatorImpl<>::executeImpl() const /src/ch/clickhouse/src/Functions/concatWithSeparator.cpp:151:9
      11 0x55555a2ad5b7 in DB::FunctionToExecutableFunctionAdaptor::executeImpl() const /src/ch/clickhouse/src/Functions/IFunctionAdaptor.h:21:26
      12 0x555584312297 in DB::IExecutableFunction::executeWithoutLowCardinalityColumns() const /src/ch/clickhouse/src/Functions/IFunction.cpp:249:15
      13 0x555584317640 in DB::IExecutableFunction::executeWithoutSparseColumns() const /src/ch/clickhouse/src/Functions/IFunction.cpp:283:24
      14 0x55558431bf5c in DB::IExecutableFunction::execute() const /src/ch/clickhouse/src/Functions/IFunction.cpp:380:16
      15 0x555587bf3e20 in DB::executeAction() /src/ch/clickhouse/src/Interpreters/ExpressionActions.cpp:613:60

    Uninitialized value was created by a heap allocation
      ...
      6 0x55558b1c1a05 in DB::ColumnString::reserve(unsigned long) /src/ch/clickhouse/src/Columns/ColumnString.cpp:494:13
      7 0x55558980095d in DB::prepareOutputBlockColumns() /src/ch/clickhouse/src/Interpreters/AggregationUtils.cpp:32:25

The problem is that during query analysis
(QueryAnalyzer::resolveFunction()), the return value of the function had
been executed as LowCardinality(String), but the 'a' argument that is
passed to the concatWithSeparator() is not-const, because it had been
reused from the GROUP BY step, and this causes UB, since column 'a' does
not have enough rows (it should have 2 rows, since LowCardinality always
contains the default, while it has only 1).

v2: fix GROUPING SETs
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Planner/PlannerExpressionAnalysis.cpp     | 33 +++++++++++++------
 .../02992_analyzer_group_by_const.reference   |  5 +++
 .../02992_analyzer_group_by_const.sql         | 10 ++++++
 3 files changed, 38 insertions(+), 10 deletions(-)
 create mode 100644 tests/queries/0_stateless/02992_analyzer_group_by_const.reference
 create mode 100644 tests/queries/0_stateless/02992_analyzer_group_by_const.sql

diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp
index 52001eb27c5..175a8cc9030 100644
--- a/src/Planner/PlannerExpressionAnalysis.cpp
+++ b/src/Planner/PlannerExpressionAnalysis.cpp
@@ -81,6 +81,27 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(const QueryTreeNodeP
     bool group_by_use_nulls = planner_context->getQueryContext()->getSettingsRef().group_by_use_nulls &&
         (query_node.isGroupByWithGroupingSets() || query_node.isGroupByWithRollup() || query_node.isGroupByWithCube());
 
+    auto add_aggregation_column = [&](const auto & expression_dag_node)
+    {
+        auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type;
+        /// Const columns are special, the return value of the
+        /// function depens on it, and there is a special case for
+        /// const columns with LowCardinality column (see
+        /// IFunctionOverloadResolver::getReturnType()), and return
+        /// values of the function had been already obtained by the
+        /// QueryAnalyzer::resolveFunction(), and if const-ness of
+        /// the column will be ignored here, the return value of
+        /// the function can be changed, and this is UB.
+        if (typeid_cast<const ColumnConst *>(expression_dag_node->column.get()))
+            available_columns_after_aggregation.emplace_back(nullptr, expression_type_after_aggregation, expression_dag_node->result_name + "__group_by");
+        else
+            available_columns_after_aggregation.emplace_back(nullptr, expression_type_after_aggregation, expression_dag_node->result_name);
+
+        aggregation_keys.push_back(expression_dag_node->result_name);
+        before_aggregation_actions->getOutputs().push_back(expression_dag_node);
+        before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name);
+    };
+
     if (query_node.hasGroupBy())
     {
         if (query_node.isGroupByWithGroupingSets())
@@ -108,11 +129,7 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(const QueryTreeNodeP
                         if (before_aggregation_actions_output_node_names.contains(expression_dag_node->result_name))
                             continue;
 
-                        auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type;
-                        available_columns_after_aggregation.emplace_back(nullptr, expression_type_after_aggregation, expression_dag_node->result_name);
-                        aggregation_keys.push_back(expression_dag_node->result_name);
-                        before_aggregation_actions->getOutputs().push_back(expression_dag_node);
-                        before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name);
+                        add_aggregation_column(expression_dag_node);
                     }
                 }
             }
@@ -158,11 +175,7 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(const QueryTreeNodeP
                     if (before_aggregation_actions_output_node_names.contains(expression_dag_node->result_name))
                         continue;
 
-                    auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type;
-                    available_columns_after_aggregation.emplace_back(nullptr, expression_type_after_aggregation, expression_dag_node->result_name);
-                    aggregation_keys.push_back(expression_dag_node->result_name);
-                    before_aggregation_actions->getOutputs().push_back(expression_dag_node);
-                    before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name);
+                    add_aggregation_column(expression_dag_node);
                 }
             }
         }
diff --git a/tests/queries/0_stateless/02992_analyzer_group_by_const.reference b/tests/queries/0_stateless/02992_analyzer_group_by_const.reference
new file mode 100644
index 00000000000..d1fab5a89a5
--- /dev/null
+++ b/tests/queries/0_stateless/02992_analyzer_group_by_const.reference
@@ -0,0 +1,5 @@
+b
+a|a
+a|x
+String, Const(size = 1, String(size = 1))
+String, Const(size = 1, String(size = 1))
diff --git a/tests/queries/0_stateless/02992_analyzer_group_by_const.sql b/tests/queries/0_stateless/02992_analyzer_group_by_const.sql
new file mode 100644
index 00000000000..a027678ec1f
--- /dev/null
+++ b/tests/queries/0_stateless/02992_analyzer_group_by_const.sql
@@ -0,0 +1,10 @@
+SET allow_experimental_analyzer=1;
+
+-- Illegal column String of first argument of function concatWithSeparator. Must be a constant String.
+SELECT concatWithSeparator('a', 'b') GROUP BY 'a';
+-- use-of-uninitialized-value
+SELECT concatWithSeparator('|', 'a', concatWithSeparator('|', CAST('a', 'LowCardinality(String)'))) GROUP BY 'a';
+SELECT concatWithSeparator('|', 'a', concatWithSeparator('|', CAST('x', 'LowCardinality(String)'))) GROUP BY 'a';
+-- should be const like for the query w/o GROUP BY
+select dumpColumnStructure('x') GROUP BY 'x';
+select dumpColumnStructure('x');

From d9c4972de4a44e9e79facefbb904263baf9be9ca Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 15 Feb 2024 09:58:04 +0100
Subject: [PATCH 15/92] Update tests references for GROUP BY const WITH TOTAL
 under analyzer

Now it preserves the original header in case of GROUP BY const, though
not for remote queries.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 ..._no_aggregates_and_constant_keys.reference |  2 +-
 ..._shard_no_aggregates_and_constant_keys.sql |  8 ++++++++
 .../00378_json_quote_64bit_integers.reference | 20 +++++++++----------
 ...01013_totals_without_aggregation.reference |  6 +++---
 .../02734_optimize_group_by.reference         |  8 ++++----
 5 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference
index 866dc4f1894..63b8a9d14fc 100644
--- a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference
+++ b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference
@@ -5,7 +5,7 @@
 11
 40
 
-0
+40
 41
 
 0
diff --git a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.sql b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.sql
index 5cc06a41132..e3634141613 100644
--- a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.sql
+++ b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.sql
@@ -10,6 +10,14 @@ select 43 AS z from remote('127.0.0.{2,3}', system.one) group by 42, 43, 44;
 select 11 AS z from (SELECT 2 UNION ALL SELECT 3) group by 42, 43, 44;
 
 select 40 as z from (select * from system.numbers limit 3) group by z WITH TOTALS;
+-- NOTE: non-analyzer preserves the original header (i.e. 41) for TOTALS in
+-- case of remote queries with GROUP BY some_requested_const and there were no
+-- aggregate functions, the query above. But everything else works in the same
+-- way, i.e.:
+--
+--     select 41 as z, count() from remote('127.0.0.{2,3}', system.one) group by z WITH TOTALS;
+--     select 41 as z from remote('127.0.0.{2,3}', system.one) group by 1 WITH TOTALS;
+--
 select 41 as z from remote('127.0.0.{2,3}', system.one) group by z WITH TOTALS;
 select count(), 42 AS z from remote('127.0.0.{2,3}', system.one) group by z WITH TOTALS;
 select 43 AS z from remote('127.0.0.{2,3}', system.one) group by 42, 43, 44 WITH TOTALS;
diff --git a/tests/queries/0_stateless/00378_json_quote_64bit_integers.reference b/tests/queries/0_stateless/00378_json_quote_64bit_integers.reference
index b8d51e5d078..5174c13a9e0 100644
--- a/tests/queries/0_stateless/00378_json_quote_64bit_integers.reference
+++ b/tests/queries/0_stateless/00378_json_quote_64bit_integers.reference
@@ -48,10 +48,10 @@
 	{
 		"i0": "0",
 		"u0": "0",
-		"ip": "0",
-		"in": "0",
-		"up": "0",
-		"arr": [],
+		"ip": "9223372036854775807",
+		"in": "-9223372036854775808",
+		"up": "18446744073709551615",
+		"arr": ["0"],
 		"tuple": ["0","0"]
 	},
 
@@ -119,7 +119,7 @@
 		["0", "0", "9223372036854775807", "-9223372036854775808", "18446744073709551615", ["0"], ["0","0"]]
 	],
 
-	"totals": ["0", "0", "0", "0", "0", [], ["0","0"]],
+	"totals": ["0", "0", "9223372036854775807", "-9223372036854775808", "18446744073709551615", ["0"], ["0","0"]],
 
 	"extremes":
 	{
@@ -180,10 +180,10 @@
 	{
 		"i0": 0,
 		"u0": 0,
-		"ip": 0,
-		"in": 0,
-		"up": 0,
-		"arr": [],
+		"ip": 9223372036854775807,
+		"in": -9223372036854775808,
+		"up": 18446744073709551615,
+		"arr": [0],
 		"tuple": [0,0]
 	},
 
@@ -251,7 +251,7 @@
 		[0, 0, 9223372036854775807, -9223372036854775808, 18446744073709551615, [0], [0,0]]
 	],
 
-	"totals": [0, 0, 0, 0, 0, [], [0,0]],
+	"totals": [0, 0, 9223372036854775807, -9223372036854775808, 18446744073709551615, [0], [0,0]],
 
 	"extremes":
 	{
diff --git a/tests/queries/0_stateless/01013_totals_without_aggregation.reference b/tests/queries/0_stateless/01013_totals_without_aggregation.reference
index a1f5c1aa914..6dddf22a467 100644
--- a/tests/queries/0_stateless/01013_totals_without_aggregation.reference
+++ b/tests/queries/0_stateless/01013_totals_without_aggregation.reference
@@ -1,7 +1,7 @@
 11
 
-0
+11
+12
 12
-0
 13
-0
+13
diff --git a/tests/queries/0_stateless/02734_optimize_group_by.reference b/tests/queries/0_stateless/02734_optimize_group_by.reference
index 3f5ef03cb61..f97f780a659 100644
--- a/tests/queries/0_stateless/02734_optimize_group_by.reference
+++ b/tests/queries/0_stateless/02734_optimize_group_by.reference
@@ -1,8 +1,8 @@
 a	b
 a	b
-a	b
-	b
-a	b
-a	b
+a	b
+a	b
+a	b
+a	b
 a	b
 a	b

From 8bbd0298b1a98fa28fb947b64709f343cf73cf05 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 22 Feb 2024 18:24:34 +0100
Subject: [PATCH 16/92] Allow to share const columns in GROUP BY keys

Fixes:

    select toNullable(os_name) AS os_name, count() from (SELECT CAST('iphone' AS Enum8('iphone' = 1, 'android' = 2)) AS os_name) group by os_name WITH TOTALS settings allow_experimental_analyzer=1
    Code: 10. DB::Exception: Received from localhost:9000. DB::Exception: Not found column __table1.os_name:  in block toNullable(__table1.os_name) Nullable(Enum8('iphone' = 1, 'android' = 2)) Nullable(size = 0, Int8(size = 0), UInt8(size = 0)), count() UInt64 UInt64(size = 0). (NOT_FOUND_COLUMN_IN_BLOCK)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Planner/PlannerExpressionAnalysis.cpp | 33 +++++++----------------
 1 file changed, 10 insertions(+), 23 deletions(-)

diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp
index 175a8cc9030..9b17e3b4b21 100644
--- a/src/Planner/PlannerExpressionAnalysis.cpp
+++ b/src/Planner/PlannerExpressionAnalysis.cpp
@@ -81,27 +81,6 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(const QueryTreeNodeP
     bool group_by_use_nulls = planner_context->getQueryContext()->getSettingsRef().group_by_use_nulls &&
         (query_node.isGroupByWithGroupingSets() || query_node.isGroupByWithRollup() || query_node.isGroupByWithCube());
 
-    auto add_aggregation_column = [&](const auto & expression_dag_node)
-    {
-        auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type;
-        /// Const columns are special, the return value of the
-        /// function depens on it, and there is a special case for
-        /// const columns with LowCardinality column (see
-        /// IFunctionOverloadResolver::getReturnType()), and return
-        /// values of the function had been already obtained by the
-        /// QueryAnalyzer::resolveFunction(), and if const-ness of
-        /// the column will be ignored here, the return value of
-        /// the function can be changed, and this is UB.
-        if (typeid_cast<const ColumnConst *>(expression_dag_node->column.get()))
-            available_columns_after_aggregation.emplace_back(nullptr, expression_type_after_aggregation, expression_dag_node->result_name + "__group_by");
-        else
-            available_columns_after_aggregation.emplace_back(nullptr, expression_type_after_aggregation, expression_dag_node->result_name);
-
-        aggregation_keys.push_back(expression_dag_node->result_name);
-        before_aggregation_actions->getOutputs().push_back(expression_dag_node);
-        before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name);
-    };
-
     if (query_node.hasGroupBy())
     {
         if (query_node.isGroupByWithGroupingSets())
@@ -129,7 +108,11 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(const QueryTreeNodeP
                         if (before_aggregation_actions_output_node_names.contains(expression_dag_node->result_name))
                             continue;
 
-                        add_aggregation_column(expression_dag_node);
+                        auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type;
+                        available_columns_after_aggregation.emplace_back(expression_dag_node->column, expression_type_after_aggregation, expression_dag_node->result_name);
+                        aggregation_keys.push_back(expression_dag_node->result_name);
+                        before_aggregation_actions->getOutputs().push_back(expression_dag_node);
+                        before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name);
                     }
                 }
             }
@@ -175,7 +158,11 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(const QueryTreeNodeP
                     if (before_aggregation_actions_output_node_names.contains(expression_dag_node->result_name))
                         continue;
 
-                    add_aggregation_column(expression_dag_node);
+                    auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type;
+                    available_columns_after_aggregation.emplace_back(expression_dag_node->column, expression_type_after_aggregation, expression_dag_node->result_name);
+                    aggregation_keys.push_back(expression_dag_node->result_name);
+                    before_aggregation_actions->getOutputs().push_back(expression_dag_node);
+                    before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name);
                 }
             }
         }

From b11b732e430a9b96206233fceb9d17f2cd690700 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 22 Feb 2024 18:28:29 +0100
Subject: [PATCH 17/92] Add a test from #60046

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../queries/0_stateless/02992_analyzer_group_by_const.reference | 1 +
 tests/queries/0_stateless/02992_analyzer_group_by_const.sql     | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/tests/queries/0_stateless/02992_analyzer_group_by_const.reference b/tests/queries/0_stateless/02992_analyzer_group_by_const.reference
index d1fab5a89a5..ff61ab0a515 100644
--- a/tests/queries/0_stateless/02992_analyzer_group_by_const.reference
+++ b/tests/queries/0_stateless/02992_analyzer_group_by_const.reference
@@ -3,3 +3,4 @@ a|a
 a|x
 String, Const(size = 1, String(size = 1))
 String, Const(size = 1, String(size = 1))
+5128475243952187658
diff --git a/tests/queries/0_stateless/02992_analyzer_group_by_const.sql b/tests/queries/0_stateless/02992_analyzer_group_by_const.sql
index a027678ec1f..f30a49887c7 100644
--- a/tests/queries/0_stateless/02992_analyzer_group_by_const.sql
+++ b/tests/queries/0_stateless/02992_analyzer_group_by_const.sql
@@ -8,3 +8,5 @@ SELECT concatWithSeparator('|', 'a', concatWithSeparator('|', CAST('x', 'LowCard
 -- should be const like for the query w/o GROUP BY
 select dumpColumnStructure('x') GROUP BY 'x';
 select dumpColumnStructure('x');
+-- from https://github.com/ClickHouse/ClickHouse/pull/60046
+SELECT cityHash64('limit', _CAST(materialize('World'), 'LowCardinality(String)')) FROM system.one GROUP BY GROUPING SETS ('limit');

From aa9b5741e15b1f9a5a09b208bfbe033133bc0e25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Tue, 12 Mar 2024 11:43:08 +0000
Subject: [PATCH 18/92] Improve tests

---
 .../integration/test_failed_mutations/test.py | 61 +++++++++++--------
 1 file changed, 35 insertions(+), 26 deletions(-)

diff --git a/tests/integration/test_failed_mutations/test.py b/tests/integration/test_failed_mutations/test.py
index 27bdcc3dd24..196a37ed627 100644
--- a/tests/integration/test_failed_mutations/test.py
+++ b/tests/integration/test_failed_mutations/test.py
@@ -19,10 +19,10 @@ node_no_backoff = cluster.add_instance(
     with_zookeeper=True,
 )
 
-REPLICATED_POSPONE_MUTATION_LOG = (
+REPLICATED_POSTPONE_MUTATION_LOG = (
     "According to exponential backoff policy, put aside this log entry"
 )
-POSPONE_MUTATION_LOG = (
+POSTPONE_MUTATION_LOG = (
     "According to exponential backoff policy, do not perform mutations for the part"
 )
 
@@ -56,21 +56,36 @@ def started_cluster():
 
 
 @pytest.mark.parametrize(
-    ("node"),
+    ("node, found_in_log"),
     [
-        (node_with_backoff),
+        (
+            node_with_backoff,
+            True,
+        ),
+        (
+            node_no_backoff,
+            False,
+        ),
     ],
 )
-def test_exponential_backoff_with_merge_tree(started_cluster, node):
+def test_exponential_backoff_with_merge_tree(started_cluster, node, found_in_log):
     prepare_cluster(False)
 
+    def check_logs():
+        if found_in_log:
+            assert node.wait_for_log_line(POSTPONE_MUTATION_LOG)
+            # Do not rotate the logs when we are checking the absence of a log message
+            node.rotate_logs()
+        else:
+            # Best effort, but when it fails, then the logs for sure contain the problematic message
+            assert not node.contains_in_log(POSTPONE_MUTATION_LOG)
+
     # Executing incorrect mutation.
     node.query(
         "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x FROM notexist_table) SETTINGS allow_nondeterministic_mutations=1"
     )
 
-    assert node.wait_for_log_line(POSPONE_MUTATION_LOG)
-    node.rotate_logs()
+    check_logs()
 
     node.query("KILL MUTATION WHERE table='test_mutations'")
     # Check that after kill new parts mutations are postponing.
@@ -78,7 +93,7 @@ def test_exponential_backoff_with_merge_tree(started_cluster, node):
         "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x FROM notexist_table) SETTINGS allow_nondeterministic_mutations=1"
     )
 
-    assert node.wait_for_log_line(POSPONE_MUTATION_LOG)
+    check_logs()
 
 
 def test_exponential_backoff_with_replicated_tree(started_cluster):
@@ -88,36 +103,30 @@ def test_exponential_backoff_with_replicated_tree(started_cluster):
         "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x FROM notexist_table) SETTINGS allow_nondeterministic_mutations=1"
     )
 
-    assert node_with_backoff.wait_for_log_line(REPLICATED_POSPONE_MUTATION_LOG)
-    assert not node_no_backoff.contains_in_log(REPLICATED_POSPONE_MUTATION_LOG)
+    assert node_with_backoff.wait_for_log_line(REPLICATED_POSTPONE_MUTATION_LOG)
+    assert not node_no_backoff.contains_in_log(REPLICATED_POSTPONE_MUTATION_LOG)
 
 
-@pytest.mark.parametrize(
-    ("node"),
-    [
-        (node_with_backoff),
-    ],
-)
-def test_exponential_backoff_create_dependent_table(started_cluster, node):
+def test_exponential_backoff_create_dependent_table(started_cluster):
     prepare_cluster(False)
 
     # Executing incorrect mutation.
-    node.query(
+    node_with_backoff.query(
         "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x  FROM dep_table) SETTINGS allow_nondeterministic_mutations=1"
     )
 
     # Creating dependent table for mutation.
-    node.query("CREATE TABLE dep_table(x UInt32) ENGINE MergeTree() ORDER BY x")
+    node_with_backoff.query("CREATE TABLE dep_table(x UInt32) ENGINE MergeTree() ORDER BY x")
 
     retry_count = 100
     no_unfinished_mutation = False
     for _ in range(0, retry_count):
-        if node.query("SELECT count() FROM system.mutations WHERE is_done=0") == "0\n":
+        if node_with_backoff.query("SELECT count() FROM system.mutations WHERE is_done=0") == "0\n":
             no_unfinished_mutation = True
             break
 
     assert no_unfinished_mutation
-    node.query("DROP TABLE IF EXISTS dep_table SYNC")
+    node_with_backoff   .query("DROP TABLE IF EXISTS dep_table SYNC")
 
 
 def test_exponential_backoff_setting_override(started_cluster):
@@ -133,7 +142,7 @@ def test_exponential_backoff_setting_override(started_cluster):
     node.query(
         "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x  FROM dep_table) SETTINGS allow_nondeterministic_mutations=1"
     )
-    assert not node.contains_in_log(POSPONE_MUTATION_LOG)
+    assert not node.contains_in_log(POSTPONE_MUTATION_LOG)
 
 
 @pytest.mark.parametrize(
@@ -152,14 +161,14 @@ def test_backoff_clickhouse_restart(started_cluster, replicated_table):
         "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x  FROM dep_table) SETTINGS allow_nondeterministic_mutations=1"
     )
     assert node.wait_for_log_line(
-        REPLICATED_POSPONE_MUTATION_LOG if replicated_table else POSPONE_MUTATION_LOG
+        REPLICATED_POSTPONE_MUTATION_LOG if replicated_table else POSTPONE_MUTATION_LOG
     )
 
     node.restart_clickhouse()
     node.rotate_logs()
 
     assert node.wait_for_log_line(
-        REPLICATED_POSPONE_MUTATION_LOG if replicated_table else POSPONE_MUTATION_LOG
+        REPLICATED_POSTPONE_MUTATION_LOG if replicated_table else POSTPONE_MUTATION_LOG
     )
 
 
@@ -181,7 +190,7 @@ def test_no_backoff_after_killing_mutation(started_cluster, replicated_table):
     # Executing correct mutation.
     node.query("ALTER TABLE test_mutations DELETE  WHERE x=1")
     assert node.wait_for_log_line(
-        REPLICATED_POSPONE_MUTATION_LOG if replicated_table else POSPONE_MUTATION_LOG
+        REPLICATED_POSTPONE_MUTATION_LOG if replicated_table else POSTPONE_MUTATION_LOG
     )
     mutation_ids = node.query("select mutation_id from system.mutations").split()
 
@@ -190,5 +199,5 @@ def test_no_backoff_after_killing_mutation(started_cluster, replicated_table):
     )
     node.rotate_logs()
     assert not node.contains_in_log(
-        REPLICATED_POSPONE_MUTATION_LOG if replicated_table else POSPONE_MUTATION_LOG
+        REPLICATED_POSTPONE_MUTATION_LOG if replicated_table else POSTPONE_MUTATION_LOG
     )

From e9ef18c3ac3424eab6fff0449740365d010f2eae Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Tue, 12 Mar 2024 11:54:56 +0000
Subject: [PATCH 19/92] Automatic style fix

---
 tests/integration/test_failed_mutations/test.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/tests/integration/test_failed_mutations/test.py b/tests/integration/test_failed_mutations/test.py
index 196a37ed627..c72f5264f03 100644
--- a/tests/integration/test_failed_mutations/test.py
+++ b/tests/integration/test_failed_mutations/test.py
@@ -116,17 +116,24 @@ def test_exponential_backoff_create_dependent_table(started_cluster):
     )
 
     # Creating dependent table for mutation.
-    node_with_backoff.query("CREATE TABLE dep_table(x UInt32) ENGINE MergeTree() ORDER BY x")
+    node_with_backoff.query(
+        "CREATE TABLE dep_table(x UInt32) ENGINE MergeTree() ORDER BY x"
+    )
 
     retry_count = 100
     no_unfinished_mutation = False
     for _ in range(0, retry_count):
-        if node_with_backoff.query("SELECT count() FROM system.mutations WHERE is_done=0") == "0\n":
+        if (
+            node_with_backoff.query(
+                "SELECT count() FROM system.mutations WHERE is_done=0"
+            )
+            == "0\n"
+        ):
             no_unfinished_mutation = True
             break
 
     assert no_unfinished_mutation
-    node_with_backoff   .query("DROP TABLE IF EXISTS dep_table SYNC")
+    node_with_backoff.query("DROP TABLE IF EXISTS dep_table SYNC")
 
 
 def test_exponential_backoff_setting_override(started_cluster):

From f63970769b21d9c5f882bb4eb43c71018d45fdf4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Tue, 12 Mar 2024 12:23:09 +0000
Subject: [PATCH 20/92] Make test less flaky

---
 .../test_merge_tree_load_parts/test.py        | 26 ++++++++++++++++---
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/tests/integration/test_merge_tree_load_parts/test.py b/tests/integration/test_merge_tree_load_parts/test.py
index 968225887ad..5d575fa95f5 100644
--- a/tests/integration/test_merge_tree_load_parts/test.py
+++ b/tests/integration/test_merge_tree_load_parts/test.py
@@ -156,15 +156,33 @@ def test_merge_tree_load_parts_corrupted(started_cluster):
     node1.query("SYSTEM WAIT LOADING PARTS mt_load_parts_2")
 
     def check_parts_loading(node, partition, loaded, failed, skipped):
+        # The whole test produces around 6-700 lines, so 2k is plenty enough.
+        # wait_for_log_line uses tail + grep, so the overhead is negligible
+        look_behind_lines = 2000
         for min_block, max_block in loaded:
             part_name = f"{partition}_{min_block}_{max_block}"
-            assert node.contains_in_log(f"Loading Active part {part_name}")
-            assert node.contains_in_log(f"Finished loading Active part {part_name}")
+            assert node.wait_for_log_line(
+                f"Loading Active part {part_name}", look_behind_lines=look_behind_lines
+            )
+            assert node.wait_for_log_line(
+                f"Finished loading Active part {part_name}",
+                look_behind_lines=look_behind_lines,
+            )
 
+        failed_part_names = []
+        # Let's wait until there is some information about all expected parts, and only
+        # check the absence of not expected log messages after all expected logs are present
         for min_block, max_block in failed:
             part_name = f"{partition}_{min_block}_{max_block}"
-            assert node.contains_in_log(f"Loading Active part {part_name}")
-            assert not node.contains_in_log(f"Finished loading Active part {part_name}")
+            failed_part_names.append(part_name)
+            assert node.wait_for_log_line(
+                f"Loading Active part {part_name}", look_behind_lines=look_behind_lines
+            )
+
+        for failed_part_name in failed_part_names:
+            assert not node.contains_in_log(
+                f"Finished loading Active part {failed_part_name}"
+            )
 
         for min_block, max_block in skipped:
             part_name = f"{partition}_{min_block}_{max_block}"

From df44ed173c7062ae3f9c8c8528623feea4c5e6f7 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Fri, 1 Mar 2024 17:07:08 +0100
Subject: [PATCH 21/92] Fix missed entries in system.part_log in case of fetch
 preferred over merges/mutations

The NEED_PREPARE do not call part_log_writer() before.

v2:
- Add a test for missed entries in system.part_log for merge
- Fix part_log_writer
- Add a test for missed entries in system.part_log for mutate

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../MergeTree/MergeFromLogEntryTask.cpp       | 42 ++++++++------
 .../MergeTree/MutateFromLogEntryTask.cpp      | 57 +++++++++++--------
 .../ReplicatedMergeMutateTaskBase.cpp         | 28 +++++----
 ...2_part_log_rmt_fetch_merge_error.reference | 10 ++++
 .../03002_part_log_rmt_fetch_merge_error.sql  | 34 +++++++++++
 ..._part_log_rmt_fetch_mutate_error.reference | 10 ++++
 .../03002_part_log_rmt_fetch_mutate_error.sql | 40 +++++++++++++
 7 files changed, 163 insertions(+), 58 deletions(-)
 create mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference
 create mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql
 create mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference
 create mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql

diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp
index ae6e398026d..ec4d98ab298 100644
--- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp
+++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp
@@ -47,13 +47,22 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
     int32_t metadata_version = metadata_snapshot->getMetadataVersion();
     const auto storage_settings_ptr = storage.getSettings();
 
+    stopwatch_ptr = std::make_unique<Stopwatch>();
+    auto part_log_writer = [this, stopwatch = *stopwatch_ptr](const ExecutionStatus & execution_status)
+    {
+        auto profile_counters_snapshot = std::make_shared<ProfileEvents::Counters::Snapshot>(profile_counters.getPartiallyAtomicSnapshot());
+        storage.writePartLog(
+            PartLogElement::MERGE_PARTS, execution_status, stopwatch.elapsed(),
+            entry.new_part_name, part, parts, merge_mutate_entry.get(), std::move(profile_counters_snapshot));
+    };
+
     if (storage_settings_ptr->always_fetch_merged_part)
     {
         LOG_INFO(log, "Will fetch part {} because setting 'always_fetch_merged_part' is true", entry.new_part_name);
         return PrepareResult{
             .prepared_successfully = false,
             .need_to_check_missing_part_in_fetch = true,
-            .part_log_writer = {}
+            .part_log_writer = part_log_writer,
         };
     }
 
@@ -68,7 +77,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
             return PrepareResult{
                 .prepared_successfully = false,
                 .need_to_check_missing_part_in_fetch = false,
-                .part_log_writer = {}
+                .part_log_writer = part_log_writer,
             };
     }
 
@@ -88,7 +97,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
             return PrepareResult{
                 .prepared_successfully = false,
                 .need_to_check_missing_part_in_fetch = true,
-                .part_log_writer = {}
+                .part_log_writer = part_log_writer,
             };
         }
     }
@@ -107,7 +116,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
             return PrepareResult{
                 .prepared_successfully = false,
                 .need_to_check_missing_part_in_fetch = true,
-                .part_log_writer = {}
+                .part_log_writer = part_log_writer,
             };
         }
 
@@ -127,7 +136,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
             return PrepareResult{
                 .prepared_successfully = false,
                 .need_to_check_missing_part_in_fetch = true,
-                .part_log_writer = {}
+                .part_log_writer = part_log_writer,
             };
         }
 
@@ -139,7 +148,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
             return PrepareResult{
                 .prepared_successfully = false,
                 .need_to_check_missing_part_in_fetch = false,
-                .part_log_writer = {}
+                .part_log_writer = part_log_writer,
             };
         }
 
@@ -167,7 +176,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
                 return PrepareResult{
                     .prepared_successfully = false,
                     .need_to_check_missing_part_in_fetch = false,
-                    .part_log_writer = {}
+                    .part_log_writer = part_log_writer,
                 };
             }
         }
@@ -227,7 +236,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
                 return PrepareResult{
                     .prepared_successfully = false,
                     .need_to_check_missing_part_in_fetch = false,
-                    .part_log_writer = {}
+                    .part_log_writer = part_log_writer,
                 };
             }
 
@@ -267,7 +276,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
                 return PrepareResult{
                     .prepared_successfully = false,
                     .need_to_check_missing_part_in_fetch = false,
-                    .part_log_writer = {}
+                    .part_log_writer = part_log_writer,
                 };
             }
             else if (storage.findReplicaHavingCoveringPart(entry.new_part_name, /* active */ false))
@@ -284,7 +293,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
                 return PrepareResult{
                     .prepared_successfully = false,
                     .need_to_check_missing_part_in_fetch = true,
-                    .part_log_writer = {}
+                    .part_log_writer = part_log_writer,
                 };
             }
             else
@@ -311,7 +320,6 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
         task_context);
 
     transaction_ptr = std::make_unique<MergeTreeData::Transaction>(storage, NO_TRANSACTION_RAW);
-    stopwatch_ptr = std::make_unique<Stopwatch>();
 
     merge_task = storage.merger_mutator.mergePartsToTemporaryPart(
             future_merged_part,
@@ -333,13 +341,11 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
     for (auto & item : future_merged_part->parts)
         priority.value += item->getBytesOnDisk();
 
-    return {true, true, [this, stopwatch = *stopwatch_ptr] (const ExecutionStatus & execution_status)
-    {
-        auto profile_counters_snapshot = std::make_shared<ProfileEvents::Counters::Snapshot>(profile_counters.getPartiallyAtomicSnapshot());
-        storage.writePartLog(
-            PartLogElement::MERGE_PARTS, execution_status, stopwatch.elapsed(),
-            entry.new_part_name, part, parts, merge_mutate_entry.get(), std::move(profile_counters_snapshot));
-    }};
+    return PrepareResult{
+        .prepared_successfully = true,
+        .need_to_check_missing_part_in_fetch = true,
+        .part_log_writer = part_log_writer,
+    };
 }
 
 
diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp
index a9ff687fe4d..62d7d4a7f2f 100644
--- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp
+++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp
@@ -20,6 +20,22 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
     const auto storage_settings_ptr = storage.getSettings();
     LOG_TRACE(log, "Executing log entry to mutate part {} to {}", source_part_name, entry.new_part_name);
 
+    new_part_info = MergeTreePartInfo::fromPartName(entry.new_part_name, storage.format_version);
+
+    future_mutated_part = std::make_shared<FutureMergedMutatedPart>();
+    future_mutated_part->name = entry.new_part_name;
+    future_mutated_part->uuid = entry.new_part_uuid;
+    future_mutated_part->part_info = new_part_info;
+
+    stopwatch_ptr = std::make_unique<Stopwatch>();
+    auto part_log_writer = [this](const ExecutionStatus & execution_status)
+    {
+        auto profile_counters_snapshot = std::make_shared<ProfileEvents::Counters::Snapshot>(profile_counters.getPartiallyAtomicSnapshot());
+        storage.writePartLog(
+            PartLogElement::MUTATE_PART, execution_status, stopwatch_ptr->elapsed(),
+            entry.new_part_name, new_part, future_mutated_part->parts, merge_mutate_entry.get(), std::move(profile_counters_snapshot));
+    };
+
     MergeTreeData::DataPartPtr source_part = storage.getActiveContainingPart(source_part_name);
     if (!source_part)
     {
@@ -29,10 +45,13 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
         return PrepareResult{
             .prepared_successfully = false,
             .need_to_check_missing_part_in_fetch = true,
-            .part_log_writer = {}
+            .part_log_writer = part_log_writer,
         };
     }
 
+    future_mutated_part->parts.push_back(source_part);
+    future_mutated_part->part_format = source_part->getFormat();
+
     if (source_part->name != source_part_name)
     {
         LOG_WARNING(log,
@@ -44,7 +63,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
         return PrepareResult{
             .prepared_successfully = false,
             .need_to_check_missing_part_in_fetch = true,
-            .part_log_writer = {}
+            .part_log_writer = part_log_writer,
         };
     }
 
@@ -63,7 +82,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
             return PrepareResult{
                 .prepared_successfully = false,
                 .need_to_check_missing_part_in_fetch = true,
-                .part_log_writer = {}
+                .part_log_writer = part_log_writer,
             };
         }
     }
@@ -84,13 +103,12 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
             return PrepareResult{
                 .prepared_successfully = false,
                 .need_to_check_missing_part_in_fetch = true,
-                .part_log_writer = {}
+                .part_log_writer = part_log_writer,
             };
 
         }
     }
 
-    new_part_info = MergeTreePartInfo::fromPartName(entry.new_part_name, storage.format_version);
     Strings mutation_ids;
     commands = std::make_shared<MutationCommands>(storage.queue.getMutationCommands(source_part, new_part_info.mutation, mutation_ids));
     LOG_TRACE(log, "Mutating part {} with mutation commands from {} mutations ({}): {}",
@@ -99,6 +117,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
     /// Once we mutate part, we must reserve space on the same disk, because mutations can possibly create hardlinks.
     /// Can throw an exception.
     reserved_space = storage.reserveSpace(estimated_space_for_result, source_part->getDataPartStorage());
+    future_mutated_part->updatePath(storage, reserved_space.get());
 
     table_lock_holder = storage.lockForShare(
             RWLockImpl::NO_QUERY, storage_settings_ptr->lock_acquire_timeout_for_background_operations);
@@ -106,14 +125,6 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
 
     transaction_ptr = std::make_unique<MergeTreeData::Transaction>(storage, NO_TRANSACTION_RAW);
 
-    future_mutated_part = std::make_shared<FutureMergedMutatedPart>();
-    future_mutated_part->name = entry.new_part_name;
-    future_mutated_part->uuid = entry.new_part_uuid;
-    future_mutated_part->parts.push_back(source_part);
-    future_mutated_part->part_info = new_part_info;
-    future_mutated_part->updatePath(storage, reserved_space.get());
-    future_mutated_part->part_format = source_part->getFormat();
-
     if (storage_settings_ptr->allow_remote_fs_zero_copy_replication)
     {
         if (auto disk = reserved_space->getDisk(); disk->supportZeroCopyReplication())
@@ -124,7 +135,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
                 return PrepareResult{
                     .prepared_successfully = false,
                     .need_to_check_missing_part_in_fetch = true,
-                    .part_log_writer = {}
+                    .part_log_writer = part_log_writer,
                 };
             }
 
@@ -163,7 +174,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
                 return PrepareResult{
                     .prepared_successfully = false,
                     .need_to_check_missing_part_in_fetch = false,
-                    .part_log_writer = {}
+                    .part_log_writer = part_log_writer,
                 };
             }
             else if (storage.findReplicaHavingCoveringPart(entry.new_part_name, /* active */ false))
@@ -182,7 +193,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
                 return PrepareResult{
                     .prepared_successfully = false,
                     .need_to_check_missing_part_in_fetch = true,
-                    .part_log_writer = {}
+                    .part_log_writer = part_log_writer,
                 };
             }
             else
@@ -201,8 +212,6 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
         future_mutated_part,
         task_context);
 
-    stopwatch_ptr = std::make_unique<Stopwatch>();
-
     mutate_task = storage.merger_mutator.mutatePartToTemporaryPart(
             future_mutated_part, metadata_snapshot, commands, merge_mutate_entry.get(),
             entry.create_time, task_context, NO_TRANSACTION_PTR, reserved_space, table_lock_holder);
@@ -211,13 +220,11 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
     for (auto & item : future_mutated_part->parts)
         priority.value += item->getBytesOnDisk();
 
-    return {true, true, [this] (const ExecutionStatus & execution_status)
-    {
-        auto profile_counters_snapshot = std::make_shared<ProfileEvents::Counters::Snapshot>(profile_counters.getPartiallyAtomicSnapshot());
-        storage.writePartLog(
-            PartLogElement::MUTATE_PART, execution_status, stopwatch_ptr->elapsed(),
-            entry.new_part_name, new_part, future_mutated_part->parts, merge_mutate_entry.get(), std::move(profile_counters_snapshot));
-    }};
+    return PrepareResult{
+        .prepared_successfully = true,
+        .need_to_check_missing_part_in_fetch = true,
+        .part_log_writer = part_log_writer,
+    };
 }
 
 
diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp
index 37d848ad095..2fc5238827d 100644
--- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp
@@ -164,8 +164,16 @@ bool ReplicatedMergeMutateTaskBase::executeImpl()
 
     auto execute_fetch = [&] (bool need_to_check_missing_part) -> bool
     {
-        if (storage.executeFetch(entry, need_to_check_missing_part))
-            return remove_processed_entry();
+        try
+        {
+            if (storage.executeFetch(entry, need_to_check_missing_part))
+                return remove_processed_entry();
+        }
+        catch (...)
+        {
+            part_log_writer(ExecutionStatus::fromCurrentException("", true));
+            throw;
+        }
 
         return false;
     };
@@ -205,8 +213,7 @@ bool ReplicatedMergeMutateTaskBase::executeImpl()
             }
             catch (...)
             {
-                if (part_log_writer)
-                    part_log_writer(ExecutionStatus::fromCurrentException("", true));
+                part_log_writer(ExecutionStatus::fromCurrentException("", true));
                 throw;
             }
 
@@ -214,17 +221,8 @@ bool ReplicatedMergeMutateTaskBase::executeImpl()
         }
         case State::NEED_FINALIZE :
         {
-            try
-            {
-                if (!finalize(part_log_writer))
-                    return execute_fetch(/* need_to_check_missing = */true);
-            }
-            catch (...)
-            {
-                if (part_log_writer)
-                    part_log_writer(ExecutionStatus::fromCurrentException("", true));
-                throw;
-            }
+            if (!finalize(part_log_writer))
+                return execute_fetch(/* need_to_check_missing = */true);
 
             return remove_processed_entry();
         }
diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference
new file mode 100644
index 00000000000..b19d389d8d0
--- /dev/null
+++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference
@@ -0,0 +1,10 @@
+before
+rmt_master	NewPart	0	1
+rmt_master	MergeParts	0	1
+rmt_slave	MergeParts	1	0
+rmt_slave	DownloadPart	0	1
+after
+rmt_master	NewPart	0	1
+rmt_master	MergeParts	0	1
+rmt_slave	MergeParts	1	0
+rmt_slave	DownloadPart	0	2
diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql
new file mode 100644
index 00000000000..a7381df5e11
--- /dev/null
+++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql
@@ -0,0 +1,34 @@
+-- Tags: no-replicated-database, no-parallel
+
+drop table if exists rmt_master;
+drop table if exists rmt_slave;
+
+create table rmt_master (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'master') order by key settings always_fetch_merged_part=0;
+-- always_fetch_merged_part=1, consider this table as a "slave"
+create table rmt_slave (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'slave') order by key settings always_fetch_merged_part=1;
+
+insert into rmt_master values (1);
+
+system sync replica rmt_master;
+system sync replica rmt_slave;
+system stop replicated sends rmt_master;
+optimize table rmt_master final settings alter_sync=1, optimize_throw_if_noop=1;
+
+select sleep(3) format Null;
+
+system flush logs;
+select 'before';
+select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
+
+system start replicated sends rmt_master;
+-- sleep few seconds to try rmt_slave to fetch the part and reflect this error
+-- in system.part_log
+select sleep(3) format Null;
+system sync replica rmt_slave;
+
+system flush logs;
+select 'after';
+select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
+
+drop table rmt_master;
+drop table rmt_slave;
diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference
new file mode 100644
index 00000000000..aac9e7527d1
--- /dev/null
+++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference
@@ -0,0 +1,10 @@
+before
+rmt_master	NewPart	0	1
+rmt_master	MutatePart	0	1
+rmt_slave	DownloadPart	0	1
+rmt_slave	MutatePart	1	0
+after
+rmt_master	NewPart	0	1
+rmt_master	MutatePart	0	1
+rmt_slave	DownloadPart	0	2
+rmt_slave	MutatePart	1	0
diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql
new file mode 100644
index 00000000000..f19cb9c73c9
--- /dev/null
+++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql
@@ -0,0 +1,40 @@
+-- Tags: no-replicated-database, no-parallel
+
+drop table if exists rmt_master;
+drop table if exists rmt_slave;
+
+create table rmt_master (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'master') order by tuple() settings always_fetch_merged_part=0;
+-- prefer_fetch_merged_part_*_threshold=0, consider this table as a "slave"
+create table rmt_slave (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'slave') order by tuple() settings prefer_fetch_merged_part_time_threshold=0, prefer_fetch_merged_part_size_threshold=0;
+
+insert into rmt_master values (1);
+
+system sync replica rmt_master;
+system sync replica rmt_slave;
+system stop replicated sends rmt_master;
+system stop pulling replication log rmt_slave;
+alter table rmt_master update key=key+100 where 1 settings alter_sync=1;
+
+-- first we need to make the rmt_master execute mutation so that it will have
+-- the part, and rmt_slave will consider it instead of performing mutation on
+-- it's own, otherwise prefer_fetch_merged_part_*_threshold will be simply ignored
+select sleep(3) format Null;
+system start pulling replication log rmt_slave;
+-- and sleep few more seconds to try rmt_slave to fetch the part and reflect
+-- this error in system.part_log
+select sleep(3) format Null;
+
+system flush logs;
+select 'before';
+select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
+
+system start replicated sends rmt_master;
+select sleep(3) format Null;
+system sync replica rmt_slave;
+
+system flush logs;
+select 'after';
+select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
+
+drop table rmt_master;
+drop table rmt_slave;

From fb0ad3eb1590a00d7e28554a6f7342ca63c3e214 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 12 Mar 2024 12:28:09 +0100
Subject: [PATCH 22/92] Disable retries for replicated fetches (they are
 handled by the engine itself)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Storages/MergeTree/DataPartsExchange.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp
index 9f95794cf50..6bb5ff5a4ab 100644
--- a/src/Storages/MergeTree/DataPartsExchange.cpp
+++ b/src/Storages/MergeTree/DataPartsExchange.cpp
@@ -487,10 +487,15 @@ std::pair<MergeTreeData::MutableDataPartPtr, scope_guard> Fetcher::fetchSelected
         creds.setPassword(password);
     }
 
+    ReadSettings read_settings = context->getReadSettings();
+    /// Disable retries for fetches, this will be done by the engine itself.
+    read_settings.http_max_tries = 1;
+
     auto in = BuilderRWBufferFromHTTP(uri)
                   .withConnectionGroup(HTTPConnectionGroupType::HTTP)
                   .withMethod(Poco::Net::HTTPRequest::HTTP_POST)
                   .withTimeouts(timeouts)
+                  .withSettings(read_settings)
                   .withDelayInit(false)
                   .create(creds);
 

From f71cc5d706ba63a692ddf0766691b51e2734d146 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 3 Mar 2024 12:04:22 +0100
Subject: [PATCH 23/92] Fix 02943_rmt_alter_metadata_merge_checksum_mismatch
 (filter out NO_REPLICA_HAS_PART errors)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../02943_rmt_alter_metadata_merge_checksum_mismatch.sh         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02943_rmt_alter_metadata_merge_checksum_mismatch.sh b/tests/queries/0_stateless/02943_rmt_alter_metadata_merge_checksum_mismatch.sh
index 431f59d7918..9c0c872eb06 100755
--- a/tests/queries/0_stateless/02943_rmt_alter_metadata_merge_checksum_mismatch.sh
+++ b/tests/queries/0_stateless/02943_rmt_alter_metadata_merge_checksum_mismatch.sh
@@ -95,4 +95,4 @@ trap '' EXIT
 
 $CLICKHOUSE_CLIENT -q "system flush logs"
 # check for error "Different number of files: 5 compressed (expected 3) and 2 uncompressed ones (expected 2). (CHECKSUM_DOESNT_MATCH)"
-$CLICKHOUSE_CLIENT -q "select part_name, merge_reason, event_type, errorCodeToName(error) from system.part_log where database = '$CLICKHOUSE_DATABASE' and error != 0 order by event_time_microseconds"
+$CLICKHOUSE_CLIENT -q "select part_name, merge_reason, event_type, errorCodeToName(error) from system.part_log where database = '$CLICKHOUSE_DATABASE' and error != 0 and errorCodeToName(error) != 'NO_REPLICA_HAS_PART' order by event_time_microseconds"

From e7d02654a7edf0194fc99d93ea3cff7cfa5e1fc8 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 12 Mar 2024 17:08:28 +0100
Subject: [PATCH 24/92] Fix 02265_column_ttl (filter out NO_REPLICA_HAS_PART
 errors)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/queries/0_stateless/02265_column_ttl.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02265_column_ttl.sql b/tests/queries/0_stateless/02265_column_ttl.sql
index 3ab175329bc..16ae2da2a2b 100644
--- a/tests/queries/0_stateless/02265_column_ttl.sql
+++ b/tests/queries/0_stateless/02265_column_ttl.sql
@@ -37,4 +37,4 @@ attach table ttl_02265;
 --
 optimize table ttl_02265 final;
 system flush logs;
-select * from system.part_log where database = currentDatabase() and table like 'ttl_02265%' and error != 0;
+select * from system.part_log where database = currentDatabase() and table like 'ttl_02265%' and error != 0 and errorCodeToName(error) != 'NO_REPLICA_HAS_PART';

From 0b277a55c78e6fcb840c543db6240ac000c09b74 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 15 Feb 2024 09:58:04 +0100
Subject: [PATCH 25/92] Fix tests with different results for GROUP BY const for
 analyzer

- 00757_enum_defaults - TOTALS
- 02699_polygons_sym_difference_rollup - TOTALS
- 02579_fill_empty_chunk - GROUP BY constX with arrayJoin(constX)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../0_stateless/00757_enum_defaults.reference      |  6 ------
 tests/queries/0_stateless/00757_enum_defaults.sql  |  3 ---
 .../00757_enum_defaults_const.reference            |  6 ++++++
 .../0_stateless/00757_enum_defaults_const.sql      |  3 +++
 .../00757_enum_defaults_const_analyzer.reference   |  6 ++++++
 .../00757_enum_defaults_const_analyzer.sql         |  3 +++
 .../queries/0_stateless/02579_fill_empty_chunk.sql |  1 +
 .../02579_fill_empty_chunk_analyzer.reference      | 14 ++++++++++++++
 .../02579_fill_empty_chunk_analyzer.sql            | 14 ++++++++++++++
 .../02699_polygons_sym_difference_rollup.reference |  3 ---
 .../02699_polygons_sym_difference_rollup.sql       |  1 -
 .../02699_polygons_sym_difference_total.reference  |  3 +++
 .../02699_polygons_sym_difference_total.sql        |  2 ++
 ...olygons_sym_difference_total_analyzer.reference |  3 +++
 ...2699_polygons_sym_difference_total_analyzer.sql |  2 ++
 15 files changed, 57 insertions(+), 13 deletions(-)
 create mode 100644 tests/queries/0_stateless/00757_enum_defaults_const.reference
 create mode 100644 tests/queries/0_stateless/00757_enum_defaults_const.sql
 create mode 100644 tests/queries/0_stateless/00757_enum_defaults_const_analyzer.reference
 create mode 100644 tests/queries/0_stateless/00757_enum_defaults_const_analyzer.sql
 create mode 100644 tests/queries/0_stateless/02579_fill_empty_chunk_analyzer.reference
 create mode 100644 tests/queries/0_stateless/02579_fill_empty_chunk_analyzer.sql
 create mode 100644 tests/queries/0_stateless/02699_polygons_sym_difference_total.reference
 create mode 100644 tests/queries/0_stateless/02699_polygons_sym_difference_total.sql
 create mode 100644 tests/queries/0_stateless/02699_polygons_sym_difference_total_analyzer.reference
 create mode 100644 tests/queries/0_stateless/02699_polygons_sym_difference_total_analyzer.sql

diff --git a/tests/queries/0_stateless/00757_enum_defaults.reference b/tests/queries/0_stateless/00757_enum_defaults.reference
index 6bf5cc85e87..4b15bd17e44 100644
--- a/tests/queries/0_stateless/00757_enum_defaults.reference
+++ b/tests/queries/0_stateless/00757_enum_defaults.reference
@@ -1,9 +1,3 @@
-iphone	1
-
-iphone	1
-iphone	1
-
-\N	1
 a
 b
 1
diff --git a/tests/queries/0_stateless/00757_enum_defaults.sql b/tests/queries/0_stateless/00757_enum_defaults.sql
index 71edc83abe2..45dc9b80cb7 100644
--- a/tests/queries/0_stateless/00757_enum_defaults.sql
+++ b/tests/queries/0_stateless/00757_enum_defaults.sql
@@ -1,6 +1,3 @@
-select os_name, count() from (SELECT CAST('iphone' AS Enum8('iphone' = 1, 'android' = 2)) AS os_name) group by os_name WITH TOTALS;
-select toNullable(os_name) AS os_name, count() from (SELECT CAST('iphone' AS Enum8('iphone' = 1, 'android' = 2)) AS os_name) group by os_name WITH TOTALS;
-
 DROP TABLE IF EXISTS auto_assign_enum;
 DROP TABLE IF EXISTS auto_assign_enum1;
 DROP TABLE IF EXISTS auto_assign_enum2;
diff --git a/tests/queries/0_stateless/00757_enum_defaults_const.reference b/tests/queries/0_stateless/00757_enum_defaults_const.reference
new file mode 100644
index 00000000000..56ead34ad3b
--- /dev/null
+++ b/tests/queries/0_stateless/00757_enum_defaults_const.reference
@@ -0,0 +1,6 @@
+iphone	1
+
+iphone	1
+iphone	1
+
+\N	1
diff --git a/tests/queries/0_stateless/00757_enum_defaults_const.sql b/tests/queries/0_stateless/00757_enum_defaults_const.sql
new file mode 100644
index 00000000000..64271a37473
--- /dev/null
+++ b/tests/queries/0_stateless/00757_enum_defaults_const.sql
@@ -0,0 +1,3 @@
+SET allow_experimental_analyzer=0;
+select os_name, count() from (SELECT CAST('iphone' AS Enum8('iphone' = 1, 'android' = 2)) AS os_name) group by os_name WITH TOTALS;
+select toNullable(os_name) AS os_name, count() from (SELECT CAST('iphone' AS Enum8('iphone' = 1, 'android' = 2)) AS os_name) group by os_name WITH TOTALS;
diff --git a/tests/queries/0_stateless/00757_enum_defaults_const_analyzer.reference b/tests/queries/0_stateless/00757_enum_defaults_const_analyzer.reference
new file mode 100644
index 00000000000..6895acffed1
--- /dev/null
+++ b/tests/queries/0_stateless/00757_enum_defaults_const_analyzer.reference
@@ -0,0 +1,6 @@
+iphone	1
+
+iphone	1
+iphone	1
+
+iphone	1
diff --git a/tests/queries/0_stateless/00757_enum_defaults_const_analyzer.sql b/tests/queries/0_stateless/00757_enum_defaults_const_analyzer.sql
new file mode 100644
index 00000000000..bf079539019
--- /dev/null
+++ b/tests/queries/0_stateless/00757_enum_defaults_const_analyzer.sql
@@ -0,0 +1,3 @@
+SET allow_experimental_analyzer=1;
+select os_name, count() from (SELECT CAST('iphone' AS Enum8('iphone' = 1, 'android' = 2)) AS os_name) group by os_name WITH TOTALS;
+select toNullable(os_name) AS os_name, count() from (SELECT CAST('iphone' AS Enum8('iphone' = 1, 'android' = 2)) AS os_name) group by os_name WITH TOTALS;
diff --git a/tests/queries/0_stateless/02579_fill_empty_chunk.sql b/tests/queries/0_stateless/02579_fill_empty_chunk.sql
index cbdbd7a9f84..30942b154c9 100644
--- a/tests/queries/0_stateless/02579_fill_empty_chunk.sql
+++ b/tests/queries/0_stateless/02579_fill_empty_chunk.sql
@@ -1,6 +1,7 @@
 -- this SELECT produces empty chunk in FillingTransform
 
 SET enable_positional_arguments = 0;
+SET allow_experimental_analyzer = 0;
 
 SELECT
     2 AS x,
diff --git a/tests/queries/0_stateless/02579_fill_empty_chunk_analyzer.reference b/tests/queries/0_stateless/02579_fill_empty_chunk_analyzer.reference
new file mode 100644
index 00000000000..b03cfa03815
--- /dev/null
+++ b/tests/queries/0_stateless/02579_fill_empty_chunk_analyzer.reference
@@ -0,0 +1,14 @@
+1	\N
+2	\N
+2	\N
+2	\N
+2	\N
+2	\N
+2	\N
+3	\N
+4	\N
+5	\N
+6	\N
+7	\N
+8	\N
+9	\N
diff --git a/tests/queries/0_stateless/02579_fill_empty_chunk_analyzer.sql b/tests/queries/0_stateless/02579_fill_empty_chunk_analyzer.sql
new file mode 100644
index 00000000000..8350173f443
--- /dev/null
+++ b/tests/queries/0_stateless/02579_fill_empty_chunk_analyzer.sql
@@ -0,0 +1,14 @@
+-- this SELECT produces empty chunk in FillingTransform
+
+SET enable_positional_arguments = 0;
+SET allow_experimental_analyzer = 1;
+
+-- With analyzer this special query has correct output
+SELECT
+    2 AS x,
+    arrayJoin([NULL, NULL, NULL])
+GROUP BY
+    GROUPING SETS (
+        (0),
+        ([NULL, NULL, NULL]))
+ORDER BY x ASC WITH FILL FROM 1 TO 10;
diff --git a/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.reference b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.reference
index 35c94347ac9..52af58fb67a 100644
--- a/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.reference
+++ b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.reference
@@ -6,6 +6,3 @@
 [[(2147483647,0),(10.0001,65535),(1,255),(1023,2147483646)]]	[[[(2147483647,0),(10.0001,65535),(1023,2147483646),(2147483647,0)]]]
 [[[(100.0001,1000.0001),(1000.0001,1.1920928955078125e-7),(20,-20),(20,20),(10,10),(-20,20),(100.0001,1000.0001)]]]
 [[[(100.0001,1000.0001),(1000.0001,1.1920928955078125e-7),(20,-20),(20,20),(10,10),(-20,20),(100.0001,1000.0001)]]]
-[(9223372036854775807,1.1754943508222875e-38)]	[[(1,1.0001)]]	\N	[]
-
-[(9223372036854775807,1.1754943508222875e-38)]	[]	\N	[]
diff --git a/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql
index 85307bec6e5..502cca20ab2 100644
--- a/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql
+++ b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql
@@ -2,4 +2,3 @@ SELECT polygonsSymDifferenceCartesian([[[(1., 1.)]] AS x], [x]) GROUP BY x WITH
 SELECT [[(2147483647, 0.), (10.0001, 65535), (1, 255), (1023, 2147483646)]], polygonsSymDifferenceCartesian([[[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]]], [[[(1000.0001, 10.0001)]]]) GROUP BY [[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]] WITH ROLLUP SETTINGS allow_experimental_analyzer=0;
 SELECT [[(2147483647, 0.), (10.0001, 65535), (1, 255), (1023, 2147483646)]], polygonsSymDifferenceCartesian([[[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]]], [[[(1000.0001, 10.0001)]]]) GROUP BY [[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]] WITH ROLLUP SETTINGS allow_experimental_analyzer=1;
 SELECT polygonsSymDifferenceCartesian([[[(100.0001, 1000.0001), (-20., 20.), (10., 10.), (20., 20.), (20., -20.), (1000.0001, 1.1920928955078125e-7)]],[[(0.0001, 100000000000000000000.)]] AS x],[x]) GROUP BY x WITH ROLLUP;
-SELECT [(9223372036854775807, 1.1754943508222875e-38)], x, NULL, polygonsSymDifferenceCartesian([[[(1.1754943508222875e-38, 1.1920928955078125e-7), (0.5, 0.5)]], [[(1.1754943508222875e-38, 1.1920928955078125e-7), (1.1754943508222875e-38, 1.1920928955078125e-7)], [(0., 1.0001)]], [[(1., 1.0001)]] AS x], [[[(3.4028234663852886e38, 0.9999)]]]) GROUP BY GROUPING SETS ((x)) WITH TOTALS
diff --git a/tests/queries/0_stateless/02699_polygons_sym_difference_total.reference b/tests/queries/0_stateless/02699_polygons_sym_difference_total.reference
new file mode 100644
index 00000000000..2b98454dc22
--- /dev/null
+++ b/tests/queries/0_stateless/02699_polygons_sym_difference_total.reference
@@ -0,0 +1,3 @@
+[(9223372036854775807,1.1754943508222875e-38)]	[[(1,1.0001)]]	\N	[]
+
+[(9223372036854775807,1.1754943508222875e-38)]	[]	\N	[]
diff --git a/tests/queries/0_stateless/02699_polygons_sym_difference_total.sql b/tests/queries/0_stateless/02699_polygons_sym_difference_total.sql
new file mode 100644
index 00000000000..0fac4b11320
--- /dev/null
+++ b/tests/queries/0_stateless/02699_polygons_sym_difference_total.sql
@@ -0,0 +1,2 @@
+SET allow_experimental_analyzer=0;
+SELECT [(9223372036854775807, 1.1754943508222875e-38)], x, NULL, polygonsSymDifferenceCartesian([[[(1.1754943508222875e-38, 1.1920928955078125e-7), (0.5, 0.5)]], [[(1.1754943508222875e-38, 1.1920928955078125e-7), (1.1754943508222875e-38, 1.1920928955078125e-7)], [(0., 1.0001)]], [[(1., 1.0001)]] AS x], [[[(3.4028234663852886e38, 0.9999)]]]) GROUP BY GROUPING SETS ((x)) WITH TOTALS
diff --git a/tests/queries/0_stateless/02699_polygons_sym_difference_total_analyzer.reference b/tests/queries/0_stateless/02699_polygons_sym_difference_total_analyzer.reference
new file mode 100644
index 00000000000..134420874b6
--- /dev/null
+++ b/tests/queries/0_stateless/02699_polygons_sym_difference_total_analyzer.reference
@@ -0,0 +1,3 @@
+[(9223372036854775807,1.1754943508222875e-38)]	[[(1,1.0001)]]	\N	[]
+
+[(9223372036854775807,1.1754943508222875e-38)]	[[(1,1.0001)]]	\N	[]
diff --git a/tests/queries/0_stateless/02699_polygons_sym_difference_total_analyzer.sql b/tests/queries/0_stateless/02699_polygons_sym_difference_total_analyzer.sql
new file mode 100644
index 00000000000..879e0e5297f
--- /dev/null
+++ b/tests/queries/0_stateless/02699_polygons_sym_difference_total_analyzer.sql
@@ -0,0 +1,2 @@
+SET allow_experimental_analyzer=1;
+SELECT [(9223372036854775807, 1.1754943508222875e-38)], x, NULL, polygonsSymDifferenceCartesian([[[(1.1754943508222875e-38, 1.1920928955078125e-7), (0.5, 0.5)]], [[(1.1754943508222875e-38, 1.1920928955078125e-7), (1.1754943508222875e-38, 1.1920928955078125e-7)], [(0., 1.0001)]], [[(1., 1.0001)]] AS x], [[[(3.4028234663852886e38, 0.9999)]]]) GROUP BY GROUPING SETS ((x)) WITH TOTALS

From 6ebc1e4522c7dc071e14c253fc47f87ab9b3d458 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Wed, 13 Mar 2024 17:09:01 +0800
Subject: [PATCH 26/92] fixed

---
 src/Interpreters/Aggregator.cpp | 130 +++++++++++---------------------
 1 file changed, 44 insertions(+), 86 deletions(-)

diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp
index 2902a590e01..28f8bf7972f 100644
--- a/src/Interpreters/Aggregator.cpp
+++ b/src/Interpreters/Aggregator.cpp
@@ -2051,7 +2051,7 @@ Aggregator::ConvertToBlockResVariant Aggregator::convertToBlockImplFinal(
     Table & data,
     Arena * arena,
     Arenas & aggregates_pools,
-    bool use_compiled_functions,
+    bool use_compiled_functions [[maybe_unused]],
     bool return_single_block) const
 {
     /// +1 for nullKeyData, if `data` doesn't have it - not a problem, just some memory for one excessive row will be preallocated
@@ -2091,46 +2091,28 @@ Aggregator::ConvertToBlockResVariant Aggregator::convertToBlockImplFinal(
     // should be invoked at least once, because null data might be the only content of the `data`
     init_out_cols();
 
-    if (return_single_block)
-    {
-        data.forEachValue(
-            [&](const auto & key, auto & mapped)
+    data.forEachValue(
+        [&](const auto & key, auto & mapped)
+        {
+            if (!out_cols.has_value())
+                init_out_cols();
+
+            const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes;
+            method.insertKeyIntoColumns(key, out_cols->raw_key_columns, key_sizes_ref);
+            places.emplace_back(mapped);
+
+            /// Mark the cell as destroyed so it will not be destroyed in destructor.
+            mapped = nullptr;
+
+            if (return_single_block && places.size() >= max_block_size)
             {
-                if (!out_cols.has_value())
-                    init_out_cols();
-
-                const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes;
-                method.insertKeyIntoColumns(key, out_cols->raw_key_columns, key_sizes_ref);
-                places.emplace_back(mapped);
-
-                /// Mark the cell as destroyed so it will not be destroyed in destructor.
-                mapped = nullptr;
-            });
-    }
-    else
-    {
-        data.forEachValue(
-            [&](const auto & key, auto & mapped)
-            {
-                if (!out_cols.has_value())
-                    init_out_cols();
-
-                const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes;
-                method.insertKeyIntoColumns(key, out_cols->raw_key_columns, key_sizes_ref);
-                places.emplace_back(mapped);
-
-                /// Mark the cell as destroyed so it will not be destroyed in destructor.
-                mapped = nullptr;
-
-                if (places.size() >= max_block_size)
-                {
-                    blocks.emplace_back(insertResultsIntoColumns(places, std::move(out_cols.value()), arena, has_null_key_data, use_compiled_functions));
-                    places.clear();
-                    out_cols.reset();
-                    has_null_key_data = false;
-                }
-            });
-    }
+                blocks.emplace_back(
+                    insertResultsIntoColumns(places, std::move(out_cols.value()), arena, has_null_key_data, use_compiled_functions));
+                places.clear();
+                out_cols.reset();
+                has_null_key_data = false;
+            }
+        });
 
     if (return_single_block)
     {
@@ -2183,53 +2165,29 @@ Aggregator::convertToBlockImplNotFinal(Method & method, Table & data, Arenas & a
 
     // should be invoked at least once, because null data might be the only content of the `data`
     init_out_cols();
-    if (return_single_block)
-    {
-        data.forEachValue(
-            [&](const auto & key, auto & mapped)
+    data.forEachValue(
+        [&](const auto & key, auto & mapped)
+        {
+            if (!out_cols.has_value())
+                init_out_cols();
+
+            const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes;
+            method.insertKeyIntoColumns(key, out_cols->raw_key_columns, key_sizes_ref);
+
+            /// reserved, so push_back does not throw exceptions
+            for (size_t i = 0; i < params.aggregates_size; ++i)
+                out_cols->aggregate_columns_data[i]->push_back(mapped + offsets_of_aggregate_states[i]);
+
+            mapped = nullptr;
+
+            ++rows_in_current_block;
+            if (return_single_block && rows_in_current_block >= max_block_size)
             {
-                if (!out_cols.has_value())
-                    init_out_cols();
-
-                const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes;
-                method.insertKeyIntoColumns(key, out_cols->raw_key_columns, key_sizes_ref);
-
-                /// reserved, so push_back does not throw exceptions
-                for (size_t i = 0; i < params.aggregates_size; ++i)
-                    out_cols->aggregate_columns_data[i]->push_back(mapped + offsets_of_aggregate_states[i]);
-
-                mapped = nullptr;
-
-                ++rows_in_current_block;
-            });
-    }
-    else
-    {
-        data.forEachValue(
-            [&](const auto & key, auto & mapped)
-            {
-                if (!out_cols.has_value())
-                    init_out_cols();
-
-                const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes;
-                method.insertKeyIntoColumns(key, out_cols->raw_key_columns, key_sizes_ref);
-
-                /// reserved, so push_back does not throw exceptions
-                for (size_t i = 0; i < params.aggregates_size; ++i)
-                    out_cols->aggregate_columns_data[i]->push_back(mapped + offsets_of_aggregate_states[i]);
-
-                mapped = nullptr;
-
-                ++rows_in_current_block;
-
-                if (rows_in_current_block >= max_block_size)
-                {
-                    res_blocks.emplace_back(finalizeBlock(params, getHeader(final), std::move(out_cols.value()), final, rows_in_current_block));
-                    out_cols.reset();
-                    rows_in_current_block = 0;
-                }
-            });
-    }
+                res_blocks.emplace_back(finalizeBlock(params, getHeader(final), std::move(out_cols.value()), final, rows_in_current_block));
+                out_cols.reset();
+                rows_in_current_block = 0;
+            }
+        });
 
     if (return_single_block)
     {

From 87a03ae219f376f4bae70fdc4f277b7d3b7e8e4f Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Wed, 13 Mar 2024 17:20:56 +0800
Subject: [PATCH 27/92] fixed

---
 src/Interpreters/Aggregator.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp
index 28f8bf7972f..c2a6634c55d 100644
--- a/src/Interpreters/Aggregator.cpp
+++ b/src/Interpreters/Aggregator.cpp
@@ -2104,7 +2104,7 @@ Aggregator::ConvertToBlockResVariant Aggregator::convertToBlockImplFinal(
             /// Mark the cell as destroyed so it will not be destroyed in destructor.
             mapped = nullptr;
 
-            if (return_single_block && places.size() >= max_block_size)
+            if (!return_single_block && places.size() >= max_block_size)
             {
                 blocks.emplace_back(
                     insertResultsIntoColumns(places, std::move(out_cols.value()), arena, has_null_key_data, use_compiled_functions));
@@ -2181,7 +2181,7 @@ Aggregator::convertToBlockImplNotFinal(Method & method, Table & data, Arenas & a
             mapped = nullptr;
 
             ++rows_in_current_block;
-            if (return_single_block && rows_in_current_block >= max_block_size)
+            if (!return_single_block && rows_in_current_block >= max_block_size)
             {
                 res_blocks.emplace_back(finalizeBlock(params, getHeader(final), std::move(out_cols.value()), final, rows_in_current_block));
                 out_cols.reset();
@@ -2199,7 +2199,6 @@ Aggregator::convertToBlockImplNotFinal(Method & method, Table & data, Arenas & a
             res_blocks.emplace_back(finalizeBlock(params, getHeader(final), std::move(out_cols).value(), final, rows_in_current_block));
         return res_blocks;
     }
-    return res_blocks;
 }
 
 void Aggregator::addSingleKeyToAggregateColumns(

From 8362c11c8ad4eaf34c8f51ced971025b6bd8fc46 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Wed, 13 Mar 2024 17:39:40 +0800
Subject: [PATCH 28/92] fixed

---
 src/Interpreters/Aggregator.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp
index c2a6634c55d..817664e4898 100644
--- a/src/Interpreters/Aggregator.cpp
+++ b/src/Interpreters/Aggregator.cpp
@@ -1247,7 +1247,7 @@ void Aggregator::executeAggregateInstructions(
     bool has_only_one_value_since_last_reset,
     bool no_more_keys,
     bool all_keys_are_const,
-    bool use_compiled_functions) const
+    bool use_compiled_functions [[maybe_unused]]) const
 {
 #if USE_EMBEDDED_COMPILER
     if (use_compiled_functions)
@@ -2513,7 +2513,8 @@ void NO_INLINE Aggregator::mergeDataNullKey(
 }
 
 template <typename Method, typename Table>
-void NO_INLINE Aggregator::mergeDataImpl(Table & table_dst, Table & table_src, Arena * arena, bool use_compiled_functions, bool prefetch) const
+void NO_INLINE Aggregator::mergeDataImpl(
+    Table & table_dst, Table & table_src, Arena * arena, bool use_compiled_functions [[maybe_unused]], bool prefetch) const
 {
     if constexpr (Method::low_cardinality_optimization || Method::one_key_nullable_optimization)
         mergeDataNullKey<Method, Table>(table_dst, table_src, arena);

From 4f446c566346b7cb1f33fc5fe1b7e892f90814f1 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Wed, 13 Mar 2024 19:22:16 +0800
Subject: [PATCH 29/92] fixed

---
 src/Interpreters/Aggregator.cpp | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp
index 817664e4898..0674f4f0ec0 100644
--- a/src/Interpreters/Aggregator.cpp
+++ b/src/Interpreters/Aggregator.cpp
@@ -1862,19 +1862,13 @@ Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Are
         return {finalizeBlock(params, getHeader(final), std::move(out_cols), final, rows)};
     }
     ConvertToBlockResVariant res;
+    bool use_compiled_functions = false;
     if (final)
     {
 #if USE_EMBEDDED_COMPILER
-        if (compiled_aggregate_functions_holder)
-        {
-            static constexpr bool use_compiled_functions = !Method::low_cardinality_optimization;
-            res = convertToBlockImplFinal<Method>(method, data, arena, aggregates_pools, use_compiled_functions, true);
-        }
-        else
+        use_compiled_functions = compiled_aggregate_functions_holder != nullptr && !Method::low_cardinality_optimization;
 #endif
-        {
-            res = convertToBlockImplFinal<Method>(method, data, arena, aggregates_pools, false, return_single_block);
-        }
+        res = convertToBlockImplFinal<Method>(method, data, arena, aggregates_pools, use_compiled_functions, return_single_block);
     }
     else
     {

From 2982f3b369eeb5a9ef7dce2790c4e66c68c7f41b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Wed, 13 Mar 2024 19:11:40 +0100
Subject: [PATCH 30/92] Fix stress test sanitizer report

---
 docker/test/stateless/stress_tests.lib | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/stateless/stress_tests.lib b/docker/test/stateless/stress_tests.lib
index c0fc32ab718..b69f1d28fcf 100644
--- a/docker/test/stateless/stress_tests.lib
+++ b/docker/test/stateless/stress_tests.lib
@@ -215,7 +215,7 @@ function check_server_start()
 function check_logs_for_critical_errors()
 {
     # Sanitizer asserts
-    sed -n '/WARNING:.*anitizer/,/^$/p' >> /test_output/tmp
+    sed -n '/WARNING:.*anitizer/,/^$/p' /var/log/clickhouse-server/stderr.log >> /test_output/tmp
     rg -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \
         && echo -e "Sanitizer assert (in stderr.log)$FAIL$(head_escaped /test_output/tmp)" >> /test_output/test_results.tsv \
         || echo -e "No sanitizer asserts$OK" >> /test_output/test_results.tsv

From f63368d83d5e7ed1168bed2ad112d06f663614c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Wed, 13 Mar 2024 19:12:27 +0100
Subject: [PATCH 31/92] Separate stdout and stderr in fuzzer report

---
 docker/test/fuzzer/run-fuzzer.sh | 8 +++++---
 tests/ci/ast_fuzzer_check.py     | 8 ++++++++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index 63471c288dd..1c41d299ac3 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -174,7 +174,7 @@ function fuzz
     mkdir -p /var/run/clickhouse-server
 
     # NOTE: we use process substitution here to preserve keep $! as a pid of clickhouse-server
-    clickhouse-server --config-file db/config.xml --pid-file /var/run/clickhouse-server/clickhouse-server.pid -- --path db > server.log 2>&1 &
+    clickhouse-server --config-file db/config.xml --pid-file /var/run/clickhouse-server/clickhouse-server.pid -- --path db > server.log 2>>stderr.log &
     server_pid=$!
 
     kill -0 $server_pid
@@ -303,7 +303,7 @@ quit
     if [ "$server_died" == 1 ]
     then
         # The server has died.
-        if ! rg --text -o 'Received signal.*|Logical error.*|Assertion.*failed|Failed assertion.*|.*runtime error: .*|.*is located.*|(SUMMARY|ERROR): [a-zA-Z]+Sanitizer:.*|.*_LIBCPP_ASSERT.*|.*Child process was terminated by signal 9.*' server.log > description.txt
+        if ! rg --text -o 'Received signal.*|Logical error.*|Assertion.*failed|Failed assertion.*|.*runtime error: .*|.*is located.*|(SUMMARY|ERROR): [a-zA-Z]+Sanitizer:.*|.*_LIBCPP_ASSERT.*|.*Child process was terminated by signal 9.*' server.log stderr.log > description.txt
         then
             echo "Lost connection to server. See the logs." > description.txt
         fi
@@ -392,7 +392,7 @@ if [ -f core.zst ]; then
 fi
 
 # Keep all the lines in the paragraphs containing <Fatal> that either contain <Fatal> or don't start with 20... (year)
-sed -n '/<Fatal>/,/^$/p' server.log | awk '/<Fatal>/ || !/^20/' > fatal.log ||:
+sed -n '/<Fatal>/,/^$/p' server.log stderr.log | awk '/<Fatal>/ || !/^20/' > fatal.log ||:
 FATAL_LINK=''
 if [ -s fatal.log ]; then
     FATAL_LINK='<a href="fatal.log">fatal.log</a>'
@@ -401,6 +401,7 @@ fi
 dmesg -T > dmesg.log ||:
 
 zstd --threads=0 --rm server.log
+zstd --threads=0 --rm stderr.log
 zstd --threads=0 --rm fuzzer.log
 
 cat > report.html <<EOF ||:
@@ -427,6 +428,7 @@ p.links a { padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-s
   <a href="run.log">run.log</a>
   <a href="fuzzer.log.zst">fuzzer.log.zst</a>
   <a href="server.log.zst">server.log.zst</a>
+  <a href="stderr.log.zst">stderr.log.zst</a>
   <a href="main.log">main.log</a>
   <a href="dmesg.log">dmesg.log</a>
   ${CORE_LINK}
diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py
index 1ecf805cadc..c24a67a7a6e 100644
--- a/tests/ci/ast_fuzzer_check.py
+++ b/tests/ci/ast_fuzzer_check.py
@@ -126,6 +126,14 @@ def main():
         if not_compressed_server_log_path.exists():
             paths["server.log"] = not_compressed_server_log_path
 
+    compressed_stderr_log_path = workspace_path / "stderr.log.zst"
+    if compressed_stderr_log_path.exists():
+        paths["stderr.log.zst"] = compressed_stderr_log_path
+    else:
+        not_compressed_stderr_log_path = workspace_path / "stderr.log"
+        if compressed_stderr_log_path.exists():
+            paths["stderr.log"] = not_compressed_stderr_log_path
+
     # Same idea but with the fuzzer log
     compressed_fuzzer_log_path = workspace_path / "fuzzer.log.zst"
     if compressed_fuzzer_log_path.exists():

From 6b26d05346f6d6b9879cde2a12c2ca8ddc78da4e Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Thu, 14 Mar 2024 08:55:59 +0800
Subject: [PATCH 32/92] fixed

---
 src/Interpreters/AggregatedDataVariants.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Interpreters/AggregatedDataVariants.cpp b/src/Interpreters/AggregatedDataVariants.cpp
index 9b67bdf1281..87cfdda5948 100644
--- a/src/Interpreters/AggregatedDataVariants.cpp
+++ b/src/Interpreters/AggregatedDataVariants.cpp
@@ -18,7 +18,6 @@ namespace ErrorCodes
 }
 using ColumnsHashing::HashMethodContext;
 using ColumnsHashing::HashMethodContextPtr;
-using ColumnsHashing::LastElementCacheStats;
 
 AggregatedDataVariants::AggregatedDataVariants() : aggregates_pools(1, std::make_shared<Arena>()), aggregates_pool(aggregates_pools.back().get()) {}
 

From 0e9c64899a5fe86f08d51ada8d5a779e516735fd Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 14 Mar 2024 10:31:21 +0100
Subject: [PATCH 33/92] tests: do not rely on GROUP BY const behaviour in
 01056_predicate_optimizer_bugs

The behaviour of GROUP BY const in analyzer is slightly different, it
still preserves the const property from the subqueries, and that's why
it simply do not execute filters after GROUP BY, but simply one time for
the const.

The initial bug was about predicate pushdown not about GROUP BY const,
so I will update the test.

I've also tested the initial bug with fiddle and the new test is
idential:
- 19.17 (without the fix) - https://fiddle.clickhouse.com/312a48f4-6a5b-411d-a4bb-4b1a757effaf
- 20.2 (with the fix) - https://fiddle.clickhouse.com/4e1ad2e5-1c03-4ce7-9ac3-e878ccfae83a

Refs: https://github.com/ClickHouse/ClickHouse/pull/8503
Refs: https://github.com/ClickHouse/ClickHouse/issues/5682

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../0_stateless/01056_predicate_optimizer_bugs.reference    | 6 +++---
 .../queries/0_stateless/01056_predicate_optimizer_bugs.sql  | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference
index dd52fd1c1b2..e8a9b79bf7c 100644
--- a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference
+++ b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference
@@ -36,9 +36,9 @@ FROM
     FROM
     (
         SELECT
-            1 AS co,
-            2 AS co2,
-            3 AS co3
+            dummy + 1 AS co,
+            dummy + 2 AS co2,
+            dummy + 3 AS co3
     )
     GROUP BY
         co,
diff --git a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.sql b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.sql
index f237768bc4c..13d90b9f43e 100644
--- a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.sql
+++ b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.sql
@@ -8,8 +8,8 @@ EXPLAIN SYNTAX SELECT k, v, d, i FROM (SELECT t.1 AS k, t.2 AS v, runningDiffere
 SELECT k, v, d, i FROM (SELECT t.1 AS k, t.2 AS v, runningDifference(v) AS d, runningDifference(cityHash64(t.1)) AS i FROM (   SELECT arrayJoin([('a', 1), ('a', 2), ('a', 3), ('b', 11), ('b', 13), ('b', 15)]) AS t)) WHERE i = 0;
 
 -- https://github.com/ClickHouse/ClickHouse/issues/5682
-EXPLAIN SYNTAX SELECT co,co2,co3,num FROM ( SELECT co,co2,co3,count() AS num FROM ( SELECT 1 AS co,2 AS co2 ,3 AS co3 ) GROUP BY cube (co,co2,co3) ) WHERE co!=0 AND co2 !=2;
-SELECT co,co2,co3,num FROM ( SELECT co,co2,co3,count() AS num FROM ( SELECT 1 AS co,2 AS co2 ,3 AS co3 ) GROUP BY cube (co,co2,co3) ) WHERE co!=0 AND co2 !=2;
+EXPLAIN SYNTAX SELECT co,co2,co3,num FROM ( SELECT co,co2,co3,count() AS num FROM (SELECT dummy+1 AS co,dummy+2 AS co2 ,dummy+3 AS co3) GROUP BY cube (co,co2,co3) ) WHERE co!=0 AND co2 !=2;
+SELECT co,co2,co3,num FROM ( SELECT co,co2,co3,count() AS num FROM (SELECT dummy+1 AS co,dummy+2 AS co2 ,dummy+3 AS co3) GROUP BY cube (co,co2,co3) ) WHERE co!=0 AND co2 !=2;
 
 -- https://github.com/ClickHouse/ClickHouse/issues/6734
 EXPLAIN SYNTAX SELECT name FROM ( SELECT name FROM system.settings ) ANY INNER JOIN ( SELECT name FROM system.settings ) USING (name) WHERE name = 'enable_optimize_predicate_expression';

From 1ac247ed82e583a47c30ee172a856c272f25d00d Mon Sep 17 00:00:00 2001
From: Alexander Sapin <alesapin@gmail.com>
Date: Thu, 14 Mar 2024 12:13:33 +0100
Subject: [PATCH 34/92] Tiny improvement for replication.lib

---
 tests/queries/0_stateless/replication.lib | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/replication.lib b/tests/queries/0_stateless/replication.lib
index 143332d9974..2e21f351d2a 100755
--- a/tests/queries/0_stateless/replication.lib
+++ b/tests/queries/0_stateless/replication.lib
@@ -26,12 +26,15 @@ function try_sync_replicas()
         done
     done
 
-    i=0
     for t in "${tables_arr[@]}"
     do
         # Do not start new merges (it can make SYNC a bit faster)
         $CLICKHOUSE_CLIENT -q "ALTER TABLE $t MODIFY SETTING max_replicated_merges_in_queue=0"
+    done
 
+    i=0
+    for t in "${tables_arr[@]}"
+    do
         $CLICKHOUSE_CLIENT --receive_timeout $time_left -q "SYSTEM SYNC REPLICA $t STRICT" || ($CLICKHOUSE_CLIENT -q \
             "select 'sync failed, queue:', * from system.replication_queue where database=currentDatabase() and table='$t' order by database, table, node_name" && exit 1) &
         pids[${i}]=$!

From 02ae7e54a71881fc58956561d9c8b7b9013438ea Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 14 Mar 2024 12:28:02 +0100
Subject: [PATCH 35/92] Fix bugfix check (due to "unknown object storage type:
 azure")

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docker/test/stateless/run.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index dc181339786..2a91b235df8 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -61,6 +61,18 @@ if [[ -n "$BUGFIX_VALIDATE_CHECK" ]] && [[ "$BUGFIX_VALIDATE_CHECK" -eq 1 ]]; th
     rm /etc/clickhouse-server/users.d/s3_cache_new.xml
     rm /etc/clickhouse-server/config.d/zero_copy_destructive_operations.xml
 
+    #todo: remove these after 24.3 released.
+    sudo cat /etc/clickhouse-server/config.d/azure_storage_conf.xml \
+      | sed "s|<object_storage_type>azure|<object_storage_type>azure_blob_storage|" \
+      > /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp
+    sudo mv /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp /etc/clickhouse-server/config.d/azure_storage_conf.xml
+
+    #todo: remove these after 24.3 released.
+    sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \
+      | sed "s|<object_storage_type>local|<object_storage_type>local_blob_storage|" \
+      > /etc/clickhouse-server/config.d/storage_conf.xml.tmp
+    sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml
+
     function remove_keeper_config()
     {
         sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \

From b064cff245464f972033efb535d3b69273770833 Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Thu, 14 Mar 2024 16:57:17 +0000
Subject: [PATCH 36/92] Fix test_failed_async_inserts

---
 tests/integration/test_failed_async_inserts/configs/config.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/test_failed_async_inserts/configs/config.xml b/tests/integration/test_failed_async_inserts/configs/config.xml
index 038c0792b44..7daf4bd31a1 100644
--- a/tests/integration/test_failed_async_inserts/configs/config.xml
+++ b/tests/integration/test_failed_async_inserts/configs/config.xml
@@ -1,3 +1,4 @@
 <clickhouse>
     <max_server_memory_usage>1000</max_server_memory_usage>
+    <page_cache_size>0</page_cache_size>
 </clickhouse>

From 23169117aefc65c54e29b38629948ee4157aaf9c Mon Sep 17 00:00:00 2001
From: tomershafir <tomer.shafir8@gmail.com>
Date: Fri, 15 Mar 2024 00:02:09 +0200
Subject: [PATCH 37/92] thread-fuzzer: randomize sleep time

---
 docker/test/stateless/run.sh             | 10 +++----
 docker/test/stress/run.sh                | 10 +++----
 src/Common/ThreadFuzzer.cpp              | 35 ++++++++++++------------
 src/Common/ThreadFuzzer.h                | 10 +++----
 src/Common/examples/chaos_sanitizer.cpp  |  2 +-
 utils/keeper-overload/keeper-overload.py | 10 +++----
 6 files changed, 38 insertions(+), 39 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index dc181339786..cfa3c806212 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -77,7 +77,7 @@ fi
 if [ "$NUM_TRIES" -gt "1" ]; then
     export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
     export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
-    export THREAD_FUZZER_SLEEP_TIME_US=100000
+    export THREAD_FUZZER_SLEEP_TIME_US_MAX=100000
 
     export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
     export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
@@ -88,10 +88,10 @@ if [ "$NUM_TRIES" -gt "1" ]; then
     export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
     export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
     export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
-    export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
-    export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
-    export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
-    export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
+    export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000
+    export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000
+    export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX=10000
+    export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX=10000
 
     mkdir -p /var/run/clickhouse-server
     # simplest way to forward env variables to server
diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh
index 621a6ced7f6..ea7e3aece1d 100644
--- a/docker/test/stress/run.sh
+++ b/docker/test/stress/run.sh
@@ -27,7 +27,7 @@ install_packages package_folder
 # and find more potential issues.
 export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
 export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
-export THREAD_FUZZER_SLEEP_TIME_US=100000
+export THREAD_FUZZER_SLEEP_TIME_US_MAX=100000
 
 export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
 export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
@@ -38,11 +38,11 @@ export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
 export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
 export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
 export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
-export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
+export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000
 
-export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
-export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
-export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
+export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000
+export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX=10000
+export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX=10000
 
 export THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY=0.01
 export THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY=0.01
diff --git a/src/Common/ThreadFuzzer.cpp b/src/Common/ThreadFuzzer.cpp
index 1d944f4a458..9f9ec4fa356 100644
--- a/src/Common/ThreadFuzzer.cpp
+++ b/src/Common/ThreadFuzzer.cpp
@@ -86,12 +86,12 @@ static std::atomic<int> num_cpus = 0;
         static std::atomic<double> NAME##_before_yield_probability = 0; \
         static std::atomic<double> NAME##_before_migrate_probability = 0; \
         static std::atomic<double> NAME##_before_sleep_probability = 0; \
-        static std::atomic<double> NAME##_before_sleep_time_us = 0; \
+        static std::atomic<double> NAME##_before_sleep_time_us_max = 0; \
 \
         static std::atomic<double> NAME##_after_yield_probability = 0; \
         static std::atomic<double> NAME##_after_migrate_probability = 0; \
         static std::atomic<double> NAME##_after_sleep_probability = 0; \
-        static std::atomic<double> NAME##_after_sleep_time_us = 0;
+        static std::atomic<double> NAME##_after_sleep_time_us_max = 0;
 
 FOR_EACH_WRAPPED_FUNCTION(DEFINE_WRAPPER_PARAMS)
 
@@ -110,7 +110,7 @@ void ThreadFuzzer::initConfiguration()
     initFromEnv(yield_probability, "THREAD_FUZZER_YIELD_PROBABILITY");
     initFromEnv(migrate_probability, "THREAD_FUZZER_MIGRATE_PROBABILITY");
     initFromEnv(sleep_probability, "THREAD_FUZZER_SLEEP_PROBABILITY");
-    initFromEnv(sleep_time_us, "THREAD_FUZZER_SLEEP_TIME_US");
+    initFromEnv(sleep_time_us_max, "THREAD_FUZZER_SLEEP_TIME_US_MAX");
     initFromEnv(explicit_sleep_probability, "THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY");
     initFromEnv(explicit_memory_exception_probability, "THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY");
 
@@ -119,13 +119,12 @@ void ThreadFuzzer::initConfiguration()
         initFromEnv(NAME##_before_yield_probability, "THREAD_FUZZER_" #NAME "_BEFORE_YIELD_PROBABILITY"); \
         initFromEnv(NAME##_before_migrate_probability, "THREAD_FUZZER_" #NAME "_BEFORE_MIGRATE_PROBABILITY"); \
         initFromEnv(NAME##_before_sleep_probability, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_PROBABILITY"); \
-        initFromEnv(NAME##_before_sleep_time_us, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_TIME_US"); \
+        initFromEnv(NAME##_before_sleep_time_us_max, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_TIME_US_MAX"); \
 \
         initFromEnv(NAME##_after_yield_probability, "THREAD_FUZZER_" #NAME "_AFTER_YIELD_PROBABILITY"); \
         initFromEnv(NAME##_after_migrate_probability, "THREAD_FUZZER_" #NAME "_AFTER_MIGRATE_PROBABILITY"); \
         initFromEnv(NAME##_after_sleep_probability, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_PROBABILITY"); \
-        initFromEnv(NAME##_after_sleep_time_us, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US");
-
+        initFromEnv(NAME##_after_sleep_time_us_max, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US_MAX");
     FOR_EACH_WRAPPED_FUNCTION(INIT_WRAPPER_PARAMS)
 
 #    undef INIT_WRAPPER_PARAMS
@@ -146,7 +145,7 @@ bool ThreadFuzzer::isEffective() const
             return true; \
         if (NAME##_before_sleep_probability.load(std::memory_order_relaxed) > 0.0) \
             return true; \
-        if (NAME##_before_sleep_time_us.load(std::memory_order_relaxed) > 0.0) \
+        if (NAME##_before_sleep_time_us_max.load(std::memory_order_relaxed) > 0.0) \
             return true; \
 \
         if (NAME##_after_yield_probability.load(std::memory_order_relaxed) > 0.0) \
@@ -155,7 +154,7 @@ bool ThreadFuzzer::isEffective() const
             return true; \
         if (NAME##_after_sleep_probability.load(std::memory_order_relaxed) > 0.0) \
             return true; \
-        if (NAME##_after_sleep_time_us.load(std::memory_order_relaxed) > 0.0) \
+        if (NAME##_after_sleep_time_us_max.load(std::memory_order_relaxed) > 0.0) \
             return true;
 
     FOR_EACH_WRAPPED_FUNCTION(CHECK_WRAPPER_PARAMS)
@@ -166,7 +165,7 @@ bool ThreadFuzzer::isEffective() const
     return cpu_time_period_us != 0
         && (yield_probability > 0
             || migrate_probability > 0
-            || (sleep_probability > 0 && sleep_time_us > 0));
+            || (sleep_probability > 0 && sleep_time_us_max > 0));
 }
 
 void ThreadFuzzer::stop()
@@ -190,7 +189,7 @@ static void injectionImpl(
     double yield_probability,
     double migrate_probability,
     double sleep_probability,
-    double sleep_time_us)
+    double sleep_time_us_max)
 {
     DENY_ALLOCATIONS_IN_SCOPE;
     if (!ThreadFuzzer::isStarted())
@@ -221,10 +220,10 @@ static void injectionImpl(
 #endif
 
     if (sleep_probability > 0
-        && sleep_time_us > 0
+        && sleep_time_us_max > 0
         && std::bernoulli_distribution(sleep_probability)(thread_local_rng))
     {
-        sleepForNanoseconds(static_cast<uint64_t>(sleep_time_us * 1000));
+        sleepForNanoseconds((thread_local_rng() % static_cast<uint64_t>(sleep_time_us_max)) * 1000); /*may sleep(0)*/
     }
 }
 
@@ -232,19 +231,19 @@ static ALWAYS_INLINE void injection(
     double yield_probability,
     double migrate_probability,
     double sleep_probability,
-    double sleep_time_us)
+    double sleep_time_us_max)
 {
     DENY_ALLOCATIONS_IN_SCOPE;
     if (!ThreadFuzzer::isStarted())
         return;
 
-    injectionImpl(yield_probability, migrate_probability, sleep_probability, sleep_time_us);
+    injectionImpl(yield_probability, migrate_probability, sleep_probability, sleep_time_us_max);
 }
 
 void ThreadFuzzer::maybeInjectSleep()
 {
     auto & fuzzer = ThreadFuzzer::instance();
-    injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.explicit_sleep_probability, fuzzer.sleep_time_us);
+    injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.explicit_sleep_probability, fuzzer.sleep_time_us_max);
 }
 
 /// Sometimes maybeInjectSleep() is not enough and we need to inject an exception.
@@ -265,7 +264,7 @@ void ThreadFuzzer::signalHandler(int)
     DENY_ALLOCATIONS_IN_SCOPE;
     auto saved_errno = errno;
     auto & fuzzer = ThreadFuzzer::instance();
-    injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.sleep_probability, fuzzer.sleep_time_us);
+    injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.sleep_probability, fuzzer.sleep_time_us_max);
     errno = saved_errno;
 }
 
@@ -309,13 +308,13 @@ void ThreadFuzzer::setup() const
         NAME##_before_yield_probability.load(std::memory_order_relaxed),   \
         NAME##_before_migrate_probability.load(std::memory_order_relaxed), \
         NAME##_before_sleep_probability.load(std::memory_order_relaxed),   \
-        NAME##_before_sleep_time_us.load(std::memory_order_relaxed));
+        NAME##_before_sleep_time_us_max.load(std::memory_order_relaxed));
 #define INJECTION_AFTER(NAME) \
     injectionImpl(                                                         \
         NAME##_after_yield_probability.load(std::memory_order_relaxed),    \
         NAME##_after_migrate_probability.load(std::memory_order_relaxed),  \
         NAME##_after_sleep_probability.load(std::memory_order_relaxed),    \
-        NAME##_after_sleep_time_us.load(std::memory_order_relaxed));
+        NAME##_after_sleep_time_us_max.load(std::memory_order_relaxed));
 
 /// ThreadFuzzer intercepts pthread_mutex_lock()/pthread_mutex_unlock().
 ///
diff --git a/src/Common/ThreadFuzzer.h b/src/Common/ThreadFuzzer.h
index 1cff27a7588..8956538e7d1 100644
--- a/src/Common/ThreadFuzzer.h
+++ b/src/Common/ThreadFuzzer.h
@@ -16,7 +16,7 @@ namespace DB
   * THREAD_FUZZER_YIELD_PROBABILITY   - probability to do 'sched_yield'.
   * THREAD_FUZZER_MIGRATE_PROBABILITY - probability to set CPU affinity to random CPU core.
   * THREAD_FUZZER_SLEEP_PROBABILITY   - probability to sleep.
-  * THREAD_FUZZER_SLEEP_TIME_US       - amount of time to sleep in microseconds.
+  * THREAD_FUZZER_SLEEP_TIME_US_MAX   - max amount of time to sleep in microseconds, actual sleep time is randomized.
   *
   * ThreadFuzzer will do nothing if environment variables are not set accordingly.
   *
@@ -33,16 +33,15 @@ namespace DB
   *
   * Notes:
   * - it can be also implemented with instrumentation (example: LLVM Xray) instead of signals.
-  * - we should also make the sleep time random.
   * - sleep and migration obviously helps, but the effect of yield is unclear.
   *
   * In addition, we allow to inject glitches around thread synchronization functions.
   * Example:
   *
   * THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
-  * THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
+  * THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000
   * THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
-  * THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
+  * THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000
   */
 class ThreadFuzzer
 {
@@ -67,7 +66,8 @@ private:
     double yield_probability = 0;
     double migrate_probability = 0;
     double sleep_probability = 0;
-    double sleep_time_us = 0;
+    double sleep_time_us_max = 0;
+    
     double explicit_sleep_probability = 0;
     double explicit_memory_exception_probability = 0;
 
diff --git a/src/Common/examples/chaos_sanitizer.cpp b/src/Common/examples/chaos_sanitizer.cpp
index 76e22411a2e..175c3e8d767 100644
--- a/src/Common/examples/chaos_sanitizer.cpp
+++ b/src/Common/examples/chaos_sanitizer.cpp
@@ -12,7 +12,7 @@
 /** Proves that ThreadFuzzer helps to find concurrency bugs.
   *
   * for i in {1..10}; do ./chaos_sanitizer 1000000; done
-  * for i in {1..10}; do THREAD_FUZZER_CPU_TIME_PERIOD_US=1000 THREAD_FUZZER_SLEEP_PROBABILITY=0.1 THREAD_FUZZER_SLEEP_TIME_US=100000 ./chaos_sanitizer 1000000; done
+  * for i in {1..10}; do THREAD_FUZZER_CPU_TIME_PERIOD_US=1000 THREAD_FUZZER_SLEEP_PROBABILITY=0.1 THREAD_FUZZER_SLEEP_TIME_US_MAX=100000 ./chaos_sanitizer 1000000; done
   */
 int main(int argc, char ** argv)
 {
diff --git a/utils/keeper-overload/keeper-overload.py b/utils/keeper-overload/keeper-overload.py
index 0a059b10588..1032ea656bc 100755
--- a/utils/keeper-overload/keeper-overload.py
+++ b/utils/keeper-overload/keeper-overload.py
@@ -102,7 +102,7 @@ class Keeper:
         if self.with_thread_fuzzer:
             env["THREAD_FUZZER_CPU_TIME_PERIOD_US"] = "1000"
             env["THREAD_FUZZER_SLEEP_PROBABILITY"] = "0.1"
-            env["THREAD_FUZZER_SLEEP_TIME_US"] = "100000"
+            env["THREAD_FUZZER_SLEEP_TIME_US_MAX"] = "100000"
             env["THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY"] = "1"
             env["THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY"] = "1"
             env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_MIGRATE_PROBABILITY"] = "1"
@@ -112,10 +112,10 @@ class Keeper:
             env["THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY"] = "0.001"
             env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY"] = "0.001"
             env["THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY"] = "0.001"
-            env["THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US"] = "10000"
-            env["THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US"] = "10000"
-            env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US"] = "10000"
-            env["THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US"] = "10000"
+            env["THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX"] = "10000"
+            env["THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX"] = "10000"
+            env["THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX"] = "10000"
+            env["THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX"] = "10000"
 
         self.process = subprocess.Popen(
             [

From e6c04599c1ef15718165efee4cd044ea22cefb7c Mon Sep 17 00:00:00 2001
From: tomershafir <tomer.shafir8@gmail.com>
Date: Fri, 15 Mar 2024 00:03:51 +0200
Subject: [PATCH 38/92] sched_yield matters

---
 src/Common/ThreadFuzzer.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Common/ThreadFuzzer.h b/src/Common/ThreadFuzzer.h
index 8956538e7d1..b5501ecf4e1 100644
--- a/src/Common/ThreadFuzzer.h
+++ b/src/Common/ThreadFuzzer.h
@@ -33,7 +33,6 @@ namespace DB
   *
   * Notes:
   * - it can be also implemented with instrumentation (example: LLVM Xray) instead of signals.
-  * - sleep and migration obviously helps, but the effect of yield is unclear.
   *
   * In addition, we allow to inject glitches around thread synchronization functions.
   * Example:

From ac0ed1deee9afa394d0b6ef15fcd3cfb38e4f9c4 Mon Sep 17 00:00:00 2001
From: tomershafir <tomer.shafir8@gmail.com>
Date: Fri, 15 Mar 2024 09:03:43 +0200
Subject: [PATCH 39/92] rm whitespace

---
 src/Common/ThreadFuzzer.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/ThreadFuzzer.h b/src/Common/ThreadFuzzer.h
index b5501ecf4e1..e8f1fe606f0 100644
--- a/src/Common/ThreadFuzzer.h
+++ b/src/Common/ThreadFuzzer.h
@@ -66,7 +66,7 @@ private:
     double migrate_probability = 0;
     double sleep_probability = 0;
     double sleep_time_us_max = 0;
-    
+
     double explicit_sleep_probability = 0;
     double explicit_memory_exception_probability = 0;
 

From 32ed2353f511d9e87e48cf20d807a6fcdf6d9334 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Tue, 20 Feb 2024 18:36:07 +0800
Subject: [PATCH 40/92] fix issue of actions dag split

---
 src/Interpreters/ActionsDAG.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index 37ef217cb6d..86a5b91ad63 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -1757,6 +1757,10 @@ ActionsDAG::SplitResult ActionsDAG::split(std::unordered_set<const Node *> split
         }
     }
 
+    /// All input nodes are needed by split to make sure first DAG and initial DAG have equal inputs,
+    for (const auto * input: inputs)
+        data[input].needed_by_split_node = true;
+
     /// DFS. Move nodes to one of the DAGs.
     for (const auto & node : nodes)
     {

From aaab9e4ed4977046fd7d9f04a17d2c8455dc9a3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 15 Mar 2024 16:49:54 +0100
Subject: [PATCH 41/92] Add test

---
 .../03010_view_prewhere_in.reference            |  3 +++
 .../0_stateless/03010_view_prewhere_in.sql      | 17 +++++++++++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 tests/queries/0_stateless/03010_view_prewhere_in.reference
 create mode 100644 tests/queries/0_stateless/03010_view_prewhere_in.sql

diff --git a/tests/queries/0_stateless/03010_view_prewhere_in.reference b/tests/queries/0_stateless/03010_view_prewhere_in.reference
new file mode 100644
index 00000000000..01e79c32a8c
--- /dev/null
+++ b/tests/queries/0_stateless/03010_view_prewhere_in.reference
@@ -0,0 +1,3 @@
+1
+2
+3
diff --git a/tests/queries/0_stateless/03010_view_prewhere_in.sql b/tests/queries/0_stateless/03010_view_prewhere_in.sql
new file mode 100644
index 00000000000..799c07f31ce
--- /dev/null
+++ b/tests/queries/0_stateless/03010_view_prewhere_in.sql
@@ -0,0 +1,17 @@
+DROP TABLE IF EXISTS v;
+CREATE VIEW v (`date` UInt32,`value` UInt8) AS
+WITH
+    data AS (SELECT '' id LIMIT 0),
+    r AS (SELECT'' as id, 1::UInt8 as value)
+SELECT
+    now() as date,
+    value AND (data.id IN (SELECT '' as d from system.one)) AS value
+FROM data
+         LEFT JOIN r ON data.id = r.id;
+
+SELECT 1;
+SELECT date, value FROM v;
+SELECT 2;
+SELECT date, value FROM v ORDER BY date;
+SELECT 3;
+DROP TABLE v;

From f02858a917639d08d1dff705e477523d7d4a1225 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 15 Mar 2024 15:59:19 +0000
Subject: [PATCH 42/92] Use SHARD_LOAD_QUEUE_BACKLOG for dictionaries in tests

---
 .../02098_hashed_array_dictionary_simple_key.sql.j2   | 10 ++++++----
 .../02099_hashed_array_dictionary_complex_key.sql.j2  | 11 +++++++----
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/tests/queries/0_stateless/02098_hashed_array_dictionary_simple_key.sql.j2 b/tests/queries/0_stateless/02098_hashed_array_dictionary_simple_key.sql.j2
index e5d8ad36c6d..c023f3ba6ad 100644
--- a/tests/queries/0_stateless/02098_hashed_array_dictionary_simple_key.sql.j2
+++ b/tests/queries/0_stateless/02098_hashed_array_dictionary_simple_key.sql.j2
@@ -10,8 +10,9 @@ ENGINE = TinyLog;
 INSERT INTO simple_key_simple_attributes_source_table VALUES(0, 'value_0', 'value_second_0');
 INSERT INTO simple_key_simple_attributes_source_table VALUES(1, 'value_1', 'value_second_1');
 INSERT INTO simple_key_simple_attributes_source_table VALUES(2, 'value_2', 'value_second_2');
+INSERT INTO simple_key_simple_attributes_source_table SELECT number + 10 as id, concat('value_', toString(id)), concat('value_second_', toString(id)) FROM numbers_mt(1_000_000);
 
-{% for dictionary_config in ['', 'SHARDS 16'] -%}
+{% for dictionary_config in ['', 'SHARDS 16 SHARD_LOAD_QUEUE_BACKLOG 2'] -%}
 
 DROP DICTIONARY IF EXISTS hashed_array_dictionary_simple_key_simple_attributes;
 CREATE DICTIONARY hashed_array_dictionary_simple_key_simple_attributes
@@ -42,7 +43,7 @@ SELECT dictGetOrDefault('hashed_array_dictionary_simple_key_simple_attributes',
 SELECT 'dictHas';
 SELECT dictHas('hashed_array_dictionary_simple_key_simple_attributes', number) FROM system.numbers LIMIT 4;
 SELECT 'select all values as input stream';
-SELECT * FROM hashed_array_dictionary_simple_key_simple_attributes ORDER BY id;
+SELECT * FROM hashed_array_dictionary_simple_key_simple_attributes ORDER BY id LIMIT 3;
 
 DROP DICTIONARY hashed_array_dictionary_simple_key_simple_attributes;
 {% endfor %}
@@ -61,8 +62,9 @@ ENGINE = TinyLog;
 INSERT INTO simple_key_complex_attributes_source_table VALUES(0, 'value_0', 'value_second_0');
 INSERT INTO simple_key_complex_attributes_source_table VALUES(1, 'value_1', NULL);
 INSERT INTO simple_key_complex_attributes_source_table VALUES(2, 'value_2', 'value_second_2');
+INSERT INTO simple_key_complex_attributes_source_table SELECT number + 10 as id, concat('value_', toString(id)), concat('value_second_', toString(id)) FROM numbers_mt(1_000_000);
 
-{% for dictionary_config in ['', 'SHARDS 16'] -%}
+{% for dictionary_config in ['', 'SHARDS 16 SHARD_LOAD_QUEUE_BACKLOG 2'] -%}
 
 DROP DICTIONARY IF EXISTS hashed_array_dictionary_simple_key_complex_attributes;
 CREATE DICTIONARY hashed_array_dictionary_simple_key_complex_attributes
@@ -92,7 +94,7 @@ SELECT dictGetOrDefault('hashed_array_dictionary_simple_key_complex_attributes',
 SELECT 'dictHas';
 SELECT dictHas('hashed_array_dictionary_simple_key_complex_attributes', number) FROM system.numbers LIMIT 4;
 SELECT 'select all values as input stream';
-SELECT * FROM hashed_array_dictionary_simple_key_complex_attributes ORDER BY id;
+SELECT * FROM hashed_array_dictionary_simple_key_complex_attributes ORDER BY id LIMIT 3;
 
 DROP DICTIONARY hashed_array_dictionary_simple_key_complex_attributes;
 
diff --git a/tests/queries/0_stateless/02099_hashed_array_dictionary_complex_key.sql.j2 b/tests/queries/0_stateless/02099_hashed_array_dictionary_complex_key.sql.j2
index 56f9b264a62..3a92b035666 100644
--- a/tests/queries/0_stateless/02099_hashed_array_dictionary_complex_key.sql.j2
+++ b/tests/queries/0_stateless/02099_hashed_array_dictionary_complex_key.sql.j2
@@ -12,7 +12,9 @@ INSERT INTO complex_key_simple_attributes_source_table VALUES(0, 'id_key_0', 'va
 INSERT INTO complex_key_simple_attributes_source_table VALUES(1, 'id_key_1', 'value_1', 'value_second_1');
 INSERT INTO complex_key_simple_attributes_source_table VALUES(2, 'id_key_2', 'value_2', 'value_second_2');
 
-{% for dictionary_config in ['', 'SHARDS 16'] -%}
+INSERT INTO complex_key_simple_attributes_source_table SELECT number + 10 as id, concat('id_key_', toString(id)), toString(id), toString(id) FROM numbers_mt(1_000_000);
+
+{% for dictionary_config in ['', 'SHARDS 16 SHARD_LOAD_QUEUE_BACKLOG 2'] -%}
 
 DROP DICTIONARY IF EXISTS hashed_array_dictionary_complex_key_simple_attributes;
 CREATE DICTIONARY hashed_array_dictionary_complex_key_simple_attributes
@@ -43,7 +45,7 @@ SELECT dictGetOrDefault('hashed_array_dictionary_complex_key_simple_attributes',
 SELECT 'dictHas';
 SELECT dictHas('hashed_array_dictionary_complex_key_simple_attributes', (number, concat('id_key_', toString(number)))) FROM system.numbers LIMIT 4;
 SELECT 'select all values as input stream';
-SELECT * FROM hashed_array_dictionary_complex_key_simple_attributes ORDER BY (id, id_key);
+SELECT * FROM hashed_array_dictionary_complex_key_simple_attributes ORDER BY (id, id_key) LIMIT 3;
 
 DROP DICTIONARY hashed_array_dictionary_complex_key_simple_attributes;
 
@@ -64,8 +66,9 @@ ENGINE = TinyLog;
 INSERT INTO complex_key_complex_attributes_source_table VALUES(0, 'id_key_0', 'value_0', 'value_second_0');
 INSERT INTO complex_key_complex_attributes_source_table VALUES(1, 'id_key_1', 'value_1', NULL);
 INSERT INTO complex_key_complex_attributes_source_table VALUES(2, 'id_key_2', 'value_2', 'value_second_2');
+INSERT INTO complex_key_complex_attributes_source_table SELECT number + 10 as id, concat('id_key_', toString(id)), toString(id), toString(id) FROM numbers_mt(1_000_000);
 
-{% for dictionary_config in ['', 'SHARDS 16'] -%}
+{% for dictionary_config in ['', 'SHARDS 16 SHARD_LOAD_QUEUE_BACKLOG 2'] -%}
 
 DROP DICTIONARY IF EXISTS hashed_array_dictionary_complex_key_complex_attributes;
 CREATE DICTIONARY hashed_array_dictionary_complex_key_complex_attributes
@@ -97,7 +100,7 @@ SELECT dictGetOrDefault('hashed_array_dictionary_complex_key_complex_attributes'
 SELECT 'dictHas';
 SELECT dictHas('hashed_array_dictionary_complex_key_complex_attributes', (number, concat('id_key_', toString(number)))) FROM system.numbers LIMIT 4;
 SELECT 'select all values as input stream';
-SELECT * FROM hashed_array_dictionary_complex_key_complex_attributes ORDER BY (id, id_key);
+SELECT * FROM hashed_array_dictionary_complex_key_complex_attributes ORDER BY (id, id_key) LIMIT 3;
 
 {% endfor %}
 

From e1b2e0a00bae0347f02eba4c484577ed41b51c46 Mon Sep 17 00:00:00 2001
From: Shaun Struwig <41984034+Blargian@users.noreply.github.com>
Date: Sat, 16 Mar 2024 02:34:45 +0100
Subject: [PATCH 43/92] Update polygon.md

Add WKT function to documentation
---
 .../en/sql-reference/functions/geo/polygon.md | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/docs/en/sql-reference/functions/geo/polygon.md b/docs/en/sql-reference/functions/geo/polygon.md
index 35e2280e5cc..3aee8f03e28 100644
--- a/docs/en/sql-reference/functions/geo/polygon.md
+++ b/docs/en/sql-reference/functions/geo/polygon.md
@@ -4,6 +4,34 @@ sidebar_label: Polygons
 title: "Functions for Working with Polygons"
 ---
 
+## WKT
+
+Returns a WKT (Well Known Text) geometric object from tuples which represent point coordinates.
+
+**Syntax**
+
+```sql
+WKT(points)
+```
+
+**Parameters**
+
+- `points`: a [tuple](../../data-types/tuple.md) representing coordinate point (x, y) or an [array](../../data-types/array.md) of tuples, or an array of tuple arrays.
+
+**Returned value**
+
+WKT (Well Known Text) geometric object.
+
+**Example**
+
+```sql
+SELECT wkt([[(0., 0.), (10., 0.), (10., 10.), (0., 10.)], [(4., 4.), (5., 4.), (5., 5.), (4., 5.)]]);
+```
+
+```response
+POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4))
+```
+
 ## readWKTMultiPolygon
 
 Converts a WKT (Well Known Text) MultiPolygon into a MultiPolygon type.

From 5c9f132800975f16346c70d483ff54363bc0e99f Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Sat, 16 Mar 2024 13:18:56 +0100
Subject: [PATCH 44/92] Explain behaviour for various geometric types and array
 shapes

---
 .../en/sql-reference/functions/geo/polygon.md | 38 ++++++++++++++++---
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/docs/en/sql-reference/functions/geo/polygon.md b/docs/en/sql-reference/functions/geo/polygon.md
index 3aee8f03e28..afdd6fe3f96 100644
--- a/docs/en/sql-reference/functions/geo/polygon.md
+++ b/docs/en/sql-reference/functions/geo/polygon.md
@@ -6,7 +6,11 @@ title: "Functions for Working with Polygons"
 
 ## WKT
 
-Returns a WKT (Well Known Text) geometric object from tuples which represent point coordinates.
+Returns a WKT (Well Known Text) geometric object from tuples which represent point coordinates. Supported WKT objects are: 
+
+- POINT
+- POLYGON
+- MULTIPOLYGON
 
 **Syntax**
 
@@ -20,16 +24,40 @@ WKT(points)
 
 **Returned value**
 
-WKT (Well Known Text) geometric object.
+- POINT is returned for a tuple.
+- POLYGON is returned for an array of tuples.
+- MULTIPOLYGON is returned for an array of tuple arrays. 
 
-**Example**
+**Examples**
+
+POINT from tuple:
 
 ```sql
-SELECT wkt([[(0., 0.), (10., 0.), (10., 10.), (0., 10.)], [(4., 4.), (5., 4.), (5., 5.), (4., 5.)]]);
+SELECT wkt((0., 0.));
 ```
 
 ```response
-POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4))
+POINT(0 0)
+```
+
+POLYGON from an array of tuples:
+
+```sql
+SELECT wkt([(0., 0.), (10., 0.), (10., 10.), (0., 10.)]);
+```
+
+```response
+POLYGON((0 0,10 0,10 10,0 10))
+```
+
+MULTIPOLYGON from an array of multi-dimensional tuple arrays:
+
+```sql
+SELECT wkt([[[(0., 0.), (10., 0.), (10., 10.), (0., 10.)], [(4., 4.), (5., 4.), (5., 5.), (4., 5.)]], [[(-10., -10.), (-10., -9.), (-9., 10.)]]]);
+```
+
+```response
+MULTIPOLYGON(((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))
 ```
 
 ## readWKTMultiPolygon

From f85eda34d937ad28044f063dde0378582823b8d2 Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Sat, 16 Mar 2024 13:25:07 +0100
Subject: [PATCH 45/92] Update wording for multipolygon

---
 docs/en/sql-reference/functions/geo/polygon.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/en/sql-reference/functions/geo/polygon.md b/docs/en/sql-reference/functions/geo/polygon.md
index afdd6fe3f96..886ad511b9b 100644
--- a/docs/en/sql-reference/functions/geo/polygon.md
+++ b/docs/en/sql-reference/functions/geo/polygon.md
@@ -20,13 +20,13 @@ WKT(points)
 
 **Parameters**
 
-- `points`: a [tuple](../../data-types/tuple.md) representing coordinate point (x, y) or an [array](../../data-types/array.md) of tuples, or an array of tuple arrays.
+- `points`: a [tuple](../../data-types/tuple.md) representing coordinate point (x, y) or an [array](../../data-types/array.md) of tuples, or an array of multi-dimensional tuple arrays.
 
 **Returned value**
 
 - POINT is returned for a tuple.
 - POLYGON is returned for an array of tuples.
-- MULTIPOLYGON is returned for an array of tuple arrays. 
+- MULTIPOLYGON is returned for an array of multi-dimensional tuple arrays. 
 
 **Examples**
 
@@ -40,7 +40,7 @@ SELECT wkt((0., 0.));
 POINT(0 0)
 ```
 
-POLYGON from an array of tuples:
+POLYGON from an array of tuples or an array of tuple arrays:
 
 ```sql
 SELECT wkt([(0., 0.), (10., 0.), (10., 10.), (0., 10.)]);

From 5b05bee2a9f70242c59c67ffdfb7fcade326601d Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Sat, 16 Mar 2024 15:16:53 +0100
Subject: [PATCH 46/92] Add reference to geodata types

---
 docs/en/sql-reference/functions/geo/polygon.md | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/docs/en/sql-reference/functions/geo/polygon.md b/docs/en/sql-reference/functions/geo/polygon.md
index 886ad511b9b..6483709b461 100644
--- a/docs/en/sql-reference/functions/geo/polygon.md
+++ b/docs/en/sql-reference/functions/geo/polygon.md
@@ -6,7 +6,7 @@ title: "Functions for Working with Polygons"
 
 ## WKT
 
-Returns a WKT (Well Known Text) geometric object from tuples which represent point coordinates. Supported WKT objects are: 
+Returns a WKT (Well Known Text) geometric object from various [Geo Data Types](../../data-types/geo.md). Supported WKT objects are: 
 
 - POINT
 - POLYGON
@@ -15,18 +15,23 @@ Returns a WKT (Well Known Text) geometric object from tuples which represent poi
 **Syntax**
 
 ```sql
-WKT(points)
+WKT(geo_data)
 ```
 
 **Parameters**
 
-- `points`: a [tuple](../../data-types/tuple.md) representing coordinate point (x, y) or an [array](../../data-types/array.md) of tuples, or an array of multi-dimensional tuple arrays.
+`geo_data` can be one of the following [Geo Data Types](../../data-types/geo.md) or their underlying primitive types:
+
+- [Point](../../data-types/geo.md#point) / [Tuple](../../data-types/.md)
+- [Ring](../../data-types/geo.md#ring) / [Array](../../data-types/array.md)([Tuple](../../data-types/.md))
+- [Polygon](../../data-types/geo.md#polygon) / [Array](../../data-types/array.md)([Array](../../data-types/.md)([Tuple](../../data-types/array.md))) 
+- [MultiPolygon](../../data-types/geo.md#multipolygon) / [Array](../../data-types/array.md)([Array](../../data-types/array.md)([Array](../../data-types/array.md)([Tuple](../../data-types/.md))))
 
 **Returned value**
 
-- POINT is returned for a tuple.
-- POLYGON is returned for an array of tuples.
-- MULTIPOLYGON is returned for an array of multi-dimensional tuple arrays. 
+- WKT geometric object `POINT` is returned for a Point.
+- WKT geometric object `POLYGON` is returned for a Polygon
+- WKT geometric object `MULTIPOLYGON` is returned for a MultiPolygon. 
 
 **Examples**
 

From e956656e9d99cd142cde92c50fa01558587f8813 Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Sat, 16 Mar 2024 15:19:30 +0100
Subject: [PATCH 47/92] Remove primitive types from parameters - looks ugly

---
 docs/en/sql-reference/functions/geo/polygon.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/en/sql-reference/functions/geo/polygon.md b/docs/en/sql-reference/functions/geo/polygon.md
index 6483709b461..c2572779ada 100644
--- a/docs/en/sql-reference/functions/geo/polygon.md
+++ b/docs/en/sql-reference/functions/geo/polygon.md
@@ -22,10 +22,10 @@ WKT(geo_data)
 
 `geo_data` can be one of the following [Geo Data Types](../../data-types/geo.md) or their underlying primitive types:
 
-- [Point](../../data-types/geo.md#point) / [Tuple](../../data-types/.md)
-- [Ring](../../data-types/geo.md#ring) / [Array](../../data-types/array.md)([Tuple](../../data-types/.md))
-- [Polygon](../../data-types/geo.md#polygon) / [Array](../../data-types/array.md)([Array](../../data-types/.md)([Tuple](../../data-types/array.md))) 
-- [MultiPolygon](../../data-types/geo.md#multipolygon) / [Array](../../data-types/array.md)([Array](../../data-types/array.md)([Array](../../data-types/array.md)([Tuple](../../data-types/.md))))
+- [Point](../../data-types/geo.md#point)
+- [Ring](../../data-types/geo.md#ring)
+- [Polygon](../../data-types/geo.md#polygon)
+- [MultiPolygon](../../data-types/geo.md#multipolygon)
 
 **Returned value**
 

From 43eea86ec507e1da7c3b42d8e2bb177150873a89 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Mon, 18 Mar 2024 17:13:20 +0800
Subject: [PATCH 48/92] fixed

---
 src/Interpreters/Aggregator.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h
index ee3086e1d0e..e339047063c 100644
--- a/src/Interpreters/Aggregator.h
+++ b/src/Interpreters/Aggregator.h
@@ -428,6 +428,7 @@ private:
         size_t row_begin,
         size_t row_end,
         AggregateFunctionInstruction * aggregate_instructions,
+        bool no_more_keys,
         bool all_keys_are_const,
         bool use_compiled_functions,
         AggregateDataPtr overflow_row) const;

From e27e6fb2f5410119b116bb12e9ff9c25d958d901 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 20 Feb 2024 11:41:08 +0000
Subject: [PATCH 49/92] Analyzer: support join using column from select list

---
 src/Analyzer/Passes/QueryAnalysisPass.cpp     | 140 ++++++++++++++----
 src/Planner/PlannerJoinTree.cpp               |  46 ++++--
 src/Storages/StorageMerge.cpp                 |   8 +-
 tests/analyzer_tech_debt.txt                  |   1 -
 .../02989_join_using_parent_scope.reference   |  88 +++++++++++
 .../02989_join_using_parent_scope.sql         |  45 ++++++
 6 files changed, 276 insertions(+), 52 deletions(-)
 create mode 100644 tests/queries/0_stateless/02989_join_using_parent_scope.reference
 create mode 100644 tests/queries/0_stateless/02989_join_using_parent_scope.sql

diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp
index 1da5d86edf3..702a70ddc61 100644
--- a/src/Analyzer/Passes/QueryAnalysisPass.cpp
+++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp
@@ -1276,7 +1276,7 @@ private:
         return {};
     }
 
-    static void convertJoinedColumnTypeToNullIfNeeded(QueryTreeNodePtr & resolved_identifier, const JoinKind & join_kind, std::optional<JoinTableSide> resolved_side)
+    static void convertJoinedColumnTypeToNullIfNeeded(QueryTreeNodePtr & resolved_identifier, const JoinKind & join_kind, std::optional<JoinTableSide> resolved_side, const ContextPtr & context)
     {
         if (resolved_identifier->getNodeType() == QueryTreeNodeType::COLUMN &&
             JoinCommon::canBecomeNullable(resolved_identifier->getResultType()) &&
@@ -1285,7 +1285,14 @@ private:
             (isRight(join_kind) && resolved_side && *resolved_side == JoinTableSide::Left)))
         {
             auto & resolved_column = resolved_identifier->as<ColumnNode &>();
-            resolved_column.setColumnType(makeNullableOrLowCardinalityNullable(resolved_column.getColumnType()));
+            auto new_result_type = makeNullableOrLowCardinalityNullable(resolved_column.getColumnType());
+            resolved_column.setColumnType(new_result_type);
+            if (resolved_column.hasExpression())
+            {
+                auto & resolved_expression = resolved_column.getExpression();
+                if (!resolved_expression->getResultType()->equals(*new_result_type))
+                    resolved_expression = buildCastFunction(resolved_expression, new_result_type, context, true);
+            }
         }
     }
 
@@ -3258,6 +3265,32 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id
     return {};
 }
 
+QueryTreeNodePtr checkIsMissedObjectJSONSubcolumn(const QueryTreeNodePtr & left_resolved_identifier,
+                                                  const QueryTreeNodePtr & right_resolved_identifier)
+{
+    if (left_resolved_identifier && right_resolved_identifier && left_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT
+        && right_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
+    {
+        auto & left_resolved_column = left_resolved_identifier->as<ConstantNode &>();
+        auto & right_resolved_column = right_resolved_identifier->as<ConstantNode &>();
+        if (left_resolved_column.getValueStringRepresentation() == "NULL" && right_resolved_column.getValueStringRepresentation() == "NULL")
+            return left_resolved_identifier;
+    }
+    else if (left_resolved_identifier && left_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
+    {
+        auto & left_resolved_column = left_resolved_identifier->as<ConstantNode &>();
+        if (left_resolved_column.getValueStringRepresentation() == "NULL")
+            return left_resolved_identifier;
+    }
+    else if (right_resolved_identifier && right_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
+    {
+        auto & right_resolved_column = right_resolved_identifier->as<ConstantNode &>();
+        if (right_resolved_column.getValueStringRepresentation() == "NULL")
+            return right_resolved_identifier;
+    }
+    return {};
+}
+
 QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLookup & identifier_lookup,
     const QueryTreeNodePtr & table_expression_node,
     IdentifierResolveScope & scope)
@@ -3358,28 +3391,8 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo
 
     /// If columns from left or right table were missed Object(Nullable('json')) subcolumns, they will be replaced
     /// to ConstantNode(NULL), which can't be cast to ColumnNode, so we resolve it here.
-    if (left_resolved_identifier && right_resolved_identifier && left_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT
-        && right_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
-    {
-        auto & left_resolved_column = left_resolved_identifier->as<ConstantNode &>();
-        auto & right_resolved_column = right_resolved_identifier->as<ConstantNode &>();
-        if (left_resolved_column.getValueStringRepresentation() == "NULL" && right_resolved_column.getValueStringRepresentation() == "NULL")
-            return left_resolved_identifier;
-    }
-    else if (left_resolved_identifier && left_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
-    {
-        resolved_side = JoinTableSide::Left;
-        auto & left_resolved_column = left_resolved_identifier->as<ConstantNode &>();
-        if (left_resolved_column.getValueStringRepresentation() == "NULL")
-            return left_resolved_identifier;
-    }
-    else if (right_resolved_identifier && right_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
-    {
-        resolved_side = JoinTableSide::Right;
-        auto & right_resolved_column = right_resolved_identifier->as<ConstantNode &>();
-        if (right_resolved_column.getValueStringRepresentation() == "NULL")
-            return right_resolved_identifier;
-    }
+    if (auto missed_subcolumn_identifier = checkIsMissedObjectJSONSubcolumn(left_resolved_identifier, right_resolved_identifier))
+        return missed_subcolumn_identifier;
 
     if (left_resolved_identifier && right_resolved_identifier)
     {
@@ -3522,7 +3535,7 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo
     if (scope.join_use_nulls)
     {
         resolved_identifier = resolved_identifier->clone();
-        convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side);
+        convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side, scope.context);
     }
 
     return resolved_identifier;
@@ -4402,6 +4415,31 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher(
                     const auto & join_using_column_nodes_list = join_using_column_node.getExpressionOrThrow()->as<ListNode &>();
                     const auto & join_using_column_nodes = join_using_column_nodes_list.getNodes();
 
+                    /** If column doesn't exists in the table, then do not match column from USING clause.
+                      * Example: SELECT a + 1 AS id, * FROM (SELECT 1 AS a) AS t1 JOIN (SELECT 2 AS id) AS t2 USING (id);
+                      * In this case `id` is not present in the left table expression,
+                      * so asterisk should return `id` from the right table expression.
+                      */
+                    auto is_column_from_parent_scope = [&scope](const QueryTreeNodePtr & using_node_from_table)
+                    {
+                        const auto & using_column_from_table = using_node_from_table->as<ColumnNode &>();
+                        auto table_expression_data_it = scope.table_expression_node_to_data.find(using_column_from_table.getColumnSource());
+                        if (table_expression_data_it != scope.table_expression_node_to_data.end())
+                        {
+                            const auto & table_expression_data = table_expression_data_it->second;
+                            const auto & column_name = using_column_from_table.getColumnName();
+                            if (!table_expression_data.column_name_to_column_node.contains(column_name))
+                            {
+                                return true;
+                            }
+                        }
+                        return false;
+                    };
+
+                    if (is_column_from_parent_scope(join_using_column_nodes.at(0)) ||
+                        is_column_from_parent_scope(join_using_column_nodes.at(1)))
+                        continue;
+
                     QueryTreeNodePtr matched_column_node;
 
                     if (isRight(join_node->getKind()))
@@ -4523,7 +4561,13 @@ ProjectionNames QueryAnalyzer::resolveMatcher(QueryTreeNodePtr & matcher_node, I
             for (auto & [node, node_name] : matched_expression_nodes_with_names)
             {
                 auto join_identifier_side = getColumnSideFromJoinTree(node, *nearest_scope_join_node);
-                convertJoinedColumnTypeToNullIfNeeded(node, nearest_scope_join_node->getKind(), join_identifier_side);
+                auto projection_name_it = node_to_projection_name.find(node);
+                node = node->clone();
+                /// Set the same projection name for new nullable node
+                if (projection_name_it != node_to_projection_name.end())
+                    node_to_projection_name.emplace(node, projection_name_it->second);
+
+                convertJoinedColumnTypeToNullIfNeeded(node, nearest_scope_join_node->getKind(), join_identifier_side, scope.context);
             }
         }
     }
@@ -7307,11 +7351,45 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS
             IdentifierLookup identifier_lookup{identifier_node->getIdentifier(), IdentifierLookupContext::EXPRESSION};
             auto result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getLeftTableExpression(), scope);
             if (!result_left_table_expression)
-                throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER,
-                    "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}",
-                    join_node_typed.formatASTForErrorMessage(),
-                    identifier_full_name,
-                    scope.scope_node->formatASTForErrorMessage());
+            {
+                /** Try to resolve identifier from parent subquery projection.
+                  * Example: SELECT a + 1 AS v FROM (SELECT 1 AS a) t1 JOIN (SELECT 2 AS b) USING b
+                  * In this case `b` is not in the left table expression, but it is in the parent subquery projection.
+                  */
+                QueryNode * query_node = scope.scope_node ? scope.scope_node->as<QueryNode>() : nullptr;
+                if (query_node)
+                {
+                    const auto & projection_list = query_node->getProjection();
+                    for (const auto & projection_node : projection_list.getNodes())
+                    {
+                        if (projection_node->hasAlias() && identifier_full_name == projection_node->getAlias())
+                        {
+                            auto left_subquery = std::make_shared<QueryNode>(query_node->getMutableContext());
+                            left_subquery->getProjection().getNodes().push_back(projection_node->clone());
+                            left_subquery->getJoinTree() = join_node_typed.getLeftTableExpression();
+
+                            IdentifierResolveScope left_subquery_scope(left_subquery, nullptr /*parent_scope*/);
+                            resolveQuery(left_subquery, left_subquery_scope);
+
+                            const auto & resolved_nodes = left_subquery->getProjection().getNodes();
+                            if (resolved_nodes.size() == 1)
+                            {
+                                /// Create ColumnNode with expression from parent projection
+                                result_left_table_expression = std::make_shared<ColumnNode>(
+                                    NameAndTypePair{identifier_full_name, resolved_nodes.at(0)->getResultType()}, resolved_nodes.at(0), join_node_typed.getLeftTableExpression());
+                                break;
+                            }
+                        }
+                    }
+                }
+
+                if (!result_left_table_expression)
+                    throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER,
+                        "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}",
+                        join_node_typed.formatASTForErrorMessage(),
+                        identifier_full_name,
+                        scope.scope_node->formatASTForErrorMessage());
+            }
 
             if (result_left_table_expression->getNodeType() != QueryTreeNodeType::COLUMN)
                 throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index 8ca8f0f258b..61a7a19f5be 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -585,6 +585,28 @@ UInt64 mainQueryNodeBlockSizeByLimit(const SelectQueryInfo & select_query_info)
     return 0;
 }
 
+std::unique_ptr<ExpressionStep> createComputeAliasColumnsStep(
+    const std::unordered_map<std::string, ActionsDAGPtr> & alias_column_expressions, const DataStream & current_data_stream)
+{
+    ActionsDAGPtr merged_alias_columns_actions_dag = std::make_shared<ActionsDAG>(current_data_stream.header.getColumnsWithTypeAndName());
+    ActionsDAG::NodeRawConstPtrs action_dag_outputs = merged_alias_columns_actions_dag->getInputs();
+
+    for (const auto & [column_name, alias_column_actions_dag] : alias_column_expressions)
+    {
+        const auto & current_outputs = alias_column_actions_dag->getOutputs();
+        action_dag_outputs.insert(action_dag_outputs.end(), current_outputs.begin(), current_outputs.end());
+        merged_alias_columns_actions_dag->mergeNodes(std::move(*alias_column_actions_dag));
+    }
+
+    for (const auto * output_node : action_dag_outputs)
+        merged_alias_columns_actions_dag->addOrReplaceInOutputs(*output_node);
+    merged_alias_columns_actions_dag->removeUnusedActions(false);
+
+    auto alias_column_step = std::make_unique<ExpressionStep>(current_data_stream, std::move(merged_alias_columns_actions_dag));
+    alias_column_step->setStepDescription("Compute alias columns");
+    return alias_column_step;
+}
+
 JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expression,
     const SelectQueryInfo & select_query_info,
     const SelectQueryOptions & select_query_options,
@@ -865,22 +887,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                 const auto & alias_column_expressions = table_expression_data.getAliasColumnExpressions();
                 if (!alias_column_expressions.empty() && query_plan.isInitialized() && from_stage == QueryProcessingStage::FetchColumns)
                 {
-                    ActionsDAGPtr merged_alias_columns_actions_dag = std::make_shared<ActionsDAG>(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
-                    ActionsDAG::NodeRawConstPtrs action_dag_outputs = merged_alias_columns_actions_dag->getInputs();
-
-                    for (const auto & [column_name, alias_column_actions_dag] : alias_column_expressions)
-                    {
-                        const auto & current_outputs = alias_column_actions_dag->getOutputs();
-                        action_dag_outputs.insert(action_dag_outputs.end(), current_outputs.begin(), current_outputs.end());
-                        merged_alias_columns_actions_dag->mergeNodes(std::move(*alias_column_actions_dag));
-                    }
-
-                    for (const auto * output_node : action_dag_outputs)
-                        merged_alias_columns_actions_dag->addOrReplaceInOutputs(*output_node);
-                    merged_alias_columns_actions_dag->removeUnusedActions(false);
-
-                    auto alias_column_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(merged_alias_columns_actions_dag));
-                    alias_column_step->setStepDescription("Compute alias columns");
+                    auto alias_column_step = createComputeAliasColumnsStep(alias_column_expressions, query_plan.getCurrentDataStream());
                     query_plan.addStep(std::move(alias_column_step));
                 }
 
@@ -961,6 +968,13 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
             query_node_to_plan_step_mapping.insert(mapping.begin(), mapping.end());
             query_plan = std::move(subquery_planner).extractQueryPlan();
         }
+
+        const auto & alias_column_expressions = table_expression_data.getAliasColumnExpressions();
+        if (!alias_column_expressions.empty() && query_plan.isInitialized() && from_stage == QueryProcessingStage::FetchColumns)
+        {
+            auto alias_column_step = createComputeAliasColumnsStep(alias_column_expressions, query_plan.getCurrentDataStream());
+            query_plan.addStep(std::move(alias_column_step));
+        }
     }
     else
     {
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index 9161b2773da..df3c141db1b 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -678,13 +678,13 @@ public:
 
     void visitImpl(QueryTreeNodePtr & node)
     {
-        if (auto * column = node->as<ColumnNode>(); column != nullptr)
+        if (auto * column = node->as<ColumnNode>())
         {
             if (column->hasExpression())
             {
-                auto column_name = column->getColumnName();
-                node = column->getExpressionOrThrow();
-                node->setAlias(column_name);
+                QueryTreeNodePtr column_expression = column->getExpressionOrThrow();
+                column_expression->setAlias(column->getColumnName());
+                node = std::move(column_expression);
             }
             else
                 column->setColumnSource(replacement_table_expression);
diff --git a/tests/analyzer_tech_debt.txt b/tests/analyzer_tech_debt.txt
index bcf0eee0d0e..21ab4a28ce4 100644
--- a/tests/analyzer_tech_debt.txt
+++ b/tests/analyzer_tech_debt.txt
@@ -1,7 +1,6 @@
 00223_shard_distributed_aggregation_memory_efficient
 00717_merge_and_distributed
 00725_memory_tracking
-01062_pm_all_join_with_block_continuation
 01083_expressions_in_engine_arguments
 01155_rename_move_materialized_view
 01584_distributed_buffer_cannot_find_column
diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.reference b/tests/queries/0_stateless/02989_join_using_parent_scope.reference
new file mode 100644
index 00000000000..c396431049f
--- /dev/null
+++ b/tests/queries/0_stateless/02989_join_using_parent_scope.reference
@@ -0,0 +1,88 @@
+-- { echoOn }
+SELECT 1 AS a FROM tb JOIN tabc USING (a) ORDER BY ALL;
+1
+1
+1
+1
+SELECT a + 2 AS b FROM ta JOIN tabc USING (b) ORDER BY ALL;
+2
+3
+4
+SELECT b + 2 AS a FROM tb JOIN tabc USING (a) ORDER BY ALL;
+2
+3
+SELECT a + 2 AS c FROM ta JOIN tabc USING (c) ORDER BY ALL;
+2
+3
+4
+5
+SELECT b AS a, a FROM tb JOIN tabc USING (a) ORDER BY ALL;
+0	0
+1	1
+2	2
+3	3
+SELECT 1 AS b FROM tb JOIN ta USING (b); -- { serverError UNKNOWN_IDENTIFIER }
+-- SELECT * works returns all columns from both tables in new analyzer
+SET allow_experimental_analyzer = 1;
+SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL;
+3	3	0	3
+3	3	1	3
+3	3	2	3
+3	3	3	3
+3	3	\N	0
+3	3	\N	1
+3	3	\N	2
+SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL;
+1	0	1
+2	1	2
+3	2	3
+SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL;
+1	0	1
+2	1	2
+3	2	3
+SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL;
+1	0	1
+2	1	2
+3	2	3
+4	3	\N
+SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL;
+1	0	1
+2	1	2
+3	2	3
+\N	\N	0
+SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL;
+1	0	1
+2	1	2
+3	2	3
+4	3	\N
+\N	\N	0
+SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1;
+1	0	1	2	3
+2	1	2	3	4
+3	2	3	4	5
+4	3	\N	\N	\N
+\N	\N	0	1	2
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
+1	0	1	2
+2	1	2	3
+3	2	3	4
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
+1	0	1	2
+2	1	2	3
+3	2	3	4
+4	3	\N	\N
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
+1	0	1	2
+2	1	2	3
+3	2	3	4
+\N	\N	0	1
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
+1	0	1	2
+2	1	2	3
+3	2	3	4
+4	3	\N	\N
+\N	\N	0	1
+DROP TABLE IF EXISTS tabc;
+DROP TABLE IF EXISTS ta;
+DROP TABLE IF EXISTS tb;
+DROP TABLE IF EXISTS tc;
diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.sql b/tests/queries/0_stateless/02989_join_using_parent_scope.sql
new file mode 100644
index 00000000000..05f1137f60e
--- /dev/null
+++ b/tests/queries/0_stateless/02989_join_using_parent_scope.sql
@@ -0,0 +1,45 @@
+DROP TABLE IF EXISTS tabc;
+CREATE TABLE tabc (a UInt32, b UInt32 ALIAS a + 1, c UInt32 ALIAS b + 1) ENGINE = MergeTree ORDER BY a;
+INSERT INTO tabc SELECT number FROM numbers(4);
+
+DROP TABLE IF EXISTS ta;
+CREATE TABLE ta (a Int32) ENGINE = MergeTree ORDER BY tuple();
+INSERT INTO ta SELECT number FROM numbers(4);
+
+DROP TABLE IF EXISTS tb;
+CREATE TABLE tb (b Int32) ENGINE = MergeTree ORDER BY tuple();
+INSERT INTO tb SELECT number FROM numbers(4);
+
+SET join_use_nulls = 1;
+
+-- { echoOn }
+SELECT 1 AS a FROM tb JOIN tabc USING (a) ORDER BY ALL;
+SELECT a + 2 AS b FROM ta JOIN tabc USING (b) ORDER BY ALL;
+SELECT b + 2 AS a FROM tb JOIN tabc USING (a) ORDER BY ALL;
+SELECT a + 2 AS c FROM ta JOIN tabc USING (c) ORDER BY ALL;
+
+SELECT b AS a, a FROM tb JOIN tabc USING (a) ORDER BY ALL;
+SELECT 1 AS b FROM tb JOIN ta USING (b); -- { serverError UNKNOWN_IDENTIFIER }
+
+-- SELECT * works returns all columns from both tables in new analyzer
+SET allow_experimental_analyzer = 1;
+
+SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL;
+
+SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1;
+
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
+
+
+DROP TABLE IF EXISTS tabc;
+DROP TABLE IF EXISTS ta;
+DROP TABLE IF EXISTS tb;
+DROP TABLE IF EXISTS tc;

From fbad9b799cd4d8549bf8479c94688cb7a1277c89 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Mon, 11 Mar 2024 17:45:05 +0000
Subject: [PATCH 50/92] fix PREWHERE with join_use_nulls

---
 src/Analyzer/Passes/QueryAnalysisPass.cpp     | 90 +++++++++++++------
 .../02989_join_using_parent_scope.reference   | 68 +++++++-------
 .../02989_join_using_parent_scope.sql         |  5 +-
 3 files changed, 102 insertions(+), 61 deletions(-)

diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp
index 702a70ddc61..35b6f05191e 100644
--- a/src/Analyzer/Passes/QueryAnalysisPass.cpp
+++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp
@@ -776,6 +776,7 @@ struct IdentifierResolveScope
     std::unordered_map<QueryTreeNodePtr, TableExpressionData> table_expression_node_to_data;
 
     QueryTreeNodePtrWithHashSet nullable_group_by_keys;
+    QueryTreeNodePtrWithHashMap<QueryTreeNodePtr> nullable_join_columns;
 
     /// Use identifier lookup to result cache
     bool use_identifier_lookup_to_result_cache = true;
@@ -1276,7 +1277,11 @@ private:
         return {};
     }
 
-    static void convertJoinedColumnTypeToNullIfNeeded(QueryTreeNodePtr & resolved_identifier, const JoinKind & join_kind, std::optional<JoinTableSide> resolved_side, const ContextPtr & context)
+    static QueryTreeNodePtr convertJoinedColumnTypeToNullIfNeeded(
+        const QueryTreeNodePtr & resolved_identifier,
+        const JoinKind & join_kind,
+        std::optional<JoinTableSide> resolved_side,
+        IdentifierResolveScope & scope)
     {
         if (resolved_identifier->getNodeType() == QueryTreeNodeType::COLUMN &&
             JoinCommon::canBecomeNullable(resolved_identifier->getResultType()) &&
@@ -1284,16 +1289,20 @@ private:
             (isLeft(join_kind) && resolved_side && *resolved_side == JoinTableSide::Right) ||
             (isRight(join_kind) && resolved_side && *resolved_side == JoinTableSide::Left)))
         {
-            auto & resolved_column = resolved_identifier->as<ColumnNode &>();
+            auto nullable_resolved_identifier = resolved_identifier->clone();
+            auto & resolved_column = nullable_resolved_identifier->as<ColumnNode &>();
             auto new_result_type = makeNullableOrLowCardinalityNullable(resolved_column.getColumnType());
             resolved_column.setColumnType(new_result_type);
             if (resolved_column.hasExpression())
             {
                 auto & resolved_expression = resolved_column.getExpression();
                 if (!resolved_expression->getResultType()->equals(*new_result_type))
-                    resolved_expression = buildCastFunction(resolved_expression, new_result_type, context, true);
+                    resolved_expression = buildCastFunction(resolved_expression, new_result_type, scope.context, true);
             }
+            scope.nullable_join_columns[nullable_resolved_identifier] = resolved_identifier;
+            return nullable_resolved_identifier;
         }
+        return nullptr;
     }
 
     /// Resolve identifier functions
@@ -3534,8 +3543,9 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo
 
     if (scope.join_use_nulls)
     {
-        resolved_identifier = resolved_identifier->clone();
-        convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side, scope.context);
+        auto nullable_resolved_identifier = convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side, scope);
+        if (nullable_resolved_identifier)
+            resolved_identifier = nullable_resolved_identifier;
     }
 
     return resolved_identifier;
@@ -4428,10 +4438,7 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher(
                         {
                             const auto & table_expression_data = table_expression_data_it->second;
                             const auto & column_name = using_column_from_table.getColumnName();
-                            if (!table_expression_data.column_name_to_column_node.contains(column_name))
-                            {
-                                return true;
-                            }
+                            return !table_expression_data.column_name_to_column_node.contains(column_name);
                         }
                         return false;
                     };
@@ -4562,12 +4569,14 @@ ProjectionNames QueryAnalyzer::resolveMatcher(QueryTreeNodePtr & matcher_node, I
             {
                 auto join_identifier_side = getColumnSideFromJoinTree(node, *nearest_scope_join_node);
                 auto projection_name_it = node_to_projection_name.find(node);
-                node = node->clone();
-                /// Set the same projection name for new nullable node
-                if (projection_name_it != node_to_projection_name.end())
-                    node_to_projection_name.emplace(node, projection_name_it->second);
-
-                convertJoinedColumnTypeToNullIfNeeded(node, nearest_scope_join_node->getKind(), join_identifier_side, scope.context);
+                auto nullable_node = convertJoinedColumnTypeToNullIfNeeded(node, nearest_scope_join_node->getKind(), join_identifier_side, scope);
+                if (nullable_node)
+                {
+                    node = nullable_node;
+                    /// Set the same projection name for new nullable node
+                    if (projection_name_it != node_to_projection_name.end())
+                        node_to_projection_name.emplace(node, projection_name_it->second);
+                }
             }
         }
     }
@@ -7526,6 +7535,29 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node,
     scope.table_expressions_in_resolve_process.erase(join_tree_node.get());
 }
 
+class ReplaceColumnsVisitor : public InDepthQueryTreeVisitor<ReplaceColumnsVisitor>
+{
+public:
+    explicit ReplaceColumnsVisitor(const QueryTreeNodePtrWithHashMap<QueryTreeNodePtr> & replacement_map_, const ContextPtr & context_)
+        : replacement_map(replacement_map_)
+        , context(context_)
+    {}
+
+    void visitImpl(QueryTreeNodePtr & node)
+    {
+        if (auto it = replacement_map.find(node); it != replacement_map.end())
+            node = it->second;
+        if (auto * function_node = node->as<FunctionNode>())
+            rerunFunctionResolve(function_node, context);
+    }
+
+    bool shouldTraverseTopToBottom() const { return false; }
+
+private:
+    const QueryTreeNodePtrWithHashMap<QueryTreeNodePtr> & replacement_map;
+    const ContextPtr & context;
+};
+
 /** Resolve query.
   * This function modifies query node during resolve. It is caller responsibility to clone query node before resolve
   * if it is needed for later use.
@@ -7713,21 +7745,23 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier
                 scope.scope_node->formatASTForErrorMessage());
     }
 
-    if (query_node_typed.getPrewhere())
+    if (auto & prewhere_node = query_node_typed.getPrewhere())
     {
-        /** Expression in PREWHERE with JOIN should not be modified by join_use_nulls.
-          * Example: SELECT * FROM t1 JOIN t2 USING (id) PREWHERE a = 1
-          * Column `a` should be resolved from table and should not change its type to Nullable.
-          */
-        bool join_use_nulls = scope.join_use_nulls;
-        bool use_identifier_lookup_to_result_cache = scope.use_identifier_lookup_to_result_cache;
-        scope.join_use_nulls = false;
-        scope.use_identifier_lookup_to_result_cache = false;
+        resolveExpressionNode(prewhere_node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
 
-        resolveExpressionNode(query_node_typed.getPrewhere(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
-
-        scope.join_use_nulls = join_use_nulls;
-        scope.use_identifier_lookup_to_result_cache = use_identifier_lookup_to_result_cache;
+        if (scope.join_use_nulls)
+        {
+            /** Expression in PREWHERE with JOIN should not be modified by join_use_nulls.
+              * Example: SELECT * FROM t1 JOIN t2 USING (id) PREWHERE b = 1
+              * Column `a` should be resolved from table and should not change its type to Nullable.
+              * More complicated example when column is somewhere inside an expression:
+              * SELECT a + 1 as b FROM t1 JOIN t2 USING (id) PREWHERE b = 1
+              * expression `a + 1 as b` in projection and in PREWHERE should have different `a`.
+              */
+            prewhere_node = prewhere_node->clone();
+            ReplaceColumnsVisitor replace_visitor(scope.nullable_join_columns, scope.context);
+            replace_visitor.visit(prewhere_node);
+        }
     }
 
     if (query_node_typed.getWhere())
diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.reference b/tests/queries/0_stateless/02989_join_using_parent_scope.reference
index c396431049f..cf1ae93936f 100644
--- a/tests/queries/0_stateless/02989_join_using_parent_scope.reference
+++ b/tests/queries/0_stateless/02989_join_using_parent_scope.reference
@@ -25,43 +25,43 @@ SELECT 1 AS b FROM tb JOIN ta USING (b); -- { serverError UNKNOWN_IDENTIFIER }
 -- SELECT * works returns all columns from both tables in new analyzer
 SET allow_experimental_analyzer = 1;
 SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL;
-3	3	0	3
-3	3	1	3
-3	3	2	3
-3	3	3	3
-3	3	\N	0
-3	3	\N	1
-3	3	\N	2
+3	3	0	3	abc3
+3	3	1	3	abc3
+3	3	2	3	abc3
+3	3	3	3	abc3
+3	3	\N	0	abc0
+3	3	\N	1	abc1
+3	3	\N	2	abc2
 SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL;
-1	0	1
-2	1	2
-3	2	3
+1	0	1	abc1
+2	1	2	abc2
+3	2	3	abc3
 SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL;
-1	0	1
-2	1	2
-3	2	3
+1	0	1	abc1
+2	1	2	abc2
+3	2	3	abc3
 SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL;
-1	0	1
-2	1	2
-3	2	3
-4	3	\N
+1	0	1	abc1
+2	1	2	abc2
+3	2	3	abc3
+4	3	\N	\N
 SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL;
-1	0	1
-2	1	2
-3	2	3
-\N	\N	0
+1	0	1	abc1
+2	1	2	abc2
+3	2	3	abc3
+\N	\N	0	abc0
 SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL;
-1	0	1
-2	1	2
-3	2	3
-4	3	\N
-\N	\N	0
+1	0	1	abc1
+2	1	2	abc2
+3	2	3	abc3
+4	3	\N	\N
+\N	\N	0	abc0
 SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1;
-1	0	1	2	3
-2	1	2	3	4
-3	2	3	4	5
-4	3	\N	\N	\N
-\N	\N	0	1	2
+1	0	1	abc1	2	3
+2	1	2	abc2	3	4
+3	2	3	abc3	4	5
+4	3	\N	\N	\N	\N
+\N	\N	0	abc0	1	2
 SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
 1	0	1	2
 2	1	2	3
@@ -82,6 +82,12 @@ SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc
 3	2	3	4
 4	3	\N	\N
 \N	\N	0	1
+SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL;
+3	abc3
+4	\N
+\N	abc0
+\N	abc1
+\N	abc2
 DROP TABLE IF EXISTS tabc;
 DROP TABLE IF EXISTS ta;
 DROP TABLE IF EXISTS tb;
diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.sql b/tests/queries/0_stateless/02989_join_using_parent_scope.sql
index 05f1137f60e..9c8501f6b00 100644
--- a/tests/queries/0_stateless/02989_join_using_parent_scope.sql
+++ b/tests/queries/0_stateless/02989_join_using_parent_scope.sql
@@ -1,6 +1,6 @@
 DROP TABLE IF EXISTS tabc;
-CREATE TABLE tabc (a UInt32, b UInt32 ALIAS a + 1, c UInt32 ALIAS b + 1) ENGINE = MergeTree ORDER BY a;
-INSERT INTO tabc SELECT number FROM numbers(4);
+CREATE TABLE tabc (a UInt32, b UInt32 ALIAS a + 1, c UInt32 ALIAS b + 1, s String) ENGINE = MergeTree ORDER BY a;
+INSERT INTO tabc (a, s) SELECT number, 'abc' || toString(number) FROM numbers(4);
 
 DROP TABLE IF EXISTS ta;
 CREATE TABLE ta (a Int32) ENGINE = MergeTree ORDER BY tuple();
@@ -38,6 +38,7 @@ SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc
 SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
 SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
 
+SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL;
 
 DROP TABLE IF EXISTS tabc;
 DROP TABLE IF EXISTS ta;

From dbed5d1902eee94d2db9596c74ccf0b7335bb293 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 13 Mar 2024 12:14:14 +0000
Subject: [PATCH 51/92] Add
 analyzer_compatibility_join_using_top_level_identifier

---
 src/Analyzer/Passes/QueryAnalysisPass.cpp     | 90 ++++++++++++-------
 src/Core/Settings.h                           |  1 +
 src/Core/SettingsChangesHistory.h             |  1 +
 .../02989_join_using_parent_scope.reference   | 70 ++++++++++++---
 .../02989_join_using_parent_scope.sql         | 68 +++++++++++---
 5 files changed, 168 insertions(+), 62 deletions(-)

diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp
index 35b6f05191e..0414ea93f94 100644
--- a/src/Analyzer/Passes/QueryAnalysisPass.cpp
+++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp
@@ -7357,48 +7357,70 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS
 
             join_using_identifiers.insert(identifier_full_name);
 
-            IdentifierLookup identifier_lookup{identifier_node->getIdentifier(), IdentifierLookupContext::EXPRESSION};
-            auto result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getLeftTableExpression(), scope);
-            if (!result_left_table_expression)
+            const auto & settings = scope.context->getSettingsRef();
+
+            /** While resolving JOIN USING identifier, try to resolve identifier from parent subquery projection.
+              * Example: SELECT a + 1 AS b FROM (SELECT 1 AS a) t1 JOIN (SELECT 2 AS b) USING b
+              * In this case `b` is not in the left table expression, but it is in the parent subquery projection.
+              */
+            auto try_resolve_identifier_from_query_projection = [this](const String & identifier_full_name_,
+                                                                       const QueryTreeNodePtr & left_table_expression,
+                                                                       const IdentifierResolveScope & scope_) -> QueryTreeNodePtr
             {
-                /** Try to resolve identifier from parent subquery projection.
-                  * Example: SELECT a + 1 AS v FROM (SELECT 1 AS a) t1 JOIN (SELECT 2 AS b) USING b
-                  * In this case `b` is not in the left table expression, but it is in the parent subquery projection.
-                  */
-                QueryNode * query_node = scope.scope_node ? scope.scope_node->as<QueryNode>() : nullptr;
-                if (query_node)
+                const QueryNode * query_node = scope_.scope_node ? scope_.scope_node->as<QueryNode>() : nullptr;
+                if (!query_node)
+                    return nullptr;
+
+                const auto & projection_list = query_node->getProjection();
+                for (const auto & projection_node : projection_list.getNodes())
                 {
-                    const auto & projection_list = query_node->getProjection();
-                    for (const auto & projection_node : projection_list.getNodes())
+                    if (projection_node->hasAlias() && identifier_full_name_ == projection_node->getAlias())
                     {
-                        if (projection_node->hasAlias() && identifier_full_name == projection_node->getAlias())
+                        auto left_subquery = std::make_shared<QueryNode>(query_node->getMutableContext());
+                        left_subquery->getProjection().getNodes().push_back(projection_node->clone());
+                        left_subquery->getJoinTree() = left_table_expression;
+
+                        IdentifierResolveScope left_subquery_scope(left_subquery, nullptr /*parent_scope*/);
+                        resolveQuery(left_subquery, left_subquery_scope);
+
+                        const auto & resolved_nodes = left_subquery->getProjection().getNodes();
+                        if (resolved_nodes.size() == 1)
                         {
-                            auto left_subquery = std::make_shared<QueryNode>(query_node->getMutableContext());
-                            left_subquery->getProjection().getNodes().push_back(projection_node->clone());
-                            left_subquery->getJoinTree() = join_node_typed.getLeftTableExpression();
-
-                            IdentifierResolveScope left_subquery_scope(left_subquery, nullptr /*parent_scope*/);
-                            resolveQuery(left_subquery, left_subquery_scope);
-
-                            const auto & resolved_nodes = left_subquery->getProjection().getNodes();
-                            if (resolved_nodes.size() == 1)
-                            {
-                                /// Create ColumnNode with expression from parent projection
-                                result_left_table_expression = std::make_shared<ColumnNode>(
-                                    NameAndTypePair{identifier_full_name, resolved_nodes.at(0)->getResultType()}, resolved_nodes.at(0), join_node_typed.getLeftTableExpression());
-                                break;
-                            }
+                            /// Create ColumnNode with expression from parent projection
+                            return std::make_shared<ColumnNode>(
+                                NameAndTypePair{identifier_full_name_, resolved_nodes.front()->getResultType()},
+                                resolved_nodes.front(), left_table_expression);
                         }
                     }
                 }
+                return nullptr;
+            };
 
-                if (!result_left_table_expression)
-                    throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER,
-                        "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}",
-                        join_node_typed.formatASTForErrorMessage(),
-                        identifier_full_name,
-                        scope.scope_node->formatASTForErrorMessage());
-            }
+            QueryTreeNodePtr result_left_table_expression = nullptr;
+            /** With `analyzer_compatibility_join_using_top_level_identifier` alias in projection has higher priority than column from left table.
+              * But if aliased expression cannot be resolved from left table, we get UNKNOW_IDENTIFIER error,
+              * despite the fact that column from USING could be resolved from left table.
+              * It's compatibility with a default behavior for old analyzer.
+              */
+            if (settings.analyzer_compatibility_join_using_top_level_identifier)
+                result_left_table_expression = try_resolve_identifier_from_query_projection(identifier_full_name, join_node_typed.getLeftTableExpression(), scope);
+
+            IdentifierLookup identifier_lookup{identifier_node->getIdentifier(), IdentifierLookupContext::EXPRESSION};
+            if (!result_left_table_expression)
+                result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getLeftTableExpression(), scope);
+
+            /// Here we may try to resolve identifier from projection in case it's not resolved from left table expression
+            /// and analyzer_compatibility_join_using_top_level_identifier is disabled.
+            /// For now we do not do this, because not all corner cases are clear.
+            /// if (!settings.analyzer_compatibility_join_using_top_level_identifier && !result_left_table_expression)
+            ///     result_left_table_expression = try_resolve_identifier_from_query_projection(identifier_full_name, join_node_typed.getLeftTableExpression(), scope);
+
+            if (!result_left_table_expression)
+                throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER,
+                    "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}",
+                    join_node_typed.formatASTForErrorMessage(),
+                    identifier_full_name,
+                    scope.scope_node->formatASTForErrorMessage());
 
             if (result_left_table_expression->getNodeType() != QueryTreeNodeType::COLUMN)
                 throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index e6adb00137f..bad130d6ef4 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -371,6 +371,7 @@ class IColumn;
     M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \
     M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \
     M(Bool, allow_experimental_analyzer, false, "Allow experimental analyzer", 0) \
+    M(Bool, analyzer_compatibility_join_using_top_level_identifier, false, "Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`).", 0) \
     M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \
     M(Bool, enable_vertical_final, true, "If enable, remove duplicated rows during FINAL by marking rows as deleted and filtering them later instead of merging rows", 0) \
     \
diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h
index 8e2b2915c2a..07a1c88363a 100644
--- a/src/Core/SettingsChangesHistory.h
+++ b/src/Core/SettingsChangesHistory.h
@@ -96,6 +96,7 @@ static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> sett
               {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
               {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"},
               {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"},
+              {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"},
               }},
     {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
               {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},
diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.reference b/tests/queries/0_stateless/02989_join_using_parent_scope.reference
index cf1ae93936f..965bfaf7c43 100644
--- a/tests/queries/0_stateless/02989_join_using_parent_scope.reference
+++ b/tests/queries/0_stateless/02989_join_using_parent_scope.reference
@@ -22,9 +22,8 @@ SELECT b AS a, a FROM tb JOIN tabc USING (a) ORDER BY ALL;
 2	2
 3	3
 SELECT 1 AS b FROM tb JOIN ta USING (b); -- { serverError UNKNOWN_IDENTIFIER }
--- SELECT * works returns all columns from both tables in new analyzer
-SET allow_experimental_analyzer = 1;
-SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL;
+-- SELECT * returns all columns from both tables in new analyzer
+SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
 3	3	0	3	abc3
 3	3	1	3	abc3
 3	3	2	3	abc3
@@ -32,62 +31,105 @@ SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL;
 3	3	\N	0	abc0
 3	3	\N	1	abc1
 3	3	\N	2	abc2
-SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
 1	0	1	abc1
 2	1	2	abc2
 3	2	3	abc3
-SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
 1	0	1	abc1
 2	1	2	abc2
 3	2	3	abc3
-SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
 1	0	1	abc1
 2	1	2	abc2
 3	2	3	abc3
 4	3	\N	\N
-SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
 1	0	1	abc1
 2	1	2	abc2
 3	2	3	abc3
 \N	\N	0	abc0
-SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
 1	0	1	abc1
 2	1	2	abc2
 3	2	3	abc3
 4	3	\N	\N
 \N	\N	0	abc0
-SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1;
+SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1, allow_experimental_analyzer = 1;
 1	0	1	abc1	2	3
 2	1	2	abc2	3	4
 3	2	3	abc3	4	5
 4	3	\N	\N	\N	\N
 \N	\N	0	abc0	1	2
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
 1	0	1	2
 2	1	2	3
 3	2	3	4
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
 1	0	1	2
 2	1	2	3
 3	2	3	4
 4	3	\N	\N
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
 1	0	1	2
 2	1	2	3
 3	2	3	4
 \N	\N	0	1
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
 1	0	1	2
 2	1	2	3
 3	2	3	4
 4	3	\N	\N
 \N	\N	0	1
-SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL;
+SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
 3	abc3
 4	\N
 \N	abc0
 \N	abc1
 \N	abc2
+-- It's a default behavior for old analyzer and new with analyzer_compatibility_join_using_top_level_identifier
+-- Column `b` actually exists in left table, but `b` from USING is resoled to `a + 2` and `a` is not in left table
+-- so we get UNKNOWN_IDENTIFIER error.
+SELECT a + 2 AS b FROM tb JOIN tabc USING (b) ORDER BY ALL
+SETTINGS analyzer_compatibility_join_using_top_level_identifier = 1; -- { serverError UNKNOWN_IDENTIFIER }
+-- In new analyzer with `analyzer_compatibility_join_using_top_level_identifier = 0` we get `b` from left table
+SELECT a + 2 AS b FROM tb JOIN tabc USING (b) ORDER BY ALL
+SETTINGS analyzer_compatibility_join_using_top_level_identifier = 0, allow_experimental_analyzer = 1;
+2
+3
+4
+-- This is example where query may return different results with different `analyzer_compatibility_join_using_top_level_identifier`
+
+DROP TABLE IF EXISTS users;
+CREATE TABLE users (uid Int16, name String, spouse_name String) ENGINE=Memory;
+INSERT INTO users VALUES (1231, 'John', 'Ksenia');
+INSERT INTO users VALUES (6666, 'Ksenia', '');
+SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name
+FROM users u1 JOIN users u2 USING (name)
+ORDER BY u1.uid
+FORMAT TSVWithNamesAndTypes
+SETTINGS allow_experimental_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 1;
+uid	name	u2.uid	u2.name
+Int16	String	Int16	String
+1231	Ksenia	6666	Ksenia
+SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name
+FROM users u1 JOIN users u2 USING (name)
+ORDER BY u1.uid
+FORMAT TSVWithNamesAndTypes
+SETTINGS allow_experimental_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 0;
+uid	name	u2.uid	u2.name
+Int16	String	Int16	String
+1231	Ksenia	1231	John
+6666		6666	Ksenia
+SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name
+FROM users u1 JOIN users u2 USING (name)
+ORDER BY u1.uid
+FORMAT TSVWithNamesAndTypes
+SETTINGS allow_experimental_analyzer = 0;
+uid	name	u2.uid	u2.name
+Int16	String	Int16	String
+1231	Ksenia	6666	Ksenia
+DROP TABLE IF EXISTS users;
 DROP TABLE IF EXISTS tabc;
 DROP TABLE IF EXISTS ta;
 DROP TABLE IF EXISTS tb;
diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.sql b/tests/queries/0_stateless/02989_join_using_parent_scope.sql
index 9c8501f6b00..2e4d9f097f7 100644
--- a/tests/queries/0_stateless/02989_join_using_parent_scope.sql
+++ b/tests/queries/0_stateless/02989_join_using_parent_scope.sql
@@ -12,6 +12,8 @@ INSERT INTO tb SELECT number FROM numbers(4);
 
 SET join_use_nulls = 1;
 
+SET analyzer_compatibility_join_using_top_level_identifier = 1;
+
 -- { echoOn }
 SELECT 1 AS a FROM tb JOIN tabc USING (a) ORDER BY ALL;
 SELECT a + 2 AS b FROM ta JOIN tabc USING (b) ORDER BY ALL;
@@ -21,24 +23,62 @@ SELECT a + 2 AS c FROM ta JOIN tabc USING (c) ORDER BY ALL;
 SELECT b AS a, a FROM tb JOIN tabc USING (a) ORDER BY ALL;
 SELECT 1 AS b FROM tb JOIN ta USING (b); -- { serverError UNKNOWN_IDENTIFIER }
 
--- SELECT * works returns all columns from both tables in new analyzer
-SET allow_experimental_analyzer = 1;
+-- SELECT * returns all columns from both tables in new analyzer
+SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
 
-SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL;
-SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1, allow_experimental_analyzer = 1;
 
-SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL;
-SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL;
-SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL;
-SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL;
-SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
 
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL;
+SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+
+
+-- It's a default behavior for old analyzer and new with analyzer_compatibility_join_using_top_level_identifier
+-- Column `b` actually exists in left table, but `b` from USING is resoled to `a + 2` and `a` is not in left table
+-- so we get UNKNOWN_IDENTIFIER error.
+SELECT a + 2 AS b FROM tb JOIN tabc USING (b) ORDER BY ALL
+SETTINGS analyzer_compatibility_join_using_top_level_identifier = 1; -- { serverError UNKNOWN_IDENTIFIER }
+
+-- In new analyzer with `analyzer_compatibility_join_using_top_level_identifier = 0` we get `b` from left table
+SELECT a + 2 AS b FROM tb JOIN tabc USING (b) ORDER BY ALL
+SETTINGS analyzer_compatibility_join_using_top_level_identifier = 0, allow_experimental_analyzer = 1;
+
+-- This is example where query may return different results with different `analyzer_compatibility_join_using_top_level_identifier`
+
+DROP TABLE IF EXISTS users;
+CREATE TABLE users (uid Int16, name String, spouse_name String) ENGINE=Memory;
+
+INSERT INTO users VALUES (1231, 'John', 'Ksenia');
+INSERT INTO users VALUES (6666, 'Ksenia', '');
+
+SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name
+FROM users u1 JOIN users u2 USING (name)
+ORDER BY u1.uid
+FORMAT TSVWithNamesAndTypes
+SETTINGS allow_experimental_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 1;
+
+SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name
+FROM users u1 JOIN users u2 USING (name)
+ORDER BY u1.uid
+FORMAT TSVWithNamesAndTypes
+SETTINGS allow_experimental_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 0;
+
+SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name
+FROM users u1 JOIN users u2 USING (name)
+ORDER BY u1.uid
+FORMAT TSVWithNamesAndTypes
+SETTINGS allow_experimental_analyzer = 0;
+
+DROP TABLE IF EXISTS users;
 
-SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL;
 
 DROP TABLE IF EXISTS tabc;
 DROP TABLE IF EXISTS ta;

From 589c784e7bddfba89248bde6a2957457f9fa2538 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Mon, 18 Mar 2024 11:12:14 +0000
Subject: [PATCH 52/92] upd 01062_pm_all_join_with_block_continuation

---
 .../01062_pm_all_join_with_block_continuation.sql           | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql b/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql
index 15a28e92386..6a42c725e4a 100644
--- a/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql
+++ b/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql
@@ -1,8 +1,8 @@
 SET max_memory_usage = 50000000;
 SET join_algorithm = 'partial_merge';
-
+SET analyzer_compatibility_join_using_top_level_identifier = 1;
 SELECT 'defaults';
-    
+
 SELECT count(1) FROM (
     SELECT materialize(1) as k, n FROM numbers(10) nums
     JOIN (SELECT materialize(1) AS k, number n FROM numbers(1000000)) j
@@ -20,7 +20,7 @@ SELECT count(1), uniqExact(n) FROM (
 
 -- errors
 SET max_joined_block_size_rows = 0;
-    
+
 SELECT count(1) FROM (
     SELECT materialize(1) as k, n FROM numbers(10) nums
     JOIN (SELECT materialize(1) AS k, number n FROM numbers(1000000)) j

From 718e092c9189285eaf33c13899fa5cdcbf50a6b5 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Mon, 18 Mar 2024 12:09:21 +0100
Subject: [PATCH 53/92] Make dynamic resize of cache non blocking

---
 src/Interpreters/Cache/FileCache.cpp          | 23 ++++++++++++++++---
 src/Interpreters/Cache/FileCache.h            |  1 +
 src/Interpreters/Cache/IFileCachePriority.h   |  4 ++++
 src/Interpreters/Cache/LRUFileCachePriority.h |  4 ++++
 .../Cache/SLRUFileCachePriority.cpp           | 10 ++++++++
 .../Cache/SLRUFileCachePriority.h             |  4 ++++
 6 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index 65b6a3a172d..5c1d82b90cd 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -783,6 +783,17 @@ bool FileCache::tryReserve(
     ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::FilesystemCacheReserveMicroseconds);
 
     assertInitialized();
+
+    /// A logical race on cache_is_being_resized is still possible,
+    /// in this case we will try to lock cache with timeout, this is ok, timeout is small
+    /// and as resizing of cache can take a long time then this small chance of a race is
+    /// ok compared to the number of cases this check will help.
+    if (cache_is_being_resized.load())
+    {
+        ProfileEvents::increment(ProfileEvents::FilesystemCacheFailToReserveSpaceBecauseOfLockContention);
+        return false;
+    }
+
     auto cache_lock = tryLockCache(std::chrono::milliseconds(lock_wait_timeout_milliseconds));
     if (!cache_lock)
     {
@@ -1264,12 +1275,14 @@ std::vector<String> FileCache::tryGetCachePaths(const Key & key)
 
 size_t FileCache::getUsedCacheSize() const
 {
-    return main_priority->getSize(lockCache());
+    /// We use this method for metrics, so it is ok to get approximate result.
+    return main_priority->getSizeApprox();
 }
 
 size_t FileCache::getFileSegmentsNum() const
 {
-    return main_priority->getElementsCount(lockCache());
+    /// We use this method for metrics, so it is ok to get approximate result.
+    return main_priority->getElementsCountApprox();
 }
 
 void FileCache::assertCacheCorrectness()
@@ -1327,8 +1340,12 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings,
     if (new_settings.max_size != actual_settings.max_size
         || new_settings.max_elements != actual_settings.max_elements)
     {
-        auto cache_lock = lockCache();
+        cache_is_being_resized.store(true);
+        SCOPE_EXIT({
+            cache_is_being_resized.store(false);
+        });
 
+        auto cache_lock = lockCache();
         bool updated = false;
         try
         {
diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h
index 8ea5f4dab40..1433a067e7e 100644
--- a/src/Interpreters/Cache/FileCache.h
+++ b/src/Interpreters/Cache/FileCache.h
@@ -202,6 +202,7 @@ private:
     mutable std::mutex init_mutex;
     std::unique_ptr<StatusFile> status_file;
     std::atomic<bool> shutdown = false;
+    std::atomic<bool> cache_is_being_resized = false;
 
     std::mutex apply_settings_mutex;
 
diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h
index 58011780323..09d71cebb01 100644
--- a/src/Interpreters/Cache/IFileCachePriority.h
+++ b/src/Interpreters/Cache/IFileCachePriority.h
@@ -63,8 +63,12 @@ public:
 
     virtual size_t getSize(const CachePriorityGuard::Lock &) const = 0;
 
+    virtual size_t getSizeApprox() const = 0;
+
     virtual size_t getElementsCount(const CachePriorityGuard::Lock &) const = 0;
 
+    virtual size_t getElementsCountApprox() const = 0;
+
     /// Throws exception if there is not enough size to fit it.
     virtual IteratorPtr add( /// NOLINT
         KeyMetadataPtr key_metadata,
diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h
index dcd4ee0a24c..49977c79b81 100644
--- a/src/Interpreters/Cache/LRUFileCachePriority.h
+++ b/src/Interpreters/Cache/LRUFileCachePriority.h
@@ -28,6 +28,10 @@ public:
 
     size_t getElementsCount(const CachePriorityGuard::Lock &) const override { return state->current_elements_num; }
 
+    size_t getSizeApprox() const override { return state->current_size; }
+
+    size_t getElementsCountApprox() const override { return state->current_elements_num; }
+
     bool canFit( /// NOLINT
         size_t size,
         const CachePriorityGuard::Lock &,
diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp
index 1767cb94be7..c97d05d4b84 100644
--- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp
+++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp
@@ -44,6 +44,16 @@ size_t SLRUFileCachePriority::getElementsCount(const CachePriorityGuard::Lock &
     return protected_queue.getElementsCount(lock) + probationary_queue.getElementsCount(lock);
 }
 
+size_t SLRUFileCachePriority::getSizeApprox() const
+{
+    return protected_queue.getSizeApprox() + probationary_queue.getSizeApprox();
+}
+
+size_t SLRUFileCachePriority::getElementsCountApprox() const
+{
+    return protected_queue.getElementsCountApprox() + probationary_queue.getElementsCountApprox();
+}
+
 bool SLRUFileCachePriority::canFit( /// NOLINT
     size_t size,
     const CachePriorityGuard::Lock & lock,
diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.h b/src/Interpreters/Cache/SLRUFileCachePriority.h
index d81ce1bc480..f90918f7878 100644
--- a/src/Interpreters/Cache/SLRUFileCachePriority.h
+++ b/src/Interpreters/Cache/SLRUFileCachePriority.h
@@ -25,6 +25,10 @@ public:
 
     size_t getElementsCount(const CachePriorityGuard::Lock &) const override;
 
+    size_t getSizeApprox() const override;
+
+    size_t getElementsCountApprox() const override;
+
     bool canFit( /// NOLINT
         size_t size,
         const CachePriorityGuard::Lock &,

From c6e86e3bb33cbd7e2f86d41e4726ffdd5c4545b7 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Mon, 18 Mar 2024 13:06:39 +0100
Subject: [PATCH 54/92] Review suggestion

---
 src/Interpreters/Cache/FileCache.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index 5c1d82b90cd..916bdb8f898 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -788,7 +788,7 @@ bool FileCache::tryReserve(
     /// in this case we will try to lock cache with timeout, this is ok, timeout is small
     /// and as resizing of cache can take a long time then this small chance of a race is
     /// ok compared to the number of cases this check will help.
-    if (cache_is_being_resized.load())
+    if (cache_is_being_resized.load(std::memory_order_relaxed))
     {
         ProfileEvents::increment(ProfileEvents::FilesystemCacheFailToReserveSpaceBecauseOfLockContention);
         return false;
@@ -1340,9 +1340,9 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings,
     if (new_settings.max_size != actual_settings.max_size
         || new_settings.max_elements != actual_settings.max_elements)
     {
-        cache_is_being_resized.store(true);
+        cache_is_being_resized.store(true, std::memory_order_relaxed);
         SCOPE_EXIT({
-            cache_is_being_resized.store(false);
+            cache_is_being_resized.store(false, std::memory_order_relaxed);
         });
 
         auto cache_lock = lockCache();

From e208907f39b1bcccc16d56cc672a30282c9223b6 Mon Sep 17 00:00:00 2001
From: PapaToemmsn <160967948+PapaToemmsn@users.noreply.github.com>
Date: Thu, 22 Feb 2024 22:46:13 +0100
Subject: [PATCH 55/92] fix for #57598 (ROLLBACK should be no-op...)

Change to verify proper behaviour for ROLLBACK should be no-op when no active tranaction is running (MySQL compatibility) #57598
---
 src/Interpreters/InterpreterTransactionControlQuery.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/InterpreterTransactionControlQuery.cpp b/src/Interpreters/InterpreterTransactionControlQuery.cpp
index b1758013f18..fc71874f7a8 100644
--- a/src/Interpreters/InterpreterTransactionControlQuery.cpp
+++ b/src/Interpreters/InterpreterTransactionControlQuery.cpp
@@ -52,7 +52,7 @@ BlockIO InterpreterTransactionControlQuery::executeCommit(ContextMutablePtr sess
 {
     auto txn = session_context->getCurrentTransaction();
     if (!txn)
-        throw Exception(ErrorCodes::INVALID_TRANSACTION, "There is no current transaction");
+        return {};
     if (txn->getState() != MergeTreeTransaction::RUNNING)
         throw Exception(ErrorCodes::INVALID_TRANSACTION, "Transaction is not in RUNNING state");
 
@@ -111,7 +111,7 @@ BlockIO InterpreterTransactionControlQuery::executeRollback(ContextMutablePtr se
 {
     auto txn = session_context->getCurrentTransaction();
     if (!txn)
-        throw Exception(ErrorCodes::INVALID_TRANSACTION, "There is no current transaction");
+        return {};
     if (txn->getState() == MergeTreeTransaction::COMMITTED)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Transaction is in COMMITTED state");
     if (txn->getState() == MergeTreeTransaction::COMMITTING)

From c68fd07cff040e567df3affd6d2de34eaf6e98e3 Mon Sep 17 00:00:00 2001
From: PapaToemmsn <160967948+PapaToemmsn@users.noreply.github.com>
Date: Fri, 23 Feb 2024 10:57:45 +0100
Subject: [PATCH 56/92] adjusted 01173_transaction_control_queries.sql to
 changed no transaction active bahaviour

Adapted transaction test queries to changed beahviour
---
 .../queries/0_stateless/01173_transaction_control_queries.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01173_transaction_control_queries.sql b/tests/queries/0_stateless/01173_transaction_control_queries.sql
index 03c98f50cc4..afad79c6353 100644
--- a/tests/queries/0_stateless/01173_transaction_control_queries.sql
+++ b/tests/queries/0_stateless/01173_transaction_control_queries.sql
@@ -6,8 +6,8 @@ drop table if exists mt2;
 create table mt1 (n Int64) engine=MergeTree order by n;
 create table mt2 (n Int64) engine=MergeTree order by n;
 
-commit; -- { serverError INVALID_TRANSACTION } -- no transaction
-rollback; -- { serverError INVALID_TRANSACTION }
+commit;
+rollback;
 
 begin transaction;
 insert into mt1 values (1);

From 6eafc21211d6123c4ff51ab5af577fd67aa53044 Mon Sep 17 00:00:00 2001
From: PapaToemmsn <160967948+PapaToemmsn@users.noreply.github.com>
Date: Wed, 13 Mar 2024 16:26:11 +0100
Subject: [PATCH 57/92] Reveted 01173_transaction_control_queries.sql back to
 original state

As changes in Txn handling for COMMIT and ROLLBACK should only be honoured for MySQL client mode the test was reverted back
---
 .../queries/0_stateless/01173_transaction_control_queries.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01173_transaction_control_queries.sql b/tests/queries/0_stateless/01173_transaction_control_queries.sql
index afad79c6353..03c98f50cc4 100644
--- a/tests/queries/0_stateless/01173_transaction_control_queries.sql
+++ b/tests/queries/0_stateless/01173_transaction_control_queries.sql
@@ -6,8 +6,8 @@ drop table if exists mt2;
 create table mt1 (n Int64) engine=MergeTree order by n;
 create table mt2 (n Int64) engine=MergeTree order by n;
 
-commit;
-rollback;
+commit; -- { serverError INVALID_TRANSACTION } -- no transaction
+rollback; -- { serverError INVALID_TRANSACTION }
 
 begin transaction;
 insert into mt1 values (1);

From 723f860333e767a693b064104c8c2b48f9d975bd Mon Sep 17 00:00:00 2001
From: PapaToemmsn <160967948+PapaToemmsn@users.noreply.github.com>
Date: Wed, 13 Mar 2024 18:06:46 +0100
Subject: [PATCH 58/92] COMMIT/ROLLBACK with no active transaction do not throw
 an exception for MySQL clients anymore

---
 .../0_stateless/02989_mysql_transaction_test.sh        | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 tests/queries/0_stateless/02989_mysql_transaction_test.sh

diff --git a/tests/queries/0_stateless/02989_mysql_transaction_test.sh b/tests/queries/0_stateless/02989_mysql_transaction_test.sh
new file mode 100644
index 00000000000..ffb113255f7
--- /dev/null
+++ b/tests/queries/0_stateless/02989_mysql_transaction_test.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+# Tag no-fasttest: requires mysql client
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${MYSQL_CLIENT} --verbose --execute "COMMIT;" | grep -c "COMMIT"
+${MYSQL_CLIENT} --verbose --execute "ROLLBACK;" | grep -c "ROLLBACK"

From 36e53007de9649c72165fb5283369a1b1be1f903 Mon Sep 17 00:00:00 2001
From: PapaToemmsn <160967948+PapaToemmsn@users.noreply.github.com>
Date: Wed, 13 Mar 2024 18:07:41 +0100
Subject: [PATCH 59/92] COMMIT/ROLLBACK with no active transaction do not throw
 an exception for MySQL clients anymore

---
 .../queries/0_stateless/02989_mysql_transaction_test.reference  | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 tests/queries/0_stateless/02989_mysql_transaction_test.reference

diff --git a/tests/queries/0_stateless/02989_mysql_transaction_test.reference b/tests/queries/0_stateless/02989_mysql_transaction_test.reference
new file mode 100644
index 00000000000..6ed281c757a
--- /dev/null
+++ b/tests/queries/0_stateless/02989_mysql_transaction_test.reference
@@ -0,0 +1,2 @@
+1
+1

From da3cb02df5985b0425facb8543f67236e5d9bf38 Mon Sep 17 00:00:00 2001
From: PapaToemmsn <160967948+PapaToemmsn@users.noreply.github.com>
Date: Wed, 13 Mar 2024 18:10:26 +0100
Subject: [PATCH 60/92] COMMIT/ROLLBACK with no active transaction do not throw
 an exception for MySQL clients anymore

---
 .../InterpreterTransactionControlQuery.cpp       | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/InterpreterTransactionControlQuery.cpp b/src/Interpreters/InterpreterTransactionControlQuery.cpp
index fc71874f7a8..d0f7ddb5fcf 100644
--- a/src/Interpreters/InterpreterTransactionControlQuery.cpp
+++ b/src/Interpreters/InterpreterTransactionControlQuery.cpp
@@ -52,7 +52,13 @@ BlockIO InterpreterTransactionControlQuery::executeCommit(ContextMutablePtr sess
 {
     auto txn = session_context->getCurrentTransaction();
     if (!txn)
-        return {};
+    {
+        if (session_context->getClientInfo().interface == ClientInfo::Interface::MYSQL)
+            return {};
+        else
+            throw Exception(ErrorCodes::INVALID_TRANSACTION, "There is no current transaction");
+    }
+    
     if (txn->getState() != MergeTreeTransaction::RUNNING)
         throw Exception(ErrorCodes::INVALID_TRANSACTION, "Transaction is not in RUNNING state");
 
@@ -111,7 +117,13 @@ BlockIO InterpreterTransactionControlQuery::executeRollback(ContextMutablePtr se
 {
     auto txn = session_context->getCurrentTransaction();
     if (!txn)
-        return {};
+    {
+        if (session_context->getClientInfo().interface == ClientInfo::Interface::MYSQL)
+            return {};
+        else
+            throw Exception(ErrorCodes::INVALID_TRANSACTION, "There is no current transaction");
+    }
+    
     if (txn->getState() == MergeTreeTransaction::COMMITTED)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Transaction is in COMMITTED state");
     if (txn->getState() == MergeTreeTransaction::COMMITTING)

From 364fe95b76af03d392c91a780bbda4339f14113c Mon Sep 17 00:00:00 2001
From: PapaToemmsn <160967948+PapaToemmsn@users.noreply.github.com>
Date: Wed, 13 Mar 2024 18:33:41 +0100
Subject: [PATCH 61/92] fix style check errors

---
 src/Interpreters/InterpreterTransactionControlQuery.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/Interpreters/InterpreterTransactionControlQuery.cpp b/src/Interpreters/InterpreterTransactionControlQuery.cpp
index d0f7ddb5fcf..d31ace758c4 100644
--- a/src/Interpreters/InterpreterTransactionControlQuery.cpp
+++ b/src/Interpreters/InterpreterTransactionControlQuery.cpp
@@ -58,7 +58,6 @@ BlockIO InterpreterTransactionControlQuery::executeCommit(ContextMutablePtr sess
         else
             throw Exception(ErrorCodes::INVALID_TRANSACTION, "There is no current transaction");
     }
-    
     if (txn->getState() != MergeTreeTransaction::RUNNING)
         throw Exception(ErrorCodes::INVALID_TRANSACTION, "Transaction is not in RUNNING state");
 
@@ -123,7 +122,6 @@ BlockIO InterpreterTransactionControlQuery::executeRollback(ContextMutablePtr se
         else
             throw Exception(ErrorCodes::INVALID_TRANSACTION, "There is no current transaction");
     }
-    
     if (txn->getState() == MergeTreeTransaction::COMMITTED)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Transaction is in COMMITTED state");
     if (txn->getState() == MergeTreeTransaction::COMMITTING)

From 4f9702aa164a5daaaf2d9fe769fcb23d9a31f557 Mon Sep 17 00:00:00 2001
From: PapaToemmsn <th+github@litterbox.online>
Date: Fri, 15 Mar 2024 10:13:21 +0000
Subject: [PATCH 62/92] made 02989_mysql_transaction_test.sh executable

---
 tests/queries/0_stateless/02989_mysql_transaction_test.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 tests/queries/0_stateless/02989_mysql_transaction_test.sh

diff --git a/tests/queries/0_stateless/02989_mysql_transaction_test.sh b/tests/queries/0_stateless/02989_mysql_transaction_test.sh
old mode 100644
new mode 100755

From 53159070b5598f43865e80d0957fccbc7314fa15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 18 Mar 2024 15:52:16 +0100
Subject: [PATCH 63/92] Remove default cluster in tests

---
 tests/config/config.d/clusters.xml                     | 3 +++
 tests/integration/helpers/0_common_instance_config.xml | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/tests/config/config.d/clusters.xml b/tests/config/config.d/clusters.xml
index 7ade716902c..9490f98d6d8 100644
--- a/tests/config/config.d/clusters.xml
+++ b/tests/config/config.d/clusters.xml
@@ -1,5 +1,8 @@
 <clickhouse>
     <remote_servers>
+        <!-- Remove the default remote server to avoid people depending on its hardcoded config
+             Use test_shard_localhost instead -->
+        <default remove="remove"></default>
         <test_shard_localhost>
             <shard>
                 <replica>
diff --git a/tests/integration/helpers/0_common_instance_config.xml b/tests/integration/helpers/0_common_instance_config.xml
index 73792affee6..1d97bcb7bcf 100644
--- a/tests/integration/helpers/0_common_instance_config.xml
+++ b/tests/integration/helpers/0_common_instance_config.xml
@@ -29,4 +29,7 @@
     </access_control_improvements>
 
     <custom_cached_disks_base_directory replace="replace">/</custom_cached_disks_base_directory>
+
+    <!-- Remove the default remote server to avoid people depending on its hardcoded config -->
+    <remote_servers remove="remove"></remote_servers>
 </clickhouse>

From 3df9e3febc652f1d268e6ffe1d026e0561007b77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 18 Mar 2024 15:52:54 +0100
Subject: [PATCH 64/92] Adapt stateless tests

---
 .../02860_distributed_flush_on_detach.reference  |  4 ++--
 .../02860_distributed_flush_on_detach.sql        |  4 ++--
 .../02890_describe_table_options.reference       | 16 ++++++++--------
 .../0_stateless/02890_describe_table_options.sql | 16 ++++++++--------
 .../02903_rmt_retriable_merge_exception.sh       |  2 +-
 ...ngs_background_insert_compatibility.reference | 16 ++++++++--------
 ...ttings_background_insert_compatibility.sql.j2 |  4 ++--
 7 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/tests/queries/0_stateless/02860_distributed_flush_on_detach.reference b/tests/queries/0_stateless/02860_distributed_flush_on_detach.reference
index 8e5815188e7..2bb78f92948 100644
--- a/tests/queries/0_stateless/02860_distributed_flush_on_detach.reference
+++ b/tests/queries/0_stateless/02860_distributed_flush_on_detach.reference
@@ -1,7 +1,7 @@
 -- { echoOn }
 
 create table data (key Int) engine=Memory();
-create table dist (key Int) engine=Distributed(default, currentDatabase(), data);
+create table dist (key Int) engine=Distributed(test_shard_localhost, currentDatabase(), data);
 system stop distributed sends dist;
 -- check that FLUSH DISTRIBUTED does flushing anyway
 insert into dist values (1);
@@ -19,7 +19,7 @@ attach table dist;
 truncate table data;
 -- check flush_on_detach=0
 drop table dist;
-create table dist (key Int) engine=Distributed(default, currentDatabase(), data) settings flush_on_detach=0;
+create table dist (key Int) engine=Distributed(test_shard_localhost, currentDatabase(), data) settings flush_on_detach=0;
 system stop distributed sends dist;
 insert into dist values (1);
 detach table dist;
diff --git a/tests/queries/0_stateless/02860_distributed_flush_on_detach.sql b/tests/queries/0_stateless/02860_distributed_flush_on_detach.sql
index 5ba45d72c08..10442650478 100644
--- a/tests/queries/0_stateless/02860_distributed_flush_on_detach.sql
+++ b/tests/queries/0_stateless/02860_distributed_flush_on_detach.sql
@@ -6,7 +6,7 @@ drop table if exists dist;
 -- { echoOn }
 
 create table data (key Int) engine=Memory();
-create table dist (key Int) engine=Distributed(default, currentDatabase(), data);
+create table dist (key Int) engine=Distributed(test_shard_localhost, currentDatabase(), data);
 system stop distributed sends dist;
 
 -- check that FLUSH DISTRIBUTED does flushing anyway
@@ -25,7 +25,7 @@ truncate table data;
 
 -- check flush_on_detach=0
 drop table dist;
-create table dist (key Int) engine=Distributed(default, currentDatabase(), data) settings flush_on_detach=0;
+create table dist (key Int) engine=Distributed(test_shard_localhost, currentDatabase(), data) settings flush_on_detach=0;
 system stop distributed sends dist;
 insert into dist values (1);
 detach table dist;
diff --git a/tests/queries/0_stateless/02890_describe_table_options.reference b/tests/queries/0_stateless/02890_describe_table_options.reference
index ff58202ae49..05daade09d8 100644
--- a/tests/queries/0_stateless/02890_describe_table_options.reference
+++ b/tests/queries/0_stateless/02890_describe_table_options.reference
@@ -7,7 +7,7 @@ DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
 │ arr  │ Array(UInt64)             │ DEFAULT      │ [10, 20]           │              │ ZSTD(1)          │                │
 │ t    │ Tuple(a String, b UInt64) │ DEFAULT      │ ('foo', 0)         │              │ ZSTD(1)          │                │
 └──────┴───────────────────────────┴──────────────┴────────────────────┴──────────────┴──────────────────┴────────────────┘
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 ┌─name─┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment──────┬─codec_expression─┬─ttl_expression─┐
 │ id   │ UInt64                    │              │                    │ index column │                  │                │
 │ arr  │ Array(UInt64)             │ DEFAULT      │ [10, 20]           │              │ ZSTD(1)          │                │
@@ -23,7 +23,7 @@ DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
 │ t.a       │ String                    │              │                    │              │ ZSTD(1)          │                │            1 │
 │ t.b       │ UInt64                    │              │                    │              │ ZSTD(1)          │                │            1 │
 └───────────┴───────────────────────────┴──────────────┴────────────────────┴──────────────┴──────────────────┴────────────────┴──────────────┘
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 ┌─name──────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment──────┬─codec_expression─┬─ttl_expression─┬─is_subcolumn─┐
 │ id        │ UInt64                    │              │                    │ index column │                  │                │            0 │
 │ arr       │ Array(UInt64)             │ DEFAULT      │ [10, 20]           │              │ ZSTD(1)          │                │            0 │
@@ -47,7 +47,7 @@ DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
 │ _row_exists    │ UInt8                     │              │                    │ Persisted mask created by lightweight delete that show whether row exists or is deleted │                  │                │          1 │
 │ _block_number  │ UInt64                    │              │                    │ Persisted original number of block that was assigned at insert                          │ Delta, LZ4       │                │          1 │
 └────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴────────────┘
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 ┌─name───────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_virtual─┐
 │ id             │ UInt64                    │              │                    │ index column                                                                            │                  │                │          0 │
 │ arr            │ Array(UInt64)             │ DEFAULT      │ [10, 20]           │                                                                                         │ ZSTD(1)          │                │          0 │
@@ -80,7 +80,7 @@ DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
 │ t.a            │ String                    │              │                    │                                                                                         │ ZSTD(1)          │                │            1 │          0 │
 │ t.b            │ UInt64                    │              │                    │                                                                                         │ ZSTD(1)          │                │            1 │          0 │
 └────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴──────────────┴────────────┘
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 ┌─name───────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_subcolumn─┬─is_virtual─┐
 │ id             │ UInt64                    │              │                    │ index column                                                                            │                  │                │            0 │          0 │
 │ arr            │ Array(UInt64)             │ DEFAULT      │ [10, 20]           │                                                                                         │ ZSTD(1)          │                │            0 │          0 │
@@ -105,7 +105,7 @@ DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
 │ arr  │ Array(UInt64)             │
 │ t    │ Tuple(a String, b UInt64) │
 └──────┴───────────────────────────┘
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 ┌─name─┬─type──────────────────────┐
 │ id   │ UInt64                    │
 │ arr  │ Array(UInt64)             │
@@ -121,7 +121,7 @@ DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
 │ t.a       │ String                    │            1 │
 │ t.b       │ UInt64                    │            1 │
 └───────────┴───────────────────────────┴──────────────┘
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 ┌─name──────┬─type──────────────────────┬─is_subcolumn─┐
 │ id        │ UInt64                    │            0 │
 │ arr       │ Array(UInt64)             │            0 │
@@ -145,7 +145,7 @@ DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
 │ _row_exists    │ UInt8                     │          1 │
 │ _block_number  │ UInt64                    │          1 │
 └────────────────┴───────────────────────────┴────────────┘
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 ┌─name───────────┬─type──────────────────────┬─is_virtual─┐
 │ id             │ UInt64                    │          0 │
 │ arr            │ Array(UInt64)             │          0 │
@@ -178,7 +178,7 @@ DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
 │ t.a            │ String                    │            1 │          0 │
 │ t.b            │ UInt64                    │            1 │          0 │
 └────────────────┴───────────────────────────┴──────────────┴────────────┘
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 ┌─name───────────┬─type──────────────────────┬─is_subcolumn─┬─is_virtual─┐
 │ id             │ UInt64                    │            0 │          0 │
 │ arr            │ Array(UInt64)             │            0 │          0 │
diff --git a/tests/queries/0_stateless/02890_describe_table_options.sql b/tests/queries/0_stateless/02890_describe_table_options.sql
index 63806c7ee3d..419aec1a0f8 100644
--- a/tests/queries/0_stateless/02890_describe_table_options.sql
+++ b/tests/queries/0_stateless/02890_describe_table_options.sql
@@ -14,42 +14,42 @@ ORDER BY id;
 SET describe_compact_output = 0, describe_include_virtual_columns = 0, describe_include_subcolumns = 0;
 
 DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 
 SET describe_compact_output = 0, describe_include_virtual_columns = 0, describe_include_subcolumns = 1;
 
 DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 
 SET describe_compact_output = 0, describe_include_virtual_columns = 1, describe_include_subcolumns = 0;
 
 DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 
 SET describe_compact_output = 0, describe_include_virtual_columns = 1, describe_include_subcolumns = 1;
 
 DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 
 SET describe_compact_output = 1, describe_include_virtual_columns = 0, describe_include_subcolumns = 0;
 
 DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 
 SET describe_compact_output = 1, describe_include_virtual_columns = 0, describe_include_subcolumns = 1;
 
 DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 
 SET describe_compact_output = 1, describe_include_virtual_columns = 1, describe_include_subcolumns = 0;
 
 DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 
 SET describe_compact_output = 1, describe_include_virtual_columns = 1, describe_include_subcolumns = 1;
 
 DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes;
-DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
+DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes;
 
 -- { echoOff }
 
diff --git a/tests/queries/0_stateless/02903_rmt_retriable_merge_exception.sh b/tests/queries/0_stateless/02903_rmt_retriable_merge_exception.sh
index 095239954f4..5065da371a8 100755
--- a/tests/queries/0_stateless/02903_rmt_retriable_merge_exception.sh
+++ b/tests/queries/0_stateless/02903_rmt_retriable_merge_exception.sh
@@ -10,7 +10,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # (i.e. "No active replica has part X or covering part")
 # does not appears as errors (level=Error), only as info message (level=Information).
 
-cluster=default
+cluster=test_shard_localhost
 if [[ $($CLICKHOUSE_CLIENT -q "select count()>0 from system.clusters where cluster = 'test_cluster_database_replicated'") = 1 ]]; then
     cluster=test_cluster_database_replicated
 fi
diff --git a/tests/queries/0_stateless/02904_distributed_settings_background_insert_compatibility.reference b/tests/queries/0_stateless/02904_distributed_settings_background_insert_compatibility.reference
index fc7383b32bc..0694f895e38 100644
--- a/tests/queries/0_stateless/02904_distributed_settings_background_insert_compatibility.reference
+++ b/tests/queries/0_stateless/02904_distributed_settings_background_insert_compatibility.reference
@@ -1,8 +1,8 @@
-CREATE TABLE default.dist_monitor_batch_inserts\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'default\', \'system\', \'one\')\nSETTINGS monitor_batch_inserts = 1
-CREATE TABLE default.dist_monitor_split_batch_on_failure\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'default\', \'system\', \'one\')\nSETTINGS monitor_split_batch_on_failure = 1
-CREATE TABLE default.dist_monitor_sleep_time_ms\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'default\', \'system\', \'one\')\nSETTINGS monitor_sleep_time_ms = 1
-CREATE TABLE default.dist_monitor_max_sleep_time_ms\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'default\', \'system\', \'one\')\nSETTINGS monitor_max_sleep_time_ms = 1
-CREATE TABLE default.dist_background_insert_batch\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'default\', \'system\', \'one\')\nSETTINGS background_insert_batch = 1
-CREATE TABLE default.dist_background_insert_split_batch_on_failure\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'default\', \'system\', \'one\')\nSETTINGS background_insert_split_batch_on_failure = 1
-CREATE TABLE default.dist_background_insert_sleep_time_ms\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'default\', \'system\', \'one\')\nSETTINGS background_insert_sleep_time_ms = 1
-CREATE TABLE default.dist_background_insert_max_sleep_time_ms\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'default\', \'system\', \'one\')\nSETTINGS background_insert_max_sleep_time_ms = 1
+CREATE TABLE default.dist_monitor_batch_inserts\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'system\', \'one\')\nSETTINGS monitor_batch_inserts = 1
+CREATE TABLE default.dist_monitor_split_batch_on_failure\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'system\', \'one\')\nSETTINGS monitor_split_batch_on_failure = 1
+CREATE TABLE default.dist_monitor_sleep_time_ms\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'system\', \'one\')\nSETTINGS monitor_sleep_time_ms = 1
+CREATE TABLE default.dist_monitor_max_sleep_time_ms\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'system\', \'one\')\nSETTINGS monitor_max_sleep_time_ms = 1
+CREATE TABLE default.dist_background_insert_batch\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'system\', \'one\')\nSETTINGS background_insert_batch = 1
+CREATE TABLE default.dist_background_insert_split_batch_on_failure\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'system\', \'one\')\nSETTINGS background_insert_split_batch_on_failure = 1
+CREATE TABLE default.dist_background_insert_sleep_time_ms\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'system\', \'one\')\nSETTINGS background_insert_sleep_time_ms = 1
+CREATE TABLE default.dist_background_insert_max_sleep_time_ms\n(\n    `dummy` UInt8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'system\', \'one\')\nSETTINGS background_insert_max_sleep_time_ms = 1
diff --git a/tests/queries/0_stateless/02904_distributed_settings_background_insert_compatibility.sql.j2 b/tests/queries/0_stateless/02904_distributed_settings_background_insert_compatibility.sql.j2
index 3116f0f5973..20e848baa88 100644
--- a/tests/queries/0_stateless/02904_distributed_settings_background_insert_compatibility.sql.j2
+++ b/tests/queries/0_stateless/02904_distributed_settings_background_insert_compatibility.sql.j2
@@ -10,13 +10,13 @@
     'background_insert_max_sleep_time_ms',
 ] %}
 drop table if exists dist_{{ table_setting }};
-create table dist_{{ table_setting }} as system.one engine=Distributed(default, system, one) settings {{ table_setting }}=1;
+create table dist_{{ table_setting }} as system.one engine=Distributed(test_shard_localhost, system, one) settings {{ table_setting }}=1;
 show create dist_{{ table_setting }};
 drop table dist_{{ table_setting }};
 {% endfor %}
 
 create table data (key Int) engine=Null();
-create table dist (key Int) engine=Distributed(default, currentDatabase(), data, key);
+create table dist (key Int) engine=Distributed(test_shard_localhost, currentDatabase(), data, key);
 {% for query_setting in [
     'distributed_directory_monitor_sleep_time_ms',
     'distributed_directory_monitor_max_sleep_time_ms',

From 6b19c5ea753c1990d3bb0636c5108b75a017ae7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 18 Mar 2024 16:07:10 +0100
Subject: [PATCH 65/92] Fix integration tests

---
 .../configs/remote_servers.xml                       | 12 ++++++++++++
 .../test_select_from_system_tables.py                |  5 ++++-
 .../test_quota/configs/remote_servers.xml            | 12 ++++++++++++
 tests/integration/test_quota/test.py                 |  1 +
 .../configs/remote_servers.xml                       | 12 ++++++++++++
 .../test_from_system_tables.py                       |  1 +
 6 files changed, 42 insertions(+), 1 deletion(-)
 create mode 100644 tests/integration/test_disabled_access_control_improvements/configs/remote_servers.xml
 create mode 100644 tests/integration/test_quota/configs/remote_servers.xml
 create mode 100644 tests/integration/test_select_access_rights/configs/remote_servers.xml

diff --git a/tests/integration/test_disabled_access_control_improvements/configs/remote_servers.xml b/tests/integration/test_disabled_access_control_improvements/configs/remote_servers.xml
new file mode 100644
index 00000000000..b9aee7f5ca6
--- /dev/null
+++ b/tests/integration/test_disabled_access_control_improvements/configs/remote_servers.xml
@@ -0,0 +1,12 @@
+<clickhouse>
+    <remote_servers>
+        <default>
+            <shard>
+                <replica>
+                    <host>node1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </default>
+    </remote_servers>
+</clickhouse>
diff --git a/tests/integration/test_disabled_access_control_improvements/test_select_from_system_tables.py b/tests/integration/test_disabled_access_control_improvements/test_select_from_system_tables.py
index 894464fd813..4ea5a524fed 100644
--- a/tests/integration/test_disabled_access_control_improvements/test_select_from_system_tables.py
+++ b/tests/integration/test_disabled_access_control_improvements/test_select_from_system_tables.py
@@ -6,7 +6,10 @@ from helpers.test_tools import TSV
 cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance(
     "node",
-    main_configs=["configs/config.d/disable_access_control_improvements.xml"],
+    main_configs=[
+        "configs/config.d/disable_access_control_improvements.xml",
+        "configs/remote_servers.xml",
+    ],
     user_configs=[
         "configs/users.d/another_user.xml",
     ],
diff --git a/tests/integration/test_quota/configs/remote_servers.xml b/tests/integration/test_quota/configs/remote_servers.xml
new file mode 100644
index 00000000000..b9aee7f5ca6
--- /dev/null
+++ b/tests/integration/test_quota/configs/remote_servers.xml
@@ -0,0 +1,12 @@
+<clickhouse>
+    <remote_servers>
+        <default>
+            <shard>
+                <replica>
+                    <host>node1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </default>
+    </remote_servers>
+</clickhouse>
diff --git a/tests/integration/test_quota/test.py b/tests/integration/test_quota/test.py
index bf64b57a7bf..bb50c84b340 100644
--- a/tests/integration/test_quota/test.py
+++ b/tests/integration/test_quota/test.py
@@ -9,6 +9,7 @@ from helpers.test_tools import assert_eq_with_retry, TSV
 cluster = ClickHouseCluster(__file__)
 instance = cluster.add_instance(
     "instance",
+    main_configs=["configs/remote_servers.xml"],
     user_configs=[
         "configs/users.d/assign_myquota_to_default_user.xml",
         "configs/users.d/drop_default_quota.xml",
diff --git a/tests/integration/test_select_access_rights/configs/remote_servers.xml b/tests/integration/test_select_access_rights/configs/remote_servers.xml
new file mode 100644
index 00000000000..b9aee7f5ca6
--- /dev/null
+++ b/tests/integration/test_select_access_rights/configs/remote_servers.xml
@@ -0,0 +1,12 @@
+<clickhouse>
+    <remote_servers>
+        <default>
+            <shard>
+                <replica>
+                    <host>node1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </default>
+    </remote_servers>
+</clickhouse>
diff --git a/tests/integration/test_select_access_rights/test_from_system_tables.py b/tests/integration/test_select_access_rights/test_from_system_tables.py
index 3cd6b90109c..7076e5c8ce7 100644
--- a/tests/integration/test_select_access_rights/test_from_system_tables.py
+++ b/tests/integration/test_select_access_rights/test_from_system_tables.py
@@ -6,6 +6,7 @@ from helpers.test_tools import TSV
 cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance(
     "node",
+    main_configs=["configs/remote_servers.xml"],
     user_configs=[
         "configs/another_user.xml",
     ],

From 10dd47cf5cf76c4e6a8475e56d4d9e39f562319b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 18 Mar 2024 16:40:03 +0100
Subject: [PATCH 66/92] Separate CH and sanitizer logs

---
 docker/test/fuzzer/run-fuzzer.sh | 16 +++++++++++-----
 tests/ci/ast_fuzzer_check.py     |  9 +--------
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index 1c41d299ac3..e8eb2d21e4d 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -174,7 +174,14 @@ function fuzz
     mkdir -p /var/run/clickhouse-server
 
     # NOTE: we use process substitution here to preserve keep $! as a pid of clickhouse-server
-    clickhouse-server --config-file db/config.xml --pid-file /var/run/clickhouse-server/clickhouse-server.pid -- --path db > server.log 2>>stderr.log &
+    # server.log -> CH logs
+    # stderr.log -> Process logs (sanitizer)
+    clickhouse-server \
+        --config-file db/config.xml \
+        --pid-file /var/run/clickhouse-server/clickhouse-server.pid \
+        --  --path db \
+            --logger.console=0 \
+            --logger.log=server.log > stderr.log 2>&1
     server_pid=$!
 
     kill -0 $server_pid
@@ -310,7 +317,7 @@ quit
 
         IS_SANITIZED=$(clickhouse-local --query "SELECT value LIKE '%-fsanitize=%' FROM system.build_options WHERE name = 'CXX_FLAGS'")
 
-        if [ "${IS_SANITIZED}" -eq "1" ] && rg --text 'Sanitizer:? (out-of-memory|out of memory|failed to allocate)|Child process was terminated by signal 9' description.txt
+        if [ "${IS_SANITIZED}" -eq "1" ] && rg --text 'Sanitizer:? (out-of-memory|out of memory|failed to allocate)|Child process was terminated by signal 9' stderr.log
         then
             # OOM of sanitizer is not a problem we can handle - treat it as success, but preserve the description.
             # Why? Because sanitizers have the memory overhead, that is not controllable from inside clickhouse-server.
@@ -392,7 +399,7 @@ if [ -f core.zst ]; then
 fi
 
 # Keep all the lines in the paragraphs containing <Fatal> that either contain <Fatal> or don't start with 20... (year)
-sed -n '/<Fatal>/,/^$/p' server.log stderr.log | awk '/<Fatal>/ || !/^20/' > fatal.log ||:
+sed -n '/<Fatal>/,/^$/p' server.log | awk '/<Fatal>/ || !/^20/' > fatal.log ||:
 FATAL_LINK=''
 if [ -s fatal.log ]; then
     FATAL_LINK='<a href="fatal.log">fatal.log</a>'
@@ -401,7 +408,6 @@ fi
 dmesg -T > dmesg.log ||:
 
 zstd --threads=0 --rm server.log
-zstd --threads=0 --rm stderr.log
 zstd --threads=0 --rm fuzzer.log
 
 cat > report.html <<EOF ||:
@@ -428,7 +434,7 @@ p.links a { padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-s
   <a href="run.log">run.log</a>
   <a href="fuzzer.log.zst">fuzzer.log.zst</a>
   <a href="server.log.zst">server.log.zst</a>
-  <a href="stderr.log.zst">stderr.log.zst</a>
+  <a href="stderr.log">stderr.log</a>
   <a href="main.log">main.log</a>
   <a href="dmesg.log">dmesg.log</a>
   ${CORE_LINK}
diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py
index c24a67a7a6e..b88a9476a6d 100644
--- a/tests/ci/ast_fuzzer_check.py
+++ b/tests/ci/ast_fuzzer_check.py
@@ -115,6 +115,7 @@ def main():
         "core.zst": workspace_path / "core.zst",
         "dmesg.log": workspace_path / "dmesg.log",
         "fatal.log": workspace_path / "fatal.log",
+        "stderr.log": workspace_path / "stderr.log",
     }
 
     compressed_server_log_path = workspace_path / "server.log.zst"
@@ -126,14 +127,6 @@ def main():
         if not_compressed_server_log_path.exists():
             paths["server.log"] = not_compressed_server_log_path
 
-    compressed_stderr_log_path = workspace_path / "stderr.log.zst"
-    if compressed_stderr_log_path.exists():
-        paths["stderr.log.zst"] = compressed_stderr_log_path
-    else:
-        not_compressed_stderr_log_path = workspace_path / "stderr.log"
-        if compressed_stderr_log_path.exists():
-            paths["stderr.log"] = not_compressed_stderr_log_path
-
     # Same idea but with the fuzzer log
     compressed_fuzzer_log_path = workspace_path / "fuzzer.log.zst"
     if compressed_fuzzer_log_path.exists():

From b814fabcd3224ab6904e5f8f947ca79a8f8ba02b Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Mon, 18 Mar 2024 17:18:01 +0100
Subject: [PATCH 67/92] Disallow s3queue sharded mode

---
 src/Core/Settings.h                     |  1 +
 src/Core/SettingsChangesHistory.h       |  1 +
 src/Storages/S3Queue/StorageS3Queue.cpp | 14 ++++++++++++--
 src/Storages/S3Queue/StorageS3Queue.h   |  3 ++-
 4 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 415063eee84..8508fc5d2d3 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -114,6 +114,7 @@ class IColumn;
     M(Bool, enable_s3_requests_logging, false, "Enable very explicit logging of S3 requests. Makes sense for debug only.", 0) \
     M(String, s3queue_default_zookeeper_path, "/clickhouse/s3queue/", "Default zookeeper path prefix for S3Queue engine", 0) \
     M(Bool, s3queue_enable_logging_to_s3queue_log, false, "Enable writing to system.s3queue_log. The value can be overwritten per table with table settings", 0) \
+    M(Bool, s3queue_allow_experimental_sharded_mode, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten", 0) \
     M(UInt64, hdfs_replication, 0, "The actual number of replications can be specified when the hdfs file is created.", 0) \
     M(Bool, hdfs_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables", 0) \
     M(Bool, hdfs_create_new_file_on_insert, false, "Enables or disables creating a new file on each insert in hdfs engine tables", 0) \
diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h
index 072b9803682..ff5289230ca 100644
--- a/src/Core/SettingsChangesHistory.h
+++ b/src/Core/SettingsChangesHistory.h
@@ -100,6 +100,7 @@ static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> sett
               {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"},
               {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"},
               {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"},
+              {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"},
               }},
     {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
               {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},
diff --git a/src/Storages/S3Queue/StorageS3Queue.cpp b/src/Storages/S3Queue/StorageS3Queue.cpp
index 6e7ac2b47b8..048816fe4f1 100644
--- a/src/Storages/S3Queue/StorageS3Queue.cpp
+++ b/src/Storages/S3Queue/StorageS3Queue.cpp
@@ -105,7 +105,8 @@ StorageS3Queue::StorageS3Queue(
     const String & comment,
     ContextPtr context_,
     std::optional<FormatSettings> format_settings_,
-    ASTStorage * engine_args)
+    ASTStorage * engine_args,
+    LoadingStrictnessLevel mode)
     : IStorage(table_id_)
     , WithContext(context_)
     , s3queue_settings(std::move(s3queue_settings_))
@@ -129,6 +130,14 @@ StorageS3Queue::StorageS3Queue(
         throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "S3Queue url must either end with '/' or contain globs");
     }
 
+    if (mode == LoadingStrictnessLevel::CREATE
+        && !context_->getSettingsRef().s3queue_allow_experimental_sharded_mode
+        && s3queue_settings->mode == S3QueueMode::ORDERED
+        && s3queue_settings->s3queue_total_shards_num)
+    {
+        throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "S3Queue sharded mode is not allowed. To enable use `s3queue_allow_experimental_sharded_mode`");
+    }
+
     checkAndAdjustSettings(*s3queue_settings, context_->getSettingsRef());
 
     configuration.update(context_);
@@ -663,7 +672,8 @@ void registerStorageS3QueueImpl(const String & name, StorageFactory & factory)
                 args.comment,
                 args.getContext(),
                 format_settings,
-                args.storage_def);
+                args.storage_def,
+                args.mode);
         },
         {
             .supports_settings = true,
diff --git a/src/Storages/S3Queue/StorageS3Queue.h b/src/Storages/S3Queue/StorageS3Queue.h
index bdd3ab7b687..33fccc20b84 100644
--- a/src/Storages/S3Queue/StorageS3Queue.h
+++ b/src/Storages/S3Queue/StorageS3Queue.h
@@ -37,7 +37,8 @@ public:
         const String & comment,
         ContextPtr context_,
         std::optional<FormatSettings> format_settings_,
-        ASTStorage * engine_args);
+        ASTStorage * engine_args,
+        LoadingStrictnessLevel mode);
 
     String getName() const override { return "S3Queue"; }
 

From b3e9f07e2eaf3703c19497e3fe86a90a7e0b846d Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Mon, 18 Mar 2024 16:54:59 +0000
Subject: [PATCH 68/92] Deprecate name collisions for now.

---
 src/Interpreters/ActionsDAG.cpp                        |  4 ----
 .../QueryPlan/Optimizations/liftUpFunctions.cpp        | 10 +++++-----
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index 86a5b91ad63..37ef217cb6d 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -1757,10 +1757,6 @@ ActionsDAG::SplitResult ActionsDAG::split(std::unordered_set<const Node *> split
         }
     }
 
-    /// All input nodes are needed by split to make sure first DAG and initial DAG have equal inputs,
-    for (const auto * input: inputs)
-        data[input].needed_by_split_node = true;
-
     /// DFS. Move nodes to one of the DAGs.
     for (const auto & node : nodes)
     {
diff --git a/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp b/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp
index 3fc2d64b11f..b280e2d3cc6 100644
--- a/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp
+++ b/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp
@@ -29,14 +29,14 @@ const DB::DataStream & getChildOutputStream(DB::QueryPlan::Node & node)
 namespace DB::QueryPlanOptimizations
 {
 
-/// This is a check that output columns does not have the same name
+/// This is a check that nodes columns does not have the same name
 /// This is ok for DAG, but may introduce a bug in a SotringStep cause columns are selected by name.
-static bool areOutputsConvertableToBlock(const ActionsDAG::NodeRawConstPtrs & outputs)
+static bool areNodesConvertableToBlock(const ActionsDAG::NodeRawConstPtrs & nodes)
 {
     std::unordered_set<std::string_view> names;
-    for (const auto & output : outputs)
+    for (const auto & node : nodes)
     {
-        if (!names.emplace(output->result_name).second)
+        if (!names.emplace(node->result_name).second)
             return false;
     }
 
@@ -72,7 +72,7 @@ size_t tryExecuteFunctionsAfterSorting(QueryPlan::Node * parent_node, QueryPlan:
     if (unneeded_for_sorting->trivial())
         return 0;
 
-    if (!areOutputsConvertableToBlock(needed_for_sorting->getOutputs()))
+    if (!areNodesConvertableToBlock(needed_for_sorting->getOutputs()) || !areNodesConvertableToBlock(unneeded_for_sorting->getInputs()))
         return 0;
 
     // Sorting (parent_node) -> Expression (child_node)

From 7dbd96cf4ee028a2b74342ff1a5a1f13d1c7ff5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 18 Mar 2024 19:43:13 +0100
Subject: [PATCH 69/92] Fix

---
 docker/test/fuzzer/run-fuzzer.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index e8eb2d21e4d..e3b5a99018f 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -317,7 +317,7 @@ quit
 
         IS_SANITIZED=$(clickhouse-local --query "SELECT value LIKE '%-fsanitize=%' FROM system.build_options WHERE name = 'CXX_FLAGS'")
 
-        if [ "${IS_SANITIZED}" -eq "1" ] && rg --text 'Sanitizer:? (out-of-memory|out of memory|failed to allocate)|Child process was terminated by signal 9' stderr.log
+        if [ "${IS_SANITIZED}" -eq "1" ] && rg --text 'Sanitizer:? (out-of-memory|out of memory|failed to allocate)|Child process was terminated by signal 9' description.txt
         then
             # OOM of sanitizer is not a problem we can handle - treat it as success, but preserve the description.
             # Why? Because sanitizers have the memory overhead, that is not controllable from inside clickhouse-server.

From 3def01f1f6951d11a142f80ef64f3d315ba92a4d Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Mon, 18 Mar 2024 18:53:01 +0000
Subject: [PATCH 70/92] Fixing 01083_expressions_in_engine_arguments with
 analyzer.

---
 src/Storages/StorageBuffer.cpp | 29 ++++++++++++++++++++++++-----
 src/Storages/StorageValues.cpp | 33 +++++++++++++++++++++++++++++++++
 src/Storages/StorageValues.h   |  7 +++++++
 tests/analyzer_tech_debt.txt   |  1 -
 4 files changed, 64 insertions(+), 6 deletions(-)

diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index 2925038ec8e..ab656e5e3ff 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -1,5 +1,6 @@
 #include <Interpreters/InterpreterInsertQuery.h>
 #include <Interpreters/InterpreterSelectQuery.h>
+#include <Interpreters/InterpreterSelectQueryAnalyzer.h>
 #include <Interpreters/addMissingDefaults.h>
 #include <Interpreters/castColumn.h>
 #include <Interpreters/evaluateConstantExpression.h>
@@ -24,6 +25,7 @@
 #include <Storages/AlterCommands.h>
 #include <Storages/StorageBuffer.h>
 #include <Storages/StorageFactory.h>
+#include <Storages/StorageValues.h>
 #include <Storages/checkAndGetLiteralArgument.h>
 #include <base/getThreadId.h>
 #include <base/range.h>
@@ -366,11 +368,28 @@ void StorageBuffer::read(
     if (processed_stage > QueryProcessingStage::FetchColumns)
     {
         /// TODO: Find a way to support projections for StorageBuffer
-        auto interpreter = InterpreterSelectQuery(
-                query_info.query, local_context, std::move(pipe_from_buffers),
-                SelectQueryOptions(processed_stage));
-        interpreter.addStorageLimits(*query_info.storage_limits);
-        interpreter.buildQueryPlan(buffers_plan);
+        if (local_context->getSettingsRef().allow_experimental_analyzer)
+        {
+            auto storage = std::make_shared<StorageValues>(
+                    getStorageID(),
+                    storage_snapshot->getAllColumnsDescription(),
+                    std::move(pipe_from_buffers),
+                    *getVirtualsPtr());
+
+            auto interpreter = InterpreterSelectQueryAnalyzer(
+                    query_info.query, local_context, storage,
+                    SelectQueryOptions(processed_stage));
+            interpreter.addStorageLimits(*query_info.storage_limits);
+            buffers_plan = std::move(interpreter).extractQueryPlan();
+        }
+        else
+        {
+            auto interpreter = InterpreterSelectQuery(
+                    query_info.query, local_context, std::move(pipe_from_buffers),
+                    SelectQueryOptions(processed_stage));
+            interpreter.addStorageLimits(*query_info.storage_limits);
+            interpreter.buildQueryPlan(buffers_plan);
+        }
     }
     else
     {
diff --git a/src/Storages/StorageValues.cpp b/src/Storages/StorageValues.cpp
index a930ffd1307..3a293c06b37 100644
--- a/src/Storages/StorageValues.cpp
+++ b/src/Storages/StorageValues.cpp
@@ -2,6 +2,7 @@
 #include <Storages/ColumnsDescription.h>
 #include <Storages/StorageValues.h>
 #include <Processors/Sources/SourceFromSingleChunk.h>
+#include <Processors/Transforms/ExpressionTransform.h>
 #include <QueryPipeline/Pipe.h>
 
 
@@ -21,6 +22,19 @@ StorageValues::StorageValues(
     setVirtuals(std::move(virtuals_));
 }
 
+StorageValues::StorageValues(
+    const StorageID & table_id_,
+    const ColumnsDescription & columns_,
+    Pipe prepared_pipe_,
+    VirtualColumnsDescription virtuals_)
+    : IStorage(table_id_), prepared_pipe(std::move(prepared_pipe_))
+{
+    StorageInMemoryMetadata storage_metadata;
+    storage_metadata.setColumns(columns_);
+    setInMemoryMetadata(storage_metadata);
+    setVirtuals(std::move(virtuals_));
+}
+
 Pipe StorageValues::read(
     const Names & column_names,
     const StorageSnapshotPtr & storage_snapshot,
@@ -32,6 +46,25 @@ Pipe StorageValues::read(
 {
     storage_snapshot->check(column_names);
 
+    if (!prepared_pipe.empty())
+    {
+        auto dag = std::make_shared<ActionsDAG>(prepared_pipe.getHeader().getColumnsWithTypeAndName());
+        ActionsDAG::NodeRawConstPtrs outputs;
+        outputs.reserve(column_names.size());
+        for (const auto & name : column_names)
+            outputs.push_back(dag->getOutputs()[prepared_pipe.getHeader().getPositionByName(name)]);
+
+        dag->getOutputs().swap( outputs);
+        auto expression = std::make_shared<ExpressionActions>(dag);
+
+        prepared_pipe.addSimpleTransform([&](const Block & header)
+        {
+            return std::make_shared<ExpressionTransform>(header, expression);
+        });
+
+        return std::move(prepared_pipe);
+    }
+
     /// Get only required columns.
     Block block;
     for (const auto & name : column_names)
diff --git a/src/Storages/StorageValues.h b/src/Storages/StorageValues.h
index ebb182ab667..37992928d55 100644
--- a/src/Storages/StorageValues.h
+++ b/src/Storages/StorageValues.h
@@ -20,6 +20,12 @@ public:
         const Block & res_block_,
         VirtualColumnsDescription virtuals_ = {});
 
+    StorageValues(
+        const StorageID & table_id_,
+        const ColumnsDescription & columns_,
+        Pipe prepared_pipe_,
+        VirtualColumnsDescription virtuals_ = {});
+
     std::string getName() const override { return "Values"; }
 
     Pipe read(
@@ -39,6 +45,7 @@ public:
 
 private:
     Block res_block;
+    Pipe prepared_pipe;
 };
 
 }
diff --git a/tests/analyzer_tech_debt.txt b/tests/analyzer_tech_debt.txt
index 42aa579658e..8335108e243 100644
--- a/tests/analyzer_tech_debt.txt
+++ b/tests/analyzer_tech_debt.txt
@@ -2,7 +2,6 @@
 00717_merge_and_distributed
 00725_memory_tracking
 01062_pm_all_join_with_block_continuation
-01083_expressions_in_engine_arguments
 01155_rename_move_materialized_view
 01584_distributed_buffer_cannot_find_column
 01624_soft_constraints

From c99a874305693856362b4682c7b635ac04457a1c Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Mon, 18 Mar 2024 20:13:52 +0100
Subject: [PATCH 71/92] Update StorageValues.cpp

---
 src/Storages/StorageValues.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/StorageValues.cpp b/src/Storages/StorageValues.cpp
index 3a293c06b37..894b1404a21 100644
--- a/src/Storages/StorageValues.cpp
+++ b/src/Storages/StorageValues.cpp
@@ -54,7 +54,7 @@ Pipe StorageValues::read(
         for (const auto & name : column_names)
             outputs.push_back(dag->getOutputs()[prepared_pipe.getHeader().getPositionByName(name)]);
 
-        dag->getOutputs().swap( outputs);
+        dag->getOutputs().swap(outputs);
         auto expression = std::make_shared<ExpressionActions>(dag);
 
         prepared_pipe.addSimpleTransform([&](const Block & header)

From e97443ddd16123d8b5aa854abc510f26cf67a147 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Mon, 18 Mar 2024 20:15:34 +0100
Subject: [PATCH 72/92] Update StorageBuffer.cpp

---
 src/Storages/StorageBuffer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index ab656e5e3ff..dbcd47c5745 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -365,9 +365,9 @@ void StorageBuffer::read(
     /** If the sources from the table were processed before some non-initial stage of query execution,
       * then sources from the buffers must also be wrapped in the processing pipeline before the same stage.
       */
+    /// TODO: Find a way to support projections for StorageBuffer
     if (processed_stage > QueryProcessingStage::FetchColumns)
     {
-        /// TODO: Find a way to support projections for StorageBuffer
         if (local_context->getSettingsRef().allow_experimental_analyzer)
         {
             auto storage = std::make_shared<StorageValues>(

From 8c55162e92661bcbde94470c1eb822b7e3dc4879 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 18 Mar 2024 19:24:52 +0100
Subject: [PATCH 73/92] Fix string search with const position

---
 src/Functions/FunctionsStringSearch.h            | 16 ++++++++++++++--
 .../03013_position_const_start_pos.reference     |  0
 .../03013_position_const_start_pos.sql           |  3 +++
 3 files changed, 17 insertions(+), 2 deletions(-)
 create mode 100644 tests/queries/0_stateless/03013_position_const_start_pos.reference
 create mode 100644 tests/queries/0_stateless/03013_position_const_start_pos.sql

diff --git a/src/Functions/FunctionsStringSearch.h b/src/Functions/FunctionsStringSearch.h
index 53d99198134..64de5d98ae3 100644
--- a/src/Functions/FunctionsStringSearch.h
+++ b/src/Functions/FunctionsStringSearch.h
@@ -190,14 +190,26 @@ public:
         {
             if (col_haystack_const && col_needle_const)
             {
-                const auto is_col_start_pos_const = !column_start_pos || isColumnConst(*column_start_pos);
+                auto column_start_position_arg = column_start_pos;
+                bool is_col_start_pos_const = false;
+                if (column_start_pos)
+                {
+                    if (const ColumnConst * const_column_start_pos = typeid_cast<const ColumnConst *>(&*column_start_pos))
+                    {
+                        is_col_start_pos_const = true;
+                        column_start_position_arg = const_column_start_pos->getDataColumnPtr();
+                    }
+                }
+                else
+                    is_col_start_pos_const = true;
+
                 vec_res.resize(is_col_start_pos_const ? 1 : column_start_pos->size());
                 const auto null_map = create_null_map();
 
                 Impl::constantConstant(
                     col_haystack_const->getValue<String>(),
                     col_needle_const->getValue<String>(),
-                    column_start_pos,
+                    column_start_position_arg,
                     vec_res,
                     null_map.get());
 
diff --git a/tests/queries/0_stateless/03013_position_const_start_pos.reference b/tests/queries/0_stateless/03013_position_const_start_pos.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03013_position_const_start_pos.sql b/tests/queries/0_stateless/03013_position_const_start_pos.sql
new file mode 100644
index 00000000000..0c6da694b6b
--- /dev/null
+++ b/tests/queries/0_stateless/03013_position_const_start_pos.sql
@@ -0,0 +1,3 @@
+CREATE TABLE 03013_position_const_start_pos (n Int16) ENGINE = Memory;
+INSERT INTO 03013_position_const_start_pos SELECT * FROM generateRandom() LIMIT 1000;
+SELECT position(concat(NULLIF(1, 1), materialize(3)), 'ca', 2) FROM 03013_position_const_start_pos FORMAT Null;

From c1daf104afda843fdefbc635fec1f92ba14bfe45 Mon Sep 17 00:00:00 2001
From: danila-ermakov <152204142+danila-ermakov@users.noreply.github.com>
Date: Mon, 18 Mar 2024 18:23:31 -0400
Subject: [PATCH 74/92] queries -> queries-file  cli.md

queries-file should be used to specify file
---
 docs/en/interfaces/cli.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md
index 2b90d684c13..1eb426af617 100644
--- a/docs/en/interfaces/cli.md
+++ b/docs/en/interfaces/cli.md
@@ -178,7 +178,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
 - `--password` – The password. Default value: empty string.
 - `--ask-password` - Prompt the user to enter a password.
 - `--query, -q` – The query to process when using non-interactive mode. `--query` can be specified multiple times, e.g. `--query "SELECT 1" --query "SELECT 2"`. Cannot be used simultaneously with `--queries-file`.
-- `--queries-file` – file path with queries to execute. `--queries-file` can be specified multiple times, e.g. `--query queries1.sql --query queries2.sql`. Cannot be used simultaneously with `--query`.
+- `--queries-file` – file path with queries to execute. `--queries-file` can be specified multiple times, e.g. `--queries-file  queries1.sql --queries-file  queries2.sql`. Cannot be used simultaneously with `--query`.
 - `--multiquery, -n` – If specified, multiple queries separated by semicolons can be listed after the `--query` option. For convenience, it is also possible to omit `--query` and pass the queries directly after `--multiquery`.
 - `--multiline, -m` – If specified, allow multiline queries (do not send the query on Enter).
 - `--database, -d` – Select the current default database. Default value: the current database from the server settings (‘default’ by default).

From 12cd566681143d017b9b4fbfc3ff5d11af7b4f1a Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Tue, 19 Mar 2024 11:47:53 +0100
Subject: [PATCH 75/92] Update config for test

---
 tests/integration/test_storage_s3_queue/configs/users.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/test_storage_s3_queue/configs/users.xml b/tests/integration/test_storage_s3_queue/configs/users.xml
index 1c096c7da8f..e47df8e4b94 100644
--- a/tests/integration/test_storage_s3_queue/configs/users.xml
+++ b/tests/integration/test_storage_s3_queue/configs/users.xml
@@ -3,6 +3,7 @@
         <default>
             <stream_like_engine_allow_direct_select>1</stream_like_engine_allow_direct_select>
             <s3queue_enable_logging_to_s3queue_log>1</s3queue_enable_logging_to_s3queue_log>
+            <s3queue_allow_experimental_sharded_mode>1</s3queue_allow_experimental_sharded_mode>
         </default>
     </profiles>
 </clickhouse>

From 118e30882ae2ed33f622dd1eb57f45440523cb67 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 19 Mar 2024 12:01:19 +0100
Subject: [PATCH 76/92] Fix test with token

---
 tests/queries/0_stateless/02481_async_insert_dedup.python | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02481_async_insert_dedup.python b/tests/queries/0_stateless/02481_async_insert_dedup.python
index 0e80a21bf46..24512836290 100644
--- a/tests/queries/0_stateless/02481_async_insert_dedup.python
+++ b/tests/queries/0_stateless/02481_async_insert_dedup.python
@@ -119,7 +119,7 @@ q = queue.Queue(100)
 total_number = 10000
 
 use_token = False
-if len(sys.argv) > 3 and sys.argv[2] == "token":
+if len(sys.argv) >= 3 and sys.argv[2] == "token":
     use_token = True
 
 gen = Thread(target=generate_data, args=[q, total_number, use_token])

From a6506e8e581759f188988daa6c57c07c3c012c86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 19 Mar 2024 12:05:13 +0100
Subject: [PATCH 77/92] Missed sending the process to background

---
 docker/test/fuzzer/run-fuzzer.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index e3b5a99018f..c9e1681401d 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -181,7 +181,7 @@ function fuzz
         --pid-file /var/run/clickhouse-server/clickhouse-server.pid \
         --  --path db \
             --logger.console=0 \
-            --logger.log=server.log > stderr.log 2>&1
+            --logger.log=server.log > stderr.log 2>&1 &
     server_pid=$!
 
     kill -0 $server_pid

From ba6d6e9258d86c9e83f3cc945c880ef29dbe1ead Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 19 Mar 2024 11:40:03 +0000
Subject: [PATCH 78/92] Add a comment after #61458

---
 src/Interpreters/ActionsDAG.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h
index bd56d20d0e8..6b0b098a4ba 100644
--- a/src/Interpreters/ActionsDAG.h
+++ b/src/Interpreters/ActionsDAG.h
@@ -336,8 +336,12 @@ public:
 
     /// Split ActionsDAG into two DAGs, where first part contains all nodes from split_nodes and their children.
     /// Execution of first then second parts on block is equivalent to execution of initial DAG.
-    /// First DAG and initial DAG have equal inputs, second DAG and initial DAG has equal outputs.
-    /// Second DAG inputs may contain less inputs then first DAG (but also include other columns).
+    /// Inputs and outputs of original DAG are splitted between the first and the second DAGs.
+    /// Intermediate result can apper in first outputs and second inputs.
+    /// Example:
+    ///   initial DAG    : (a, b, c, d, e) -> (w, x, y, z)  | 1 a 2 b 3 c 4 d 5 e 6      ->  1 2 3 4 5 6 w x y z
+    ///   split (first)  : (a, c, d) -> (i, j, k, w, y)     | 1 a 2 b 3 c 4 d 5 e 6      ->  1 2 b 3 4 5 e 6 i j k w y
+    ///   split (second) : (i, j, k, y, b, e) -> (x, y, z)  | 1 2 b 3 4 5 e 6 i j k w y  ->  1 2 3 4 5 6 w x y z
     SplitResult split(std::unordered_set<const Node *> split_nodes, bool create_split_nodes_mapping = false) const;
 
     /// Splits actions into two parts. Returned first half may be swapped with ARRAY JOIN.

From f2f770d73a002eb559af43f3e25d338b009c1daa Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 19 Mar 2024 13:37:21 +0100
Subject: [PATCH 79/92] Update src/Interpreters/ActionsDAG.h

Co-authored-by: Sergei Trifonov <sergei@clickhouse.com>
---
 src/Interpreters/ActionsDAG.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h
index 6b0b098a4ba..469fe9ea7f1 100644
--- a/src/Interpreters/ActionsDAG.h
+++ b/src/Interpreters/ActionsDAG.h
@@ -336,7 +336,7 @@ public:
 
     /// Split ActionsDAG into two DAGs, where first part contains all nodes from split_nodes and their children.
     /// Execution of first then second parts on block is equivalent to execution of initial DAG.
-    /// Inputs and outputs of original DAG are splitted between the first and the second DAGs.
+    /// Inputs and outputs of original DAG are split between the first and the second DAGs.
     /// Intermediate result can apper in first outputs and second inputs.
     /// Example:
     ///   initial DAG    : (a, b, c, d, e) -> (w, x, y, z)  | 1 a 2 b 3 c 4 d 5 e 6      ->  1 2 3 4 5 6 w x y z

From 3f46a5d93d5d7dee5ec0a81481c737ab9d7e4f7f Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 19 Mar 2024 13:01:25 +0000
Subject: [PATCH 80/92] Fix clickhouse-test client option and
 CLICKHOUSE_URL_PARAMS interference

---
 tests/clickhouse-test | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 057502379ed..13d1200758e 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -2900,7 +2900,7 @@ def get_additional_client_options(args):
     if args.client_option:
         client_options = " ".join("--" + option for option in args.client_option)
         if "CLICKHOUSE_CLIENT_OPT" in os.environ:
-            return os.environ["CLICKHOUSE_CLIENT_OPT"] + client_options
+            return os.environ["CLICKHOUSE_CLIENT_OPT"] + " " + client_options
         else:
             return client_options
     else:
@@ -3360,7 +3360,6 @@ if __name__ == "__main__":
         else:
             os.environ["CLICKHOUSE_CLIENT_OPT"] = ""
 
-        os.environ["CLICKHOUSE_CLIENT_OPT"] += get_additional_client_options(args)
         if args.secure:
             os.environ["CLICKHOUSE_CLIENT_OPT"] += " --secure "
 

From 4457c1a1330604756f1a5310f8331358de97eb1b Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 19 Mar 2024 14:07:25 +0100
Subject: [PATCH 81/92] Update analyzer_tech_debt.txt

---
 tests/analyzer_tech_debt.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/analyzer_tech_debt.txt b/tests/analyzer_tech_debt.txt
index 8335108e243..2f949396d0c 100644
--- a/tests/analyzer_tech_debt.txt
+++ b/tests/analyzer_tech_debt.txt
@@ -2,8 +2,8 @@
 00717_merge_and_distributed
 00725_memory_tracking
 01062_pm_all_join_with_block_continuation
+01083_expressions_in_engine_arguments
 01155_rename_move_materialized_view
-01584_distributed_buffer_cannot_find_column
 01624_soft_constraints
 01747_join_view_filter_dictionary
 01761_cast_to_enum_nullable

From 029488f336d44b58a3b685aaa9846a7d69a5c40d Mon Sep 17 00:00:00 2001
From: Peter <peter279k@gmail.com>
Date: Tue, 19 Mar 2024 23:54:08 +0800
Subject: [PATCH 82/92] Add words to ignore spell checking

---
 utils/check-style/aspell-ignore/en/aspell-dict.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt
index 57a8e0d5840..77aaacf0240 100644
--- a/utils/check-style/aspell-ignore/en/aspell-dict.txt
+++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt
@@ -162,6 +162,7 @@ ClickHouseVapor
 ClickVisual
 ClickableSquare
 CloudDetails
+CloudStorage
 CodeBlock
 CodeLLDB
 Codecs
@@ -460,6 +461,7 @@ LOCALTIME
 LOCALTIMESTAMP
 LONGLONG
 Levenshtein
+Liao
 LibFuzzer
 LightHouse
 LineAsString
@@ -957,6 +959,7 @@ TotalRowsOfMergeTreeTables
 TotalTemporaryFiles
 Tradeoff
 Transactional
+Tsai
 Tukey
 TwoColumnList
 UBSan

From 9da870864aaf4f30a34d348903c70a38f36e88a2 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Tue, 19 Mar 2024 16:02:05 +0100
Subject: [PATCH 83/92] Fix 01244_optimize_distributed_group_by_sharding_key by
 ordering output

---
 .../01244_optimize_distributed_group_by_sharding_key.reference  | 2 +-
 .../01244_optimize_distributed_group_by_sharding_key.sql        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.reference b/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.reference
index a4a6b87de25..9921a1eaac9 100644
--- a/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.reference
+++ b/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.reference
@@ -60,9 +60,9 @@ LIMIT OFFSET
 OFFSET distributed_push_down_limit=0
 1	1
 OFFSET distributed_push_down_limit=1
-1	1
 1	0
 1	1
+1	1
 WHERE LIMIT OFFSET
 1	1
 LIMIT BY 1
diff --git a/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.sql b/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.sql
index e80f41b173c..63dbd9c2cc9 100644
--- a/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.sql
+++ b/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.sql
@@ -69,7 +69,7 @@ select count(), * from dist_01247 group by number limit 1 offset 1;
 select 'OFFSET distributed_push_down_limit=0';
 select count(), * from dist_01247 group by number offset 1 settings distributed_push_down_limit=0;
 select 'OFFSET distributed_push_down_limit=1';
-select count(), * from dist_01247 group by number offset 1 settings distributed_push_down_limit=1;
+select count(), * from dist_01247 group by number order by count(), number offset 1 settings distributed_push_down_limit=1;
 -- this will emulate different data on for different shards
 select 'WHERE LIMIT OFFSET';
 select count(), * from dist_01247 where number = _shard_num-1 group by number order by number limit 1 offset 1;

From ec6c276a92a3f9d340e3fd1ac80a2db06c3e7ccb Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Mon, 18 Mar 2024 18:02:54 +0300
Subject: [PATCH 84/92] Temporary data use temporary_files_codec setting

---
 src/Compression/CompressionFactory.cpp        |  8 +++++
 src/Compression/CompressionFactory.h          |  3 ++
 src/Core/Settings.h                           |  2 +-
 src/Interpreters/Context.cpp                  | 15 ++++++---
 src/Interpreters/ProcessList.cpp              | 18 ++++++++--
 src/Interpreters/TemporaryDataOnDisk.cpp      | 24 +++++++++-----
 src/Interpreters/TemporaryDataOnDisk.h        | 33 ++++++++++++++-----
 src/Interpreters/tests/gtest_filecache.cpp    |  6 ++--
 .../MergeTreeDataPartWriterCompact.cpp        |  9 +----
 .../MergeTreeDataPartWriterOnDisk.cpp         |  4 +--
 10 files changed, 84 insertions(+), 38 deletions(-)

diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp
index f4413401667..68e0131c91b 100644
--- a/src/Compression/CompressionFactory.cpp
+++ b/src/Compression/CompressionFactory.cpp
@@ -7,6 +7,8 @@
 #include <Poco/String.h>
 #include <IO/ReadBuffer.h>
 #include <Parsers/queryToString.h>
+#include <Parsers/parseQuery.h>
+#include <Parsers/ExpressionElementParsers.h>
 #include <Compression/CompressionCodecMultiple.h>
 #include <Compression/CompressionCodecNone.h>
 #include <IO/WriteHelpers.h>
@@ -44,6 +46,12 @@ CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std
     }
 }
 
+CompressionCodecPtr CompressionCodecFactory::get(const String & compression_codec) const
+{
+    ParserCodec codec_parser;
+    auto ast = parseQuery(codec_parser, "(" + Poco::toUpper(compression_codec) + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
+    return CompressionCodecFactory::instance().get(ast, nullptr);
+}
 
 CompressionCodecPtr CompressionCodecFactory::get(
     const ASTPtr & ast, const IDataType * column_type, CompressionCodecPtr current_default, bool only_generic) const
diff --git a/src/Compression/CompressionFactory.h b/src/Compression/CompressionFactory.h
index e71476d564d..2885f35d7bd 100644
--- a/src/Compression/CompressionFactory.h
+++ b/src/Compression/CompressionFactory.h
@@ -68,6 +68,9 @@ public:
     /// For backward compatibility with config settings
     CompressionCodecPtr get(const String & family_name, std::optional<int> level) const;
 
+    /// Get codec by name with optional params. Example: LZ4, ZSTD(3)
+    CompressionCodecPtr get(const String & compression_codec) const;
+
     /// Register codec with parameters and column type
     void registerCompressionCodecWithType(const String & family_name, std::optional<uint8_t> byte_code, CreatorWithType creator);
     /// Register codec with parameters
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index a3c5638d97f..a04a8e430e9 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -450,7 +450,7 @@ class IColumn;
     \
     M(Bool, compatibility_ignore_collation_in_create_table, true, "Compatibility ignore collation in create table", 0) \
     \
-    M(String, temporary_files_codec, "LZ4", "Set compression codec for temporary files (sort and join on disk). I.e. LZ4, NONE.", 0) \
+    M(String, temporary_files_codec, "LZ4", "Set compression codec for temporary files produced by (JOINs, external GROUP BY, external ORDER BY). I.e. LZ4, NONE.", 0) \
     \
     M(UInt64, max_rows_to_transfer, 0, "Maximum size (in rows) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \
     M(UInt64, max_bytes_to_transfer, 0, "Maximum size (in uncompressed bytes) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index 6a0657a842c..038b9712b0c 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -1073,7 +1073,9 @@ void Context::setTemporaryStoragePath(const String & path, size_t max_size)
         setupTmpPath(shared->log, disk->getPath());
     }
 
-    shared->root_temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(volume, max_size);
+    TemporaryDataOnDiskSettings temporary_data_on_disk_settings;
+    temporary_data_on_disk_settings.max_size_on_disk = max_size;
+    shared->root_temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(std::move(volume), std::move(temporary_data_on_disk_settings));
 }
 
 void Context::setTemporaryStoragePolicy(const String & policy_name, size_t max_size)
@@ -1093,7 +1095,7 @@ void Context::setTemporaryStoragePolicy(const String & policy_name, size_t max_s
     VolumePtr volume = tmp_policy->getVolume(0);
 
     if (volume->getDisks().empty())
-         throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "No disks volume for temporary files");
+        throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "No disks volume for temporary files");
 
     for (const auto & disk : volume->getDisks())
     {
@@ -1119,7 +1121,9 @@ void Context::setTemporaryStoragePolicy(const String & policy_name, size_t max_s
     if (shared->root_temp_data_on_disk)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary storage is already set");
 
-    shared->root_temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(volume, max_size);
+    TemporaryDataOnDiskSettings temporary_data_on_disk_settings;
+    temporary_data_on_disk_settings.max_size_on_disk = max_size;
+    shared->root_temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(std::move(volume), std::move(temporary_data_on_disk_settings));
 }
 
 void Context::setTemporaryStorageInCache(const String & cache_disk_name, size_t max_size)
@@ -1140,7 +1144,10 @@ void Context::setTemporaryStorageInCache(const String & cache_disk_name, size_t
 
     shared->tmp_path = file_cache->getBasePath();
     VolumePtr volume = createLocalSingleDiskVolume(shared->tmp_path, shared->getConfigRefWithLock(lock));
-    shared->root_temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(volume, file_cache.get(), max_size);
+
+    TemporaryDataOnDiskSettings temporary_data_on_disk_settings;
+    temporary_data_on_disk_settings.max_size_on_disk = max_size;
+    shared->root_temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(std::move(volume), file_cache.get(), std::move(temporary_data_on_disk_settings));
 }
 
 void Context::setFlagsPath(const String & path)
diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp
index 3bd7b2d4206..889ad79f4ef 100644
--- a/src/Interpreters/ProcessList.cpp
+++ b/src/Interpreters/ProcessList.cpp
@@ -208,8 +208,13 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q
             thread_group->memory_tracker.setParent(&user_process_list.user_memory_tracker);
             if (user_process_list.user_temp_data_on_disk)
             {
+                TemporaryDataOnDiskSettings temporary_data_on_disk_settings
+                {
+                    .max_size_on_disk = settings.max_temporary_data_on_disk_size_for_query,
+                    .compression_codec = settings.temporary_files_codec
+                };
                 query_context->setTempDataOnDisk(std::make_shared<TemporaryDataOnDiskScope>(
-                    user_process_list.user_temp_data_on_disk, settings.max_temporary_data_on_disk_size_for_query));
+                    user_process_list.user_temp_data_on_disk, std::move(temporary_data_on_disk_settings)));
             }
 
             /// Set query-level memory trackers
@@ -682,8 +687,15 @@ ProcessListForUser::ProcessListForUser(ContextPtr global_context, ProcessList *
 
     if (global_context)
     {
-        size_t size_limit = global_context->getSettingsRef().max_temporary_data_on_disk_size_for_user;
-        user_temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(global_context->getSharedTempDataOnDisk(), size_limit);
+        auto & settings = global_context->getSettingsRef();
+        TemporaryDataOnDiskSettings temporary_data_on_disk_settings
+        {
+            .max_size_on_disk = settings.max_temporary_data_on_disk_size_for_user,
+            .compression_codec = settings.temporary_files_codec
+        };
+
+        user_temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(global_context->getSharedTempDataOnDisk(),
+            std::move(temporary_data_on_disk_settings));
     }
 }
 
diff --git a/src/Interpreters/TemporaryDataOnDisk.cpp b/src/Interpreters/TemporaryDataOnDisk.cpp
index a48e7d8e040..4a30c0ae726 100644
--- a/src/Interpreters/TemporaryDataOnDisk.cpp
+++ b/src/Interpreters/TemporaryDataOnDisk.cpp
@@ -44,20 +44,20 @@ void TemporaryDataOnDiskScope::deltaAllocAndCheck(ssize_t compressed_delta, ssiz
     }
 
     size_t new_consumprion = stat.compressed_size + compressed_delta;
-    if (compressed_delta > 0 && limit && new_consumprion > limit)
+    if (compressed_delta > 0 && settings.max_size_on_disk && new_consumprion > settings.max_size_on_disk)
         throw Exception(ErrorCodes::TOO_MANY_ROWS_OR_BYTES,
-            "Limit for temporary files size exceeded (would consume {} / {} bytes)", new_consumprion, limit);
+            "Limit for temporary files size exceeded (would consume {} / {} bytes)", new_consumprion, settings.max_size_on_disk);
 
     stat.compressed_size += compressed_delta;
     stat.uncompressed_size += uncompressed_delta;
 }
 
 TemporaryDataOnDisk::TemporaryDataOnDisk(TemporaryDataOnDiskScopePtr parent_)
-    : TemporaryDataOnDiskScope(std::move(parent_), /* limit_ = */ 0)
+    : TemporaryDataOnDiskScope(parent_, parent_->getSettings())
 {}
 
 TemporaryDataOnDisk::TemporaryDataOnDisk(TemporaryDataOnDiskScopePtr parent_, CurrentMetrics::Metric metric_scope)
-    : TemporaryDataOnDiskScope(std::move(parent_), /* limit_ = */ 0)
+    : TemporaryDataOnDiskScope(parent_, parent_->getSettings())
     , current_metric_scope(metric_scope)
 {}
 
@@ -153,11 +153,19 @@ bool TemporaryDataOnDisk::empty() const
     return streams.empty();
 }
 
+static inline CompressionCodecPtr getCodec(const TemporaryDataOnDiskSettings & settings)
+{
+    if (settings.compression_codec.empty())
+        return CompressionCodecFactory::instance().get("NONE");
+
+    return CompressionCodecFactory::instance().get(settings.compression_codec);
+}
+
 struct TemporaryFileStream::OutputWriter
 {
-    OutputWriter(std::unique_ptr<WriteBuffer> out_buf_, const Block & header_)
+    OutputWriter(std::unique_ptr<WriteBuffer> out_buf_, const Block & header_, const TemporaryDataOnDiskSettings & settings)
         : out_buf(std::move(out_buf_))
-        , out_compressed_buf(*out_buf)
+        , out_compressed_buf(*out_buf, getCodec(settings))
         , out_writer(out_compressed_buf, DBMS_TCP_PROTOCOL_VERSION, header_)
     {
     }
@@ -248,7 +256,7 @@ TemporaryFileStream::TemporaryFileStream(TemporaryFileOnDiskHolder file_, const
     : parent(parent_)
     , header(header_)
     , file(std::move(file_))
-    , out_writer(std::make_unique<OutputWriter>(std::make_unique<WriteBufferFromFile>(file->getAbsolutePath()), header))
+    , out_writer(std::make_unique<OutputWriter>(std::make_unique<WriteBufferFromFile>(file->getAbsolutePath()), header, parent->settings))
 {
     LOG_TEST(getLogger("TemporaryFileStream"), "Writing to temporary file {}", file->getAbsolutePath());
 }
@@ -263,7 +271,7 @@ TemporaryFileStream::TemporaryFileStream(FileSegmentsHolderPtr segments_, const
     auto out_buf = std::make_unique<WriteBufferToFileSegment>(&segment_holder->front());
 
     LOG_TEST(getLogger("TemporaryFileStream"), "Writing to temporary file {}", out_buf->getFileName());
-    out_writer = std::make_unique<OutputWriter>(std::move(out_buf), header);
+    out_writer = std::make_unique<OutputWriter>(std::move(out_buf), header, parent_->settings);
 }
 
 size_t TemporaryFileStream::write(const Block & block)
diff --git a/src/Interpreters/TemporaryDataOnDisk.h b/src/Interpreters/TemporaryDataOnDisk.h
index 8b0649be1b1..40100a62b44 100644
--- a/src/Interpreters/TemporaryDataOnDisk.h
+++ b/src/Interpreters/TemporaryDataOnDisk.h
@@ -28,6 +28,15 @@ using TemporaryFileStreamPtr = std::unique_ptr<TemporaryFileStream>;
 
 class FileCache;
 
+struct TemporaryDataOnDiskSettings
+{
+    /// Max size on disk, if 0 there will be no limit
+    size_t max_size_on_disk = 0;
+
+    /// Compression codec for temporary data, if empty no compression will be used. LZ4 by default
+    String compression_codec = "LZ4";
+};
+
 /*
  * Used to account amount of temporary data written to disk.
  * If limit is set, throws exception if limit is exceeded.
@@ -43,22 +52,30 @@ public:
         std::atomic<size_t> uncompressed_size;
     };
 
-    explicit TemporaryDataOnDiskScope(VolumePtr volume_, size_t limit_)
-        : volume(std::move(volume_)), limit(limit_)
+    explicit TemporaryDataOnDiskScope(VolumePtr volume_, TemporaryDataOnDiskSettings settings_)
+        : volume(std::move(volume_))
+        , settings(std::move(settings_))
     {}
 
-    explicit TemporaryDataOnDiskScope(VolumePtr volume_, FileCache * file_cache_, size_t limit_)
-        : volume(std::move(volume_)), file_cache(file_cache_), limit(limit_)
+    explicit TemporaryDataOnDiskScope(VolumePtr volume_, FileCache * file_cache_, TemporaryDataOnDiskSettings settings_)
+        : volume(std::move(volume_))
+        , file_cache(file_cache_)
+        , settings(std::move(settings_))
     {}
 
-    explicit TemporaryDataOnDiskScope(TemporaryDataOnDiskScopePtr parent_, size_t limit_)
-        : parent(std::move(parent_)), volume(parent->volume), file_cache(parent->file_cache), limit(limit_)
+    explicit TemporaryDataOnDiskScope(TemporaryDataOnDiskScopePtr parent_, TemporaryDataOnDiskSettings settings_)
+        : parent(std::move(parent_))
+        , volume(parent->volume)
+        , file_cache(parent->file_cache)
+        , settings(std::move(settings_))
     {}
 
     /// TODO: remove
     /// Refactor all code that uses volume directly to use TemporaryDataOnDisk.
     VolumePtr getVolume() const { return volume; }
 
+    const TemporaryDataOnDiskSettings & getSettings() const { return settings; }
+
 protected:
     void deltaAllocAndCheck(ssize_t compressed_delta, ssize_t uncompressed_delta);
 
@@ -68,14 +85,14 @@ protected:
     FileCache * file_cache = nullptr;
 
     StatAtomic stat;
-    size_t limit = 0;
+    const TemporaryDataOnDiskSettings settings;
 };
 
 /*
  * Holds the set of temporary files.
  * New file stream is created with `createStream`.
  * Streams are owned by this object and will be deleted when it is deleted.
- * It's a leaf node in temorarty data scope tree.
+ * It's a leaf node in temporary data scope tree.
  */
 class TemporaryDataOnDisk : private TemporaryDataOnDiskScope
 {
diff --git a/src/Interpreters/tests/gtest_filecache.cpp b/src/Interpreters/tests/gtest_filecache.cpp
index 2679d1b8d18..09afb01d7ff 100644
--- a/src/Interpreters/tests/gtest_filecache.cpp
+++ b/src/Interpreters/tests/gtest_filecache.cpp
@@ -947,7 +947,7 @@ TEST_F(FileCacheTest, temporaryData)
     file_cache.initialize();
 
     const auto user = FileCache::getCommonUser();
-    auto tmp_data_scope = std::make_shared<TemporaryDataOnDiskScope>(nullptr, &file_cache, 0);
+    auto tmp_data_scope = std::make_shared<TemporaryDataOnDiskScope>(nullptr, &file_cache, TemporaryDataOnDiskSettings{});
 
     auto some_data_holder = file_cache.getOrSet(file_cache.createKeyForPath("some_data"), 0, 5_KiB, 5_KiB, CreateFileSegmentSettings{}, 0, user);
 
@@ -1130,7 +1130,7 @@ TEST_F(FileCacheTest, TemporaryDataReadBufferSize)
         DB::FileCache file_cache("cache", settings);
         file_cache.initialize();
 
-        auto tmp_data_scope = std::make_shared<TemporaryDataOnDiskScope>(/*volume=*/nullptr, &file_cache, /*limit=*/0);
+        auto tmp_data_scope = std::make_shared<TemporaryDataOnDiskScope>(/*volume=*/nullptr, &file_cache, /*settings=*/TemporaryDataOnDiskSettings{});
 
         auto tmp_data = std::make_unique<TemporaryDataOnDisk>(tmp_data_scope);
 
@@ -1152,7 +1152,7 @@ TEST_F(FileCacheTest, TemporaryDataReadBufferSize)
         disk = createDisk("temporary_data_read_buffer_size_test_dir");
         VolumePtr volume = std::make_shared<SingleDiskVolume>("volume", disk);
 
-        auto tmp_data_scope = std::make_shared<TemporaryDataOnDiskScope>(/*volume=*/volume, /*cache=*/nullptr, /*limit=*/0);
+        auto tmp_data_scope = std::make_shared<TemporaryDataOnDiskScope>(/*volume=*/volume, /*cache=*/nullptr, /*settings=*/TemporaryDataOnDiskSettings{});
 
         auto tmp_data = std::make_unique<TemporaryDataOnDisk>(tmp_data_scope);
 
diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp
index fe45d0bee54..1605e5cdb9a 100644
--- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp
@@ -9,13 +9,6 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-static CompressionCodecPtr getMarksCompressionCodec(const String & marks_compression_codec)
-{
-    ParserCodec codec_parser;
-    auto ast = parseQuery(codec_parser, "(" + Poco::toUpper(marks_compression_codec) + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
-    return CompressionCodecFactory::instance().get(ast, nullptr);
-}
-
 MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact(
     const MergeTreeMutableDataPartPtr & data_part_,
     const NamesAndTypesList & columns_list_,
@@ -46,7 +39,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact(
     {
         marks_compressor = std::make_unique<CompressedWriteBuffer>(
             *marks_file_hashing,
-            getMarksCompressionCodec(settings_.marks_compression_codec),
+             CompressionCodecFactory::instance().get(settings_.marks_compression_codec),
             settings_.marks_compress_block_size);
 
         marks_source_hashing = std::make_unique<HashingWriteBuffer>(*marks_compressor);
diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp
index a31da5bc4fe..5a82357c6eb 100644
--- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp
@@ -242,9 +242,7 @@ void MergeTreeDataPartWriterOnDisk::initPrimaryIndex()
 
         if (compress_primary_key)
         {
-            ParserCodec codec_parser;
-            auto ast = parseQuery(codec_parser, "(" + Poco::toUpper(settings.primary_key_compression_codec) + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
-            CompressionCodecPtr primary_key_compression_codec = CompressionCodecFactory::instance().get(ast, nullptr);
+            CompressionCodecPtr primary_key_compression_codec = CompressionCodecFactory::instance().get(settings.primary_key_compression_codec);
             index_compressor_stream = std::make_unique<CompressedWriteBuffer>(*index_file_hashing_stream, primary_key_compression_codec, settings.primary_key_compress_block_size);
             index_source_hashing_stream = std::make_unique<HashingWriteBuffer>(*index_compressor_stream);
         }

From f4d20a79b08525a3d39716b9a95b6f3ae6058817 Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Tue, 19 Mar 2024 19:07:02 +0300
Subject: [PATCH 85/92] Fixed build

---
 src/Interpreters/ProcessList.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp
index 889ad79f4ef..0e2a6e14f07 100644
--- a/src/Interpreters/ProcessList.cpp
+++ b/src/Interpreters/ProcessList.cpp
@@ -687,7 +687,7 @@ ProcessListForUser::ProcessListForUser(ContextPtr global_context, ProcessList *
 
     if (global_context)
     {
-        auto & settings = global_context->getSettingsRef();
+        const auto & settings = global_context->getSettingsRef();
         TemporaryDataOnDiskSettings temporary_data_on_disk_settings
         {
             .max_size_on_disk = settings.max_temporary_data_on_disk_size_for_user,

From 574647ed2a9e78185a9dcf9d104d6f73d2588700 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Tue, 19 Mar 2024 17:16:53 +0100
Subject: [PATCH 86/92] Update config for another test

---
 tests/integration/test_mask_sensitive_info/configs/users.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/test_mask_sensitive_info/configs/users.xml b/tests/integration/test_mask_sensitive_info/configs/users.xml
index f129a5bb3e3..54ee9b2c9db 100644
--- a/tests/integration/test_mask_sensitive_info/configs/users.xml
+++ b/tests/integration/test_mask_sensitive_info/configs/users.xml
@@ -2,6 +2,7 @@
     <profiles>
         <default>
             <s3_retry_attempts>5</s3_retry_attempts>
+            <s3queue_allow_experimental_sharded_mode>1</s3queue_allow_experimental_sharded_mode>
         </default>
     </profiles>
     <users>

From 79bbcbdf1e438d98298499d0c007c7b52b979667 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Tue, 19 Mar 2024 17:20:31 +0100
Subject: [PATCH 87/92] Update StorageS3Queue.cpp

---
 src/Storages/S3Queue/StorageS3Queue.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/S3Queue/StorageS3Queue.cpp b/src/Storages/S3Queue/StorageS3Queue.cpp
index 048816fe4f1..bf4d3759d8e 100644
--- a/src/Storages/S3Queue/StorageS3Queue.cpp
+++ b/src/Storages/S3Queue/StorageS3Queue.cpp
@@ -133,7 +133,7 @@ StorageS3Queue::StorageS3Queue(
     if (mode == LoadingStrictnessLevel::CREATE
         && !context_->getSettingsRef().s3queue_allow_experimental_sharded_mode
         && s3queue_settings->mode == S3QueueMode::ORDERED
-        && s3queue_settings->s3queue_total_shards_num)
+        && (s3queue_settings->s3queue_total_shards_num > 1 || s3queue_settings->s3queue_processing_threads_num > 1))
     {
         throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "S3Queue sharded mode is not allowed. To enable use `s3queue_allow_experimental_sharded_mode`");
     }

From 7ebab45beff7bdea5c2dbef2e1aaa568c7c7fbca Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Tue, 19 Mar 2024 17:21:03 +0100
Subject: [PATCH 88/92] Update users.xml

---
 tests/integration/test_mask_sensitive_info/configs/users.xml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/integration/test_mask_sensitive_info/configs/users.xml b/tests/integration/test_mask_sensitive_info/configs/users.xml
index 54ee9b2c9db..f129a5bb3e3 100644
--- a/tests/integration/test_mask_sensitive_info/configs/users.xml
+++ b/tests/integration/test_mask_sensitive_info/configs/users.xml
@@ -2,7 +2,6 @@
     <profiles>
         <default>
             <s3_retry_attempts>5</s3_retry_attempts>
-            <s3queue_allow_experimental_sharded_mode>1</s3queue_allow_experimental_sharded_mode>
         </default>
     </profiles>
     <users>

From 301037ca6a69507418c865ab33123b21426faa5d Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Tue, 19 Mar 2024 17:53:32 +0100
Subject: [PATCH 89/92] Replace `cat | sed; mv` by `sed -i`

---
 docker/test/stateless/run.sh | 30 ++++++------------------------
 docker/test/upgrade/run.sh   | 20 ++++----------------
 2 files changed, 10 insertions(+), 40 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index f4de7677012..e531a9928ac 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -51,10 +51,7 @@ fi
 config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml
 
 if [[ -n "$BUGFIX_VALIDATE_CHECK" ]] && [[ "$BUGFIX_VALIDATE_CHECK" -eq 1 ]]; then
-    sudo cat /etc/clickhouse-server/config.d/zookeeper.xml \
-    | sed "/<use_compression>1<\/use_compression>/d" \
-    > /etc/clickhouse-server/config.d/zookeeper.xml.tmp
-    sudo mv /etc/clickhouse-server/config.d/zookeeper.xml.tmp /etc/clickhouse-server/config.d/zookeeper.xml
+    sudo sed -i "/<use_compression>1<\/use_compression>/d" /etc/clickhouse-server/config.d/zookeeper.xml
 
     # it contains some new settings, but we can safely remove it
     rm /etc/clickhouse-server/config.d/handlers.yaml
@@ -75,10 +72,7 @@ if [[ -n "$BUGFIX_VALIDATE_CHECK" ]] && [[ "$BUGFIX_VALIDATE_CHECK" -eq 1 ]]; th
 
     function remove_keeper_config()
     {
-        sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
-          | sed "/<$1>$2<\/$1>/d" \
-          > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
-        sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
+        sudo sed -i "/<$1>$2<\/$1>/d" /etc/clickhouse-server/config.d/keeper_port.xml
     }
     # commit_logs_cache_size_threshold setting doesn't exist on some older versions
     remove_keeper_config "commit_logs_cache_size_threshold" "[[:digit:]]\+"
@@ -113,25 +107,13 @@ else
 fi
 
 if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
-    sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \
-    | sed "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_1/</filesystem_caches_path>|" \
-    > /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp
-    mv /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
+    sudo sed -i "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_1/</filesystem_caches_path>|" /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
 
-    sudo cat /etc/clickhouse-server2/config.d/filesystem_caches_path.xml \
-    | sed "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_2/</filesystem_caches_path>|" \
-    > /etc/clickhouse-server2/config.d/filesystem_caches_path.xml.tmp
-    mv /etc/clickhouse-server2/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server2/config.d/filesystem_caches_path.xml
+    sudo sed -i "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_2/</filesystem_caches_path>|" /etc/clickhouse-server2/config.d/filesystem_caches_path.xml
 
-    sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \
-    | sed "s|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches/</custom_cached_disks_base_directory>|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches_1/</custom_cached_disks_base_directory>|" \
-    > /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp
-    mv /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
+    sudo sed -i "s|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches/</custom_cached_disks_base_directory>|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches_1/</custom_cached_disks_base_directory>|" /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
 
-    sudo cat /etc/clickhouse-server2/config.d/filesystem_caches_path.xml \
-    | sed "s|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches/</custom_cached_disks_base_directory>|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches_2/</custom_cached_disks_base_directory>|" \
-    > /etc/clickhouse-server2/config.d/filesystem_caches_path.xml.tmp
-    mv /etc/clickhouse-server2/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server2/config.d/filesystem_caches_path.xml
+    sudo sed -i "s|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches/</custom_cached_disks_base_directory>|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches_2/</custom_cached_disks_base_directory>|" /etc/clickhouse-server2/config.d/filesystem_caches_path.xml
 
     mkdir -p /var/run/clickhouse-server1
     sudo chown clickhouse:clickhouse /var/run/clickhouse-server1
diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh
index 69228b1bfc9..325fc274cf7 100644
--- a/docker/test/upgrade/run.sh
+++ b/docker/test/upgrade/run.sh
@@ -67,10 +67,7 @@ configure
 
 function remove_keeper_config()
 {
-  sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
-    | sed "/<$1>$2<\/$1>/d" \
-    > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
-  sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
+  sudo sed -i "/<$1>$2<\/$1>/d" /etc/clickhouse-server/config.d/keeper_port.xml
 }
 
 # async_replication setting doesn't exist on some older versions
@@ -120,10 +117,7 @@ export ZOOKEEPER_FAULT_INJECTION=0
 configure
 
 # force_sync=false doesn't work correctly on some older versions
-sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
-  | sed "s|<force_sync>false</force_sync>|<force_sync>true</force_sync>|" \
-  > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
-sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
+sudo sed -i "s|<force_sync>false</force_sync>|<force_sync>true</force_sync>|" /etc/clickhouse-server/config.d/keeper_port.xml
 
 #todo: remove these after 24.3 released.
 sudo cat /etc/clickhouse-server/config.d/azure_storage_conf.xml \
@@ -150,10 +144,7 @@ remove_keeper_config "latest_logs_cache_size_threshold" "[[:digit:]]\+"
 remove_keeper_config "commit_logs_cache_size_threshold" "[[:digit:]]\+"
 
 # But we still need default disk because some tables loaded only into it
-sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \
-  | sed "s|<main><disk>s3</disk></main>|<main><disk>s3</disk></main><default><disk>default</disk></default>|" \
-  > /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp
-mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
+sudo sed -i "s|<main><disk>s3</disk></main>|<main><disk>s3</disk></main><default><disk>default</disk></default>|" /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
 sudo chown clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
 sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
 
@@ -256,10 +247,7 @@ then
 fi
 
 # Just in case previous version left some garbage in zk
-sudo cat /etc/clickhouse-server/config.d/lost_forever_check.xml \
-  | sed "s|>1<|>0<|g" \
-  > /etc/clickhouse-server/config.d/lost_forever_check.xml.tmp
-sudo mv /etc/clickhouse-server/config.d/lost_forever_check.xml.tmp /etc/clickhouse-server/config.d/lost_forever_check.xml
+sudo sed -i "s|>1<|>0<|g" /etc/clickhouse-server/config.d/lost_forever_check.xml \
 rm /etc/clickhouse-server/config.d/filesystem_caches_path.xml
 
 start 500

From fce926446bac1f1de0591c1efb6fbf3539015b67 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Tue, 19 Mar 2024 18:05:57 +0100
Subject: [PATCH 90/92] Fix sed to not produce local_blob_storage_blob_storage

---
 docker/test/stateless/run.sh | 10 ++--------
 docker/test/upgrade/run.sh   | 20 ++++----------------
 2 files changed, 6 insertions(+), 24 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index e531a9928ac..bac9d8df7a9 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -59,16 +59,10 @@ if [[ -n "$BUGFIX_VALIDATE_CHECK" ]] && [[ "$BUGFIX_VALIDATE_CHECK" -eq 1 ]]; th
     rm /etc/clickhouse-server/config.d/zero_copy_destructive_operations.xml
 
     #todo: remove these after 24.3 released.
-    sudo cat /etc/clickhouse-server/config.d/azure_storage_conf.xml \
-      | sed "s|<object_storage_type>azure|<object_storage_type>azure_blob_storage|" \
-      > /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp
-    sudo mv /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp /etc/clickhouse-server/config.d/azure_storage_conf.xml
+    sudo sed -i "s|<object_storage_type>azure<|<object_storage_type>azure_blob_storage<|" /etc/clickhouse-server/config.d/azure_storage_conf.xml
 
     #todo: remove these after 24.3 released.
-    sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \
-      | sed "s|<object_storage_type>local|<object_storage_type>local_blob_storage|" \
-      > /etc/clickhouse-server/config.d/storage_conf.xml.tmp
-    sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml
+    sudo sed -i "s|<object_storage_type>local<|<object_storage_type>local_blob_storage<|" /etc/clickhouse-server/config.d/storage_conf.xml
 
     function remove_keeper_config()
     {
diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh
index 325fc274cf7..6761ddba3e5 100644
--- a/docker/test/upgrade/run.sh
+++ b/docker/test/upgrade/run.sh
@@ -77,16 +77,10 @@ remove_keeper_config "async_replication" "1"
 remove_keeper_config "create_if_not_exists" "[01]"
 
 #todo: remove these after 24.3 released.
-sudo cat /etc/clickhouse-server/config.d/azure_storage_conf.xml \
-  | sed "s|<object_storage_type>azure|<object_storage_type>azure_blob_storage|" \
-  > /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp
-sudo mv /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp /etc/clickhouse-server/config.d/azure_storage_conf.xml
+sudo sed -i "s|<object_storage_type>azure<|<object_storage_type>azure_blob_storage<|" /etc/clickhouse-server/config.d/azure_storage_conf.xml
 
 #todo: remove these after 24.3 released.
-sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \
-  | sed "s|<object_storage_type>local|<object_storage_type>local_blob_storage|" \
-  > /etc/clickhouse-server/config.d/storage_conf.xml.tmp
-sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml
+sudo sed -i "s|<object_storage_type>local<|<object_storage_type>local_blob_storage<|" /etc/clickhouse-server/config.d/storage_conf.xml
 
 # latest_logs_cache_size_threshold setting doesn't exist on some older versions
 remove_keeper_config "latest_logs_cache_size_threshold" "[[:digit:]]\+"
@@ -120,16 +114,10 @@ configure
 sudo sed -i "s|<force_sync>false</force_sync>|<force_sync>true</force_sync>|" /etc/clickhouse-server/config.d/keeper_port.xml
 
 #todo: remove these after 24.3 released.
-sudo cat /etc/clickhouse-server/config.d/azure_storage_conf.xml \
-  | sed "s|<object_storage_type>azure|<object_storage_type>azure_blob_storage|" \
-  > /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp
-sudo mv /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp /etc/clickhouse-server/config.d/azure_storage_conf.xml
+sudo sed -i "s|<object_storage_type>azure<|<object_storage_type>azure_blob_storage<|" /etc/clickhouse-server/config.d/azure_storage_conf.xml
 
 #todo: remove these after 24.3 released.
-sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \
-  | sed "s|<object_storage_type>local|<object_storage_type>local_blob_storage|" \
-  > /etc/clickhouse-server/config.d/storage_conf.xml.tmp
-sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml
+sudo sed -i "s|<object_storage_type>local<|<object_storage_type>local_blob_storage<|" /etc/clickhouse-server/config.d/storage_conf.xml
 
 # async_replication setting doesn't exist on some older versions
 remove_keeper_config "async_replication" "1"

From 764c72b6c9bc3c54d7e505514aef8b8c4c7a5fd7 Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Tue, 19 Mar 2024 20:41:39 +0300
Subject: [PATCH 91/92] Fixed code review issues

---
 tests/integration/test_temporary_data_in_cache/test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/test_temporary_data_in_cache/test.py b/tests/integration/test_temporary_data_in_cache/test.py
index ed06a70cf5a..5eed7563d9b 100644
--- a/tests/integration/test_temporary_data_in_cache/test.py
+++ b/tests/integration/test_temporary_data_in_cache/test.py
@@ -67,6 +67,7 @@ def test_cache_evicted_by_temporary_data(start_cluster):
             settings={
                 "max_bytes_before_external_group_by": "4M",
                 "max_bytes_before_external_sort": "4M",
+                "temporary_files_codec": "ZSTD"
             },
         )
     assert fnmatch.fnmatch(

From b19540f5b6f6b229a4c49353a01d8cc8921b5602 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 19 Mar 2024 19:03:20 +0100
Subject: [PATCH 92/92] Fix style
 tests/integration/test_temporary_data_in_cache/test.py

---
 tests/integration/test_temporary_data_in_cache/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_temporary_data_in_cache/test.py b/tests/integration/test_temporary_data_in_cache/test.py
index 5eed7563d9b..cab134dcce2 100644
--- a/tests/integration/test_temporary_data_in_cache/test.py
+++ b/tests/integration/test_temporary_data_in_cache/test.py
@@ -67,7 +67,7 @@ def test_cache_evicted_by_temporary_data(start_cluster):
             settings={
                 "max_bytes_before_external_group_by": "4M",
                 "max_bytes_before_external_sort": "4M",
-                "temporary_files_codec": "ZSTD"
+                "temporary_files_codec": "ZSTD",
             },
         )
     assert fnmatch.fnmatch(